++ echo 'Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/logs/split-horizon-manual-tls.log' Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/logs/split-horizon-manual-tls.log ++ '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/cloud-secret.yml ']' ++ SKIP_BACKUPS_TO_AWS_GCP_AZURE= ++ oc get projects ++ kubectl get nodes ++ grep '^minikube' +++ kubectl version -o json +++ jq -r .serverVersion.gitVersion +++ grep -eks- Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1 ++ '[' ']' ++ EKS=0 +++ kubectl version -o json +++ jq -r .serverVersion.gitVersion +++ grep gke Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1 ++ '[' v1.33.11-gke.1074000 ']' ++ GKE=1 +++ kubectl version -o json +++ jq -r '.serverVersion.major + "." + .serverVersion.minor' +++ /usr/sbin/sed -r 's/[^0-9.]+//g' Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1 ++ KUBE_VERSION=1.33 + main + create_infra split-horizon-manual-tls-24334 + local ns=split-horizon-manual-tls-24334 + echo split-horizon-manual-tls-24334 + [[ 0 == 1 ]] + '[' -n psmdb-operator ']' + create_namespace psmdb-operator + local namespace=psmdb-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ sed s/NAMESPACE// ++ tail -n1 ++ awk '-F ' '{print $2}' + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get crd ++ awk '{print $1}' ++ grep chaos-mesh.org + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + '[' -n '' ']' + desc 'cleaned up old namespaces psmdb-operator' + grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces psmdb-operator ----------------------------------------------------------------------------------- + awk '{print$1}' + kubectl_bin delete namespace psmdb-operator --ignore-not-found + xargs kubectl delete ns ++ mktemp ++ mktemp + local LAST_OUT=/tmp/tmp.dlmzIl6TsR ++ mktemp + local LAST_OUT=/tmp/tmp.QKrslXWBNl + local LAST_ERR=/tmp/tmp.OGwGPIAsfA + local exit_status=0 + local timeout=4 ++ mktemp ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get ns + local LAST_ERR=/tmp/tmp.eKvUB7lmIo + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete namespace psmdb-operator --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.dlmzIl6TsR + cat /tmp/tmp.OGwGPIAsfA + rm /tmp/tmp.dlmzIl6TsR /tmp/tmp.OGwGPIAsfA + return 0 namespace "pvc-auto-resize-29634" deleted + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.QKrslXWBNl namespace "psmdb-operator" deleted + cat /tmp/tmp.eKvUB7lmIo + rm /tmp/tmp.QKrslXWBNl /tmp/tmp.eKvUB7lmIo + return 0 + kubectl_bin wait --for=delete namespace psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.JahaUDu5t8 ++ mktemp + local LAST_ERR=/tmp/tmp.z9FnnZoMTZ + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete namespace psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.JahaUDu5t8 + cat /tmp/tmp.z9FnnZoMTZ + rm /tmp/tmp.JahaUDu5t8 /tmp/tmp.z9FnnZoMTZ + return 0 + desc 'create namespace psmdb-operator' + set +o xtrace ----------------------------------------------------------------------------------- create namespace psmdb-operator ----------------------------------------------------------------------------------- + kubectl_bin create namespace psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.iA8AaMRjJx ++ mktemp + local LAST_ERR=/tmp/tmp.7wMfyuShU2 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.iA8AaMRjJx namespace/psmdb-operator created + cat /tmp/tmp.7wMfyuShU2 + rm /tmp/tmp.iA8AaMRjJx /tmp/tmp.7wMfyuShU2 + return 0 + set_kube_ctx psmdb-operator + local namespace=psmdb-operator ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.3mYvE2ynP8 +++ mktemp ++ local LAST_ERR=/tmp/tmp.Wf196SSwzk ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.3mYvE2ynP8 ++ cat /tmp/tmp.Wf196SSwzk ++ rm /tmp/tmp.3mYvE2ynP8 /tmp/tmp.Wf196SSwzk ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2058-fb8cc7187-17-cluster14 --namespace=psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.FvWkMzvhGL ++ mktemp + local LAST_ERR=/tmp/tmp.Jfq0q7IjaN + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2058-fb8cc7187-17-cluster14 --namespace=psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.FvWkMzvhGL Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2058-fb8cc7187-17-cluster14" modified. + cat /tmp/tmp.Jfq0q7IjaN + rm /tmp/tmp.FvWkMzvhGL /tmp/tmp.Jfq0q7IjaN + return 0 + deploy_operator + desc 'start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2058-fb8cc7187' + set +o xtrace ----------------------------------------------------------------------------------- start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2058-fb8cc7187 ----------------------------------------------------------------------------------- + local cr_file + '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/split-horizon-manual-tls/conf/crd.yaml ']' + cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.F1RWqhXXJi ++ mktemp + local LAST_ERR=/tmp/tmp.KQbOrYGF9y + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.F1RWqhXXJi customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied + cat /tmp/tmp.KQbOrYGF9y + rm /tmp/tmp.F1RWqhXXJi /tmp/tmp.KQbOrYGF9y + return 0 + '[' -n psmdb-operator ']' + apply_rbac cw-rbac + local operator_namespace=psmdb-operator + local rbac=cw-rbac + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/cw-rbac.yaml + sed -e 's^namespace: .*^namespace: psmdb-operator^' + kubectl_bin apply -n psmdb-operator -f - ++ mktemp + local LAST_OUT=/tmp/tmp.rMK5Cbc7Cf ++ mktemp + local LAST_ERR=/tmp/tmp.VdoTZyXoMO + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -n psmdb-operator -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.rMK5Cbc7Cf clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator unchanged serviceaccount/percona-server-mongodb-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator unchanged + cat /tmp/tmp.VdoTZyXoMO + rm /tmp/tmp.rMK5Cbc7Cf /tmp/tmp.VdoTZyXoMO + return 0 + yq eval $'\n\t\t\t(.spec.template.spec.containers[].image = "docker.io/perconalab/percona-server-mongodb-operator:PR-2058-fb8cc7187") |\n\t\t\t((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |\n\t\t\t((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/cw-operator.yaml + kubectl_bin apply -n psmdb-operator -f - ++ mktemp + local LAST_OUT=/tmp/tmp.cTcbVzOS9z ++ mktemp + local LAST_ERR=/tmp/tmp.hfUhGsctZV + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -n psmdb-operator -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.cTcbVzOS9z deployment.apps/percona-server-mongodb-operator created + cat /tmp/tmp.hfUhGsctZV + rm /tmp/tmp.cTcbVzOS9z /tmp/tmp.hfUhGsctZV + return 0 + sleep 20 ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.43LAVHBAHd +++ mktemp ++ local LAST_ERR=/tmp/tmp.DchbDs1KEU ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.43LAVHBAHd ++ cat /tmp/tmp.DchbDs1KEU ++ rm /tmp/tmp.43LAVHBAHd /tmp/tmp.DchbDs1KEU ++ return 0 + wait_operator_pod percona-server-mongodb-operator-859f4bc6d7-2mxtg + local pod=percona-server-mongodb-operator-859f4bc6d7-2mxtg + set +o xtrace waiting for pod/percona-server-mongodb-operator-859f4bc6d7-2mxtg to be ready.OK + echo 'Print operator info from log' Print operator info from log ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator + grep 'Manager starting up' +++ mktemp ++ local LAST_OUT=/tmp/tmp.hg2NiQuTye +++ mktemp ++ local LAST_ERR=/tmp/tmp.ALzninaQSS ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.hg2NiQuTye ++ cat /tmp/tmp.ALzninaQSS ++ rm /tmp/tmp.hg2NiQuTye /tmp/tmp.ALzninaQSS ++ return 0 + kubectl_bin logs -n psmdb-operator percona-server-mongodb-operator-859f4bc6d7-2mxtg ++ mktemp + local LAST_OUT=/tmp/tmp.HYrnzTxtB4 ++ mktemp + local LAST_ERR=/tmp/tmp.4YtoBBJ62f + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs -n psmdb-operator percona-server-mongodb-operator-859f4bc6d7-2mxtg + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.HYrnzTxtB4 + cat /tmp/tmp.4YtoBBJ62f + rm /tmp/tmp.HYrnzTxtB4 /tmp/tmp.4YtoBBJ62f + return 0 2026-06-09T10:44:45.732Z INFO setup Manager starting up {"gitCommit": "fb8cc718776dc8f9e011cd5f23fe48cf7f65143e", "gitBranch": "PR-2058-fb8cc7187", "buildTime": "", "goVersion": "go1.26.4", "os": "linux", "arch": "amd64"} + create_namespace split-horizon-manual-tls-24334 + local namespace=split-horizon-manual-tls-24334 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ sed s/NAMESPACE// ++ awk '-F ' '{print $2}' + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ grep validate-auth ++ kubectl get ValidatingWebhookConfiguration ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME' + awk '{print$1}' + '[' -n '' ']' + desc 'cleaned up old namespaces split-horizon-manual-tls-24334' + set +o xtrace + xargs kubectl delete ns ----------------------------------------------------------------------------------- cleaned up old namespaces split-horizon-manual-tls-24334 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace split-horizon-manual-tls-24334 --ignore-not-found ++ mktemp ++ mktemp + local LAST_OUT=/tmp/tmp.WfTd1vhWeR ++ mktemp + local LAST_OUT=/tmp/tmp.txd99qSxXR ++ mktemp + local LAST_ERR=/tmp/tmp.vooHNpkQeV + local exit_status=0 + local timeout=4 ++ seq 0 2 + local LAST_ERR=/tmp/tmp.yeeBoQBNSM + local exit_status=0 + local timeout=4 + for i in $(seq 0 2) + set +e + kubectl get ns ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete namespace split-horizon-manual-tls-24334 --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.WfTd1vhWeR + cat /tmp/tmp.vooHNpkQeV + rm /tmp/tmp.WfTd1vhWeR /tmp/tmp.vooHNpkQeV + return 0 error: resource(s) were provided, but no name was specified + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.txd99qSxXR + cat /tmp/tmp.yeeBoQBNSM + rm /tmp/tmp.txd99qSxXR /tmp/tmp.yeeBoQBNSM + return 0 + kubectl_bin wait --for=delete namespace split-horizon-manual-tls-24334 ++ mktemp + local LAST_OUT=/tmp/tmp.zr9YTBi8pd ++ mktemp + local LAST_ERR=/tmp/tmp.NA2D35rSDm + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete namespace split-horizon-manual-tls-24334 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.zr9YTBi8pd + cat /tmp/tmp.NA2D35rSDm + rm /tmp/tmp.zr9YTBi8pd /tmp/tmp.NA2D35rSDm + return 0 + desc 'create namespace split-horizon-manual-tls-24334' + set +o xtrace ----------------------------------------------------------------------------------- create namespace split-horizon-manual-tls-24334 ----------------------------------------------------------------------------------- + kubectl_bin create namespace split-horizon-manual-tls-24334 ++ mktemp + local LAST_OUT=/tmp/tmp.0Eb29qMm0E ++ mktemp + local LAST_ERR=/tmp/tmp.6RKmx66SEy + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace split-horizon-manual-tls-24334 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.0Eb29qMm0E namespace/split-horizon-manual-tls-24334 created + cat /tmp/tmp.6RKmx66SEy + rm /tmp/tmp.0Eb29qMm0E /tmp/tmp.6RKmx66SEy + return 0 + set_kube_ctx split-horizon-manual-tls-24334 + local namespace=split-horizon-manual-tls-24334 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.Kc2b1ooV1d +++ mktemp ++ local LAST_ERR=/tmp/tmp.hD821VUDj9 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Kc2b1ooV1d ++ cat /tmp/tmp.hD821VUDj9 ++ rm /tmp/tmp.Kc2b1ooV1d /tmp/tmp.hD821VUDj9 ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2058-fb8cc7187-17-cluster14 --namespace=split-horizon-manual-tls-24334 ++ mktemp + local LAST_OUT=/tmp/tmp.vggwqzZ1TE ++ mktemp + local LAST_ERR=/tmp/tmp.dTrnU2DUSJ + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2058-fb8cc7187-17-cluster14 --namespace=split-horizon-manual-tls-24334 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.vggwqzZ1TE Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2058-fb8cc7187-17-cluster14" modified. + cat /tmp/tmp.dTrnU2DUSJ + rm /tmp/tmp.vggwqzZ1TE /tmp/tmp.dTrnU2DUSJ + return 0 + destroy_cert_manager + kubectl_bin delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.llpxdbVyVd ++ mktemp + local LAST_ERR=/tmp/tmp.Ajm0ZabsJm + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml + exit_status=1 + set -e + '[' 1 '!=' 0 -a -n 1 ']' + cat /tmp/tmp.llpxdbVyVd customresourcedefinition.apiextensions.k8s.io "challenges.acme.cert-manager.io" deleted customresourcedefinition.apiextensions.k8s.io "orders.acme.cert-manager.io" deleted customresourcedefinition.apiextensions.k8s.io "certificaterequests.cert-manager.io" deleted customresourcedefinition.apiextensions.k8s.io "certificates.cert-manager.io" deleted customresourcedefinition.apiextensions.k8s.io "clusterissuers.cert-manager.io" deleted customresourcedefinition.apiextensions.k8s.io "issuers.cert-manager.io" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-cainjector" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-controller-issuers" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-controller-certificates" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-controller-orders" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-controller-challenges" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-cluster-view" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-view" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-edit" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-cainjector" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-issuers" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-certificates" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-orders" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-challenges" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" deleted role.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" deleted from kube-system namespace role.rbac.authorization.k8s.io "cert-manager:leaderelection" deleted from kube-system namespace rolebinding.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" deleted from kube-system namespace rolebinding.rbac.authorization.k8s.io "cert-manager:leaderelection" deleted from kube-system namespace mutatingwebhookconfiguration.admissionregistration.k8s.io "cert-manager-webhook" deleted validatingwebhookconfiguration.admissionregistration.k8s.io "cert-manager-webhook" deleted + cat /tmp/tmp.Ajm0ZabsJm Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": namespaces "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found + sleep 0 + for i in $(seq 0 2) + set +e + kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml + exit_status=1 + set -e + '[' 1 '!=' 0 -a -n 1 ']' + cat /tmp/tmp.llpxdbVyVd + cat /tmp/tmp.Ajm0ZabsJm Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": namespaces "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found + sleep 4 + for i in $(seq 0 2) + set +e + kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml + exit_status=1 + set -e + '[' 1 '!=' 0 -a -n 1 ']' + cat /tmp/tmp.llpxdbVyVd + cat /tmp/tmp.Ajm0ZabsJm Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": namespaces "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found + sleep 8 + cat /tmp/tmp.llpxdbVyVd + cat /tmp/tmp.Ajm0ZabsJm Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": namespaces "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found + rm /tmp/tmp.llpxdbVyVd /tmp/tmp.Ajm0ZabsJm + return 1 + true + cluster=some-name + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/client_with_tls.yml ++ mktemp + local LAST_OUT=/tmp/tmp.pJomyeonxX ++ mktemp + local LAST_ERR=/tmp/tmp.RrrQOJm5sL + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/client_with_tls.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.pJomyeonxX secret/some-users created deployment.apps/psmdb-client created + cat /tmp/tmp.RrrQOJm5sL + rm /tmp/tmp.pJomyeonxX /tmp/tmp.RrrQOJm5sL + return 0 + desc 'deploy cluster with 3 split horizons (manual TLS)' + set +o xtrace ----------------------------------------------------------------------------------- deploy cluster with 3 split horizons (manual TLS) ----------------------------------------------------------------------------------- + apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/split-horizon-manual-tls/conf/some-name-3horizons.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/split-horizon-manual-tls/conf/some-name-3horizons.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/split-horizon-manual-tls/conf/some-name-3horizons.yml ++ mktemp + yq eval '(.spec | select(.image == null)).image = "docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0"' + yq eval '(.spec | select(has("pmm"))).pmm.image = "docker.io/percona/pmm-client:2.44.1-1"' + yq eval '(.spec | select(has("initImage"))).initImage = "docker.io/perconalab/percona-server-mongodb-operator:PR-2058-fb8cc7187"' + local LAST_OUT=/tmp/tmp.dVjPUyegtf ++ mktemp + yq eval '(.spec | select(has("backup"))).backup.image = "docker.io/perconalab/percona-server-mongodb-operator:main-backup"' + /usr/sbin/sed -e s/NAME_SPACE/split-horizon-manual-tls-24334/g + local LAST_ERR=/tmp/tmp.D8kgGDiRBx + local exit_status=0 + local timeout=4 + yq eval '.spec.upgradeOptions.apply="Never"' ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.dVjPUyegtf perconaservermongodb.psmdb.percona.com/some-name created + cat /tmp/tmp.D8kgGDiRBx + rm /tmp/tmp.dVjPUyegtf /tmp/tmp.D8kgGDiRBx + return 0 + wait_for_running some-name-rs0 3 + local name=some-name-rs0 + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-rs0-0 + local pod=some-name-rs0-0 + set +o xtrace waiting for pod/some-name-rs0-0 to be ready.................OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-rs0-1 + local pod=some-name-rs0-1 + set +o xtrace waiting for pod/some-name-rs0-1 to be ready..................OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.5e3CesXf0E +++ mktemp ++ local LAST_ERR=/tmp/tmp.PJKkNmTrVF ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.5e3CesXf0E ++ cat /tmp/tmp.PJKkNmTrVF ++ rm /tmp/tmp.5e3CesXf0E /tmp/tmp.PJKkNmTrVF ++ return 0 + [[ '' == true ]] + wait_pod some-name-rs0-2 + local pod=some-name-rs0-2 + set +o xtrace waiting for pod/some-name-rs0-2 to be ready..................OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.BWEZOzfCD3 +++ mktemp ++ local LAST_ERR=/tmp/tmp.ulESHbgREf ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.BWEZOzfCD3 ++ cat /tmp/tmp.ulESHbgREf ++ rm /tmp/tmp.BWEZOzfCD3 /tmp/tmp.ulESHbgREf ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.sDTqO3RrwR +++ mktemp ++ local LAST_ERR=/tmp/tmp.S1ULvqpUcF ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.sDTqO3RrwR ++ cat /tmp/tmp.S1ULvqpUcF ++ rm /tmp/tmp.sDTqO3RrwR /tmp/tmp.S1ULvqpUcF ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster readyness + wait_cluster_consistency some-name + local cluster_name=some-name + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster readyness' waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.cQKIMXpGd4 +++ mktemp ++ local LAST_ERR=/tmp/tmp.LcKo9jIHZN ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.cQKIMXpGd4 ++ cat /tmp/tmp.LcKo9jIHZN ++ rm /tmp/tmp.cQKIMXpGd4 /tmp/tmp.LcKo9jIHZN ++ return 0 + [[ ready == ready ]] + echo .OK .OK + desc 'verify CA secret exists with ca.crt and ca.key' + set +o xtrace ----------------------------------------------------------------------------------- verify CA secret exists with ca.crt and ca.key ----------------------------------------------------------------------------------- + verify_ca_secret_exists some-name-ca-cert + local secret_name=some-name-ca-cert + kubectl_bin get secret some-name-ca-cert -o 'jsonpath={.data.ca\.crt}' + kubectl_bin get secret some-name-ca-cert -o 'jsonpath={.data.ca\.key}' + echo 'OK: CA secret '\''some-name-ca-cert'\'' exists with ca.crt and ca.key' OK: CA secret 'some-name-ca-cert' exists with ca.crt and ca.key + desc 'verify TLS secrets are signed by the CA' + set +o xtrace ----------------------------------------------------------------------------------- verify TLS secrets are signed by the CA ----------------------------------------------------------------------------------- + verify_cert_signed_by_ca some-name-ssl some-name-ca-cert + local tls_secret_name=some-name-ssl + local ca_secret_name=some-name-ca-cert + local ca_crt ++ base64 -d ++ kubectl_bin get secret some-name-ca-cert -o 'jsonpath={.data.ca\.crt}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.f3tF1AG1R0 +++ mktemp ++ local LAST_ERR=/tmp/tmp.SqwR1FEGTg ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get secret some-name-ca-cert -o 'jsonpath={.data.ca\.crt}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.f3tF1AG1R0 ++ cat /tmp/tmp.SqwR1FEGTg ++ rm /tmp/tmp.f3tF1AG1R0 /tmp/tmp.SqwR1FEGTg ++ return 0 + ca_crt=$'-----BEGIN CERTIFICATE-----\nMIIDHDCCAgSgAwIBAgIRAOKCbOotUMOQvcitnmwCqI8wDQYJKoZIhvcNAQELBQAw\nEjEQMA4GA1UEChMHUm9vdCBDQTAgFw0yNjA2MDkxMDQ2MDNaGA85OTk5MTIzMTIz\nNTk1OVowEjEQMA4GA1UEChMHUm9vdCBDQTCCASIwDQYJKoZIhvcNAQEBBQADggEP\nADCCAQoCggEBAKWfk5pyPbV1b6Ctsa64tAtZGxqxiAzOG9NSDB90nX+mSuYDZcZr\n4XsFD3uaCXP2zbdlOm28D67lt9n9ZP1y2gwzY2vaehHFMMB/ybJAfZ2ba9IDr8M9\njxqDapgOc82VMCrFTUPrfHI/ZeRKpcOYn8aLWeQJdsDCjfS82wf/B1cIdQ2haIvk\nTykjbKxA1CPe+ORR6655DxJlHSW+Kgo8XyMUDTxHY9B1JoogWfaNwYAG0C3mi3Rn\nrjC08gY+KGlEuCGEpI6c72klw7/FnWs/ZMQrW4DbKTkeSV+LzPGGrdpcANeQbnr4\nSQEShtVhXL5oRO2td3AZQ0vP7o2IOHnLrrsCAwEAAaNrMGkwDgYDVR0PAQH/BAQD\nAgIkMCcGA1UdJQQgMB4GCCsGAQUFBwMBBggrBgEFBQcDAgYIKwYBBQUHAwMwDwYD\nVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUHNwdIX9ky63IirIrX9VQ/AI2GIUwDQYJ\nKoZIhvcNAQELBQADggEBAJ+Tvunj7OmoyiiJfXojbqsZrc924jhJAnGJR3GzVcas\n1IQnS4bog54gfVqNf84InSM9WL38SBPJkRNE1PPlcMdy0Z+1Su/KP/LhKyVYN1ds\nBCnnMAelDQD+5h9XeH7QMkuOudgbnMXDQuoOiciv59dmVA9BoC42I/ljKCvB1EON\nEwIXjVMv05Lx4OuyThTyn8zded8hTiP9yRC0RKYB5LCdBie6g4LmpFQU4NWJRLhy\nvLtm7Mrq699SOOcIJAX3mFDjwol9EQuH/x95vPfeyQfYgAeaP/sS08W6nGcYSTGb\nCeTSH5KFnsOV3Z5+4xNs8hMyqJDTgMGk3Dw3aGoUTcU=\n-----END CERTIFICATE-----' + local tls_crt ++ kubectl_bin get secret some-name-ssl -o 'jsonpath={.data.tls\.crt}' ++ base64 -d +++ mktemp ++ local LAST_OUT=/tmp/tmp.KKNEAFzglz +++ mktemp ++ local LAST_ERR=/tmp/tmp.ZTYG4ZxD6c ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get secret some-name-ssl -o 'jsonpath={.data.tls\.crt}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.KKNEAFzglz ++ cat /tmp/tmp.ZTYG4ZxD6c ++ rm /tmp/tmp.KKNEAFzglz /tmp/tmp.ZTYG4ZxD6c ++ return 0 + tls_crt=$'-----BEGIN CERTIFICATE-----\nMIIIcjCCB1qgAwIBAgIQZY+cVtotoQd3E/k20gbUsjANBgkqhkiG9w0BAQsFADAS\nMRAwDgYDVQQKEwdSb290IENBMCAXDTI2MDYwOTEwNDYwM1oYDzk5OTkxMjMxMjM1\nOTU5WjAQMQ4wDAYDVQQKEwVQU01EQjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC\nAQoCggEBANMI1CPhRCC1shq9r4rEt3yW3iQdwR/REtQ5Lt+XpaQ2y3hfdRMuVNw8\nrKHlvVgwhh+8y7lIMXEi9rGJojkXi9dhQX5VzZRSyWD+cYMsXan3i7rR/b2pX9aH\n4zT/3SU0sThqFa/UJV2PPfbAgE+GbQNQ3tdFZb79T9tlwWafk50Xaihox4Xdm5Zu\n/Wthe8//TmcCvUnojy9ReMkXl9yigqcoeB2+xO1uDQL/Q4jHgTKSzyNp4eP7MEcf\nCujG8QSjlMZcZF0fmQX4e94ltiiNg1bKwuw6qb8LfVNkamxbakEgsHaCfmbWSCLj\nPORs9/jWeXIKB7SxFQ+xKycosCTalN0CAwEAAaOCBcIwggW+MA4GA1UdDwEB/wQE\nAwIFoDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwDAYDVR0TAQH/BAIw\nADAfBgNVHSMEGDAWgBQc3B0hf2TLrciKsitf1VD8AjYYhTCCBVwGA1UdEQSCBVMw\nggVPgglsb2NhbGhvc3SCDXNvbWUtbmFtZS1yczCCLHNvbWUtbmFtZS1yczAuc3Bs\naXQtaG9yaXpvbi1tYW51YWwtdGxzLTI0MzM0gj5zb21lLW5hbWUtcnMwLnNwbGl0\nLWhvcml6b24tbWFudWFsLXRscy0yNDMzNC5zdmMuY2x1c3Rlci5sb2NhbIIPKi5z\nb21lLW5hbWUtcnMwgi4qLnNvbWUtbmFtZS1yczAuc3BsaXQtaG9yaXpvbi1tYW51\nYWwtdGxzLTI0MzM0gkAqLnNvbWUtbmFtZS1yczAuc3BsaXQtaG9yaXpvbi1tYW51\nYWwtdGxzLTI0MzM0LnN2Yy5jbHVzdGVyLmxvY2FsgkFzb21lLW5hbWUtcnMwLnNw\nbGl0LWhvcml6b24tbWFudWFsLXRscy0yNDMzNC5zdmMuY2x1c3RlcnNldC5sb2Nh\nbIJDKi5zb21lLW5hbWUtcnMwLnNwbGl0LWhvcml6b24tbWFudWFsLXRscy0yNDMz\nNC5zdmMuY2x1c3RlcnNldC5sb2NhbIIdc29tZS1uYW1lLXJzMC0wLmNsb3VkZGVt\nby54eXqCHXNvbWUtbmFtZS1yczAtMS5jbG91ZGRlbW8ueHl6gh1zb21lLW5hbWUt\ncnMwLTIuY2xvdWRkZW1vLnh5eoI1Ki5zcGxpdC1ob3Jpem9uLW1hbnVhbC10bHMt\nMjQzMzQuc3ZjLmNsdXN0ZXJzZXQubG9jYWyCEHNvbWUtbmFtZS1tb25nb3OCL3Nv\nbWUtbmFtZS1tb25nb3Muc3BsaXQtaG9yaXpvbi1tYW51YWwtdGxzLTI0MzM0gkFz\nb21lLW5hbWUtbW9uZ29zLnNwbGl0LWhvcml6b24tbWFudWFsLXRscy0yNDMzNC5z\ndmMuY2x1c3Rlci5sb2NhbIISKi5zb21lLW5hbWUtbW9uZ29zgjEqLnNvbWUtbmFt\nZS1tb25nb3Muc3BsaXQtaG9yaXpvbi1tYW51YWwtdGxzLTI0MzM0gkMqLnNvbWUt\nbmFtZS1tb25nb3Muc3BsaXQtaG9yaXpvbi1tYW51YWwtdGxzLTI0MzM0LnN2Yy5j\nbHVzdGVyLmxvY2Fsgg1zb21lLW5hbWUtY2Zngixzb21lLW5hbWUtY2ZnLnNwbGl0\nLWhvcml6b24tbWFudWFsLXRscy0yNDMzNII+c29tZS1uYW1lLWNmZy5zcGxpdC1o\nb3Jpem9uLW1hbnVhbC10bHMtMjQzMzQuc3ZjLmNsdXN0ZXIubG9jYWyCDyouc29t\nZS1uYW1lLWNmZ4IuKi5zb21lLW5hbWUtY2ZnLnNwbGl0LWhvcml6b24tbWFudWFs\nLXRscy0yNDMzNIJAKi5zb21lLW5hbWUtY2ZnLnNwbGl0LWhvcml6b24tbWFudWFs\nLXRscy0yNDMzNC5zdmMuY2x1c3Rlci5sb2NhbIJEc29tZS1uYW1lLW1vbmdvcy5z\ncGxpdC1ob3Jpem9uLW1hbnVhbC10bHMtMjQzMzQuc3ZjLmNsdXN0ZXJzZXQubG9j\nYWyCRiouc29tZS1uYW1lLW1vbmdvcy5zcGxpdC1ob3Jpem9uLW1hbnVhbC10bHMt\nMjQzMzQuc3ZjLmNsdXN0ZXJzZXQubG9jYWyCQXNvbWUtbmFtZS1jZmcuc3BsaXQt\naG9yaXpvbi1tYW51YWwtdGxzLTI0MzM0LnN2Yy5jbHVzdGVyc2V0LmxvY2FsgkMq\nLnNvbWUtbmFtZS1jZmcuc3BsaXQtaG9yaXpvbi1tYW51YWwtdGxzLTI0MzM0LnN2\nYy5jbHVzdGVyc2V0LmxvY2FsMA0GCSqGSIb3DQEBCwUAA4IBAQAhksNJ/c17I5jY\nWVoynwQhYQ8IwgfAzs/CdsRCBXAm4cN162kuedBKHWPiOH+0PwBWGEb3XVeIXOcv\n7MvAsQrC1Nexve6i1WgKvtKG+pna+m8+MrzbrL1CuJarriXAkYlFT160vVAlFV60\njwjGYzgMpxpW/OutjSeHZI3jSkbGM/5Ngh5nMDhw3pbYNMzNiXb8I4cTK3RSidwd\nH0dqsxu5rCRInNtI2eWUVSB4RoyU5I7TBlob25vRc7Xop7JtC8xjCfbpza3CqKZs\nBbp0b7QcL3EoG3/JTzEBzysOuCXTt243OMaSgIa1/18EAWD6ebe7K7XJD2qN6Taa\n45FDJlOc\n-----END CERTIFICATE-----' + echo $'-----BEGIN CERTIFICATE-----\nMIIDHDCCAgSgAwIBAgIRAOKCbOotUMOQvcitnmwCqI8wDQYJKoZIhvcNAQELBQAw\nEjEQMA4GA1UEChMHUm9vdCBDQTAgFw0yNjA2MDkxMDQ2MDNaGA85OTk5MTIzMTIz\nNTk1OVowEjEQMA4GA1UEChMHUm9vdCBDQTCCASIwDQYJKoZIhvcNAQEBBQADggEP\nADCCAQoCggEBAKWfk5pyPbV1b6Ctsa64tAtZGxqxiAzOG9NSDB90nX+mSuYDZcZr\n4XsFD3uaCXP2zbdlOm28D67lt9n9ZP1y2gwzY2vaehHFMMB/ybJAfZ2ba9IDr8M9\njxqDapgOc82VMCrFTUPrfHI/ZeRKpcOYn8aLWeQJdsDCjfS82wf/B1cIdQ2haIvk\nTykjbKxA1CPe+ORR6655DxJlHSW+Kgo8XyMUDTxHY9B1JoogWfaNwYAG0C3mi3Rn\nrjC08gY+KGlEuCGEpI6c72klw7/FnWs/ZMQrW4DbKTkeSV+LzPGGrdpcANeQbnr4\nSQEShtVhXL5oRO2td3AZQ0vP7o2IOHnLrrsCAwEAAaNrMGkwDgYDVR0PAQH/BAQD\nAgIkMCcGA1UdJQQgMB4GCCsGAQUFBwMBBggrBgEFBQcDAgYIKwYBBQUHAwMwDwYD\nVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUHNwdIX9ky63IirIrX9VQ/AI2GIUwDQYJ\nKoZIhvcNAQELBQADggEBAJ+Tvunj7OmoyiiJfXojbqsZrc924jhJAnGJR3GzVcas\n1IQnS4bog54gfVqNf84InSM9WL38SBPJkRNE1PPlcMdy0Z+1Su/KP/LhKyVYN1ds\nBCnnMAelDQD+5h9XeH7QMkuOudgbnMXDQuoOiciv59dmVA9BoC42I/ljKCvB1EON\nEwIXjVMv05Lx4OuyThTyn8zded8hTiP9yRC0RKYB5LCdBie6g4LmpFQU4NWJRLhy\nvLtm7Mrq699SOOcIJAX3mFDjwol9EQuH/x95vPfeyQfYgAeaP/sS08W6nGcYSTGb\nCeTSH5KFnsOV3Z5+4xNs8hMyqJDTgMGk3Dw3aGoUTcU=\n-----END CERTIFICATE-----' + echo $'-----BEGIN CERTIFICATE-----\nMIIIcjCCB1qgAwIBAgIQZY+cVtotoQd3E/k20gbUsjANBgkqhkiG9w0BAQsFADAS\nMRAwDgYDVQQKEwdSb290IENBMCAXDTI2MDYwOTEwNDYwM1oYDzk5OTkxMjMxMjM1\nOTU5WjAQMQ4wDAYDVQQKEwVQU01EQjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC\nAQoCggEBANMI1CPhRCC1shq9r4rEt3yW3iQdwR/REtQ5Lt+XpaQ2y3hfdRMuVNw8\nrKHlvVgwhh+8y7lIMXEi9rGJojkXi9dhQX5VzZRSyWD+cYMsXan3i7rR/b2pX9aH\n4zT/3SU0sThqFa/UJV2PPfbAgE+GbQNQ3tdFZb79T9tlwWafk50Xaihox4Xdm5Zu\n/Wthe8//TmcCvUnojy9ReMkXl9yigqcoeB2+xO1uDQL/Q4jHgTKSzyNp4eP7MEcf\nCujG8QSjlMZcZF0fmQX4e94ltiiNg1bKwuw6qb8LfVNkamxbakEgsHaCfmbWSCLj\nPORs9/jWeXIKB7SxFQ+xKycosCTalN0CAwEAAaOCBcIwggW+MA4GA1UdDwEB/wQE\nAwIFoDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwDAYDVR0TAQH/BAIw\nADAfBgNVHSMEGDAWgBQc3B0hf2TLrciKsitf1VD8AjYYhTCCBVwGA1UdEQSCBVMw\nggVPgglsb2NhbGhvc3SCDXNvbWUtbmFtZS1yczCCLHNvbWUtbmFtZS1yczAuc3Bs\naXQtaG9yaXpvbi1tYW51YWwtdGxzLTI0MzM0gj5zb21lLW5hbWUtcnMwLnNwbGl0\nLWhvcml6b24tbWFudWFsLXRscy0yNDMzNC5zdmMuY2x1c3Rlci5sb2NhbIIPKi5z\nb21lLW5hbWUtcnMwgi4qLnNvbWUtbmFtZS1yczAuc3BsaXQtaG9yaXpvbi1tYW51\nYWwtdGxzLTI0MzM0gkAqLnNvbWUtbmFtZS1yczAuc3BsaXQtaG9yaXpvbi1tYW51\nYWwtdGxzLTI0MzM0LnN2Yy5jbHVzdGVyLmxvY2FsgkFzb21lLW5hbWUtcnMwLnNw\nbGl0LWhvcml6b24tbWFudWFsLXRscy0yNDMzNC5zdmMuY2x1c3RlcnNldC5sb2Nh\nbIJDKi5zb21lLW5hbWUtcnMwLnNwbGl0LWhvcml6b24tbWFudWFsLXRscy0yNDMz\nNC5zdmMuY2x1c3RlcnNldC5sb2NhbIIdc29tZS1uYW1lLXJzMC0wLmNsb3VkZGVt\nby54eXqCHXNvbWUtbmFtZS1yczAtMS5jbG91ZGRlbW8ueHl6gh1zb21lLW5hbWUt\ncnMwLTIuY2xvdWRkZW1vLnh5eoI1Ki5zcGxpdC1ob3Jpem9uLW1hbnVhbC10bHMt\nMjQzMzQuc3ZjLmNsdXN0ZXJzZXQubG9jYWyCEHNvbWUtbmFtZS1tb25nb3OCL3Nv\nbWUtbmFtZS1tb25nb3Muc3BsaXQtaG9yaXpvbi1tYW51YWwtdGxzLTI0MzM0gkFz\nb21lLW5hbWUtbW9uZ29zLnNwbGl0LWhvcml6b24tbWFudWFsLXRscy0yNDMzNC5z\ndmMuY2x1c3Rlci5sb2NhbIISKi5zb21lLW5hbWUtbW9uZ29zgjEqLnNvbWUtbmFt\nZS1tb25nb3Muc3BsaXQtaG9yaXpvbi1tYW51YWwtdGxzLTI0MzM0gkMqLnNvbWUt\nbmFtZS1tb25nb3Muc3BsaXQtaG9yaXpvbi1tYW51YWwtdGxzLTI0MzM0LnN2Yy5j\nbHVzdGVyLmxvY2Fsgg1zb21lLW5hbWUtY2Zngixzb21lLW5hbWUtY2ZnLnNwbGl0\nLWhvcml6b24tbWFudWFsLXRscy0yNDMzNII+c29tZS1uYW1lLWNmZy5zcGxpdC1o\nb3Jpem9uLW1hbnVhbC10bHMtMjQzMzQuc3ZjLmNsdXN0ZXIubG9jYWyCDyouc29t\nZS1uYW1lLWNmZ4IuKi5zb21lLW5hbWUtY2ZnLnNwbGl0LWhvcml6b24tbWFudWFs\nLXRscy0yNDMzNIJAKi5zb21lLW5hbWUtY2ZnLnNwbGl0LWhvcml6b24tbWFudWFs\nLXRscy0yNDMzNC5zdmMuY2x1c3Rlci5sb2NhbIJEc29tZS1uYW1lLW1vbmdvcy5z\ncGxpdC1ob3Jpem9uLW1hbnVhbC10bHMtMjQzMzQuc3ZjLmNsdXN0ZXJzZXQubG9j\nYWyCRiouc29tZS1uYW1lLW1vbmdvcy5zcGxpdC1ob3Jpem9uLW1hbnVhbC10bHMt\nMjQzMzQuc3ZjLmNsdXN0ZXJzZXQubG9jYWyCQXNvbWUtbmFtZS1jZmcuc3BsaXQt\naG9yaXpvbi1tYW51YWwtdGxzLTI0MzM0LnN2Yy5jbHVzdGVyc2V0LmxvY2FsgkMq\nLnNvbWUtbmFtZS1jZmcuc3BsaXQtaG9yaXpvbi1tYW51YWwtdGxzLTI0MzM0LnN2\nYy5jbHVzdGVyc2V0LmxvY2FsMA0GCSqGSIb3DQEBCwUAA4IBAQAhksNJ/c17I5jY\nWVoynwQhYQ8IwgfAzs/CdsRCBXAm4cN162kuedBKHWPiOH+0PwBWGEb3XVeIXOcv\n7MvAsQrC1Nexve6i1WgKvtKG+pna+m8+MrzbrL1CuJarriXAkYlFT160vVAlFV60\njwjGYzgMpxpW/OutjSeHZI3jSkbGM/5Ngh5nMDhw3pbYNMzNiXb8I4cTK3RSidwd\nH0dqsxu5rCRInNtI2eWUVSB4RoyU5I7TBlob25vRc7Xop7JtC8xjCfbpza3CqKZs\nBbp0b7QcL3EoG3/JTzEBzysOuCXTt243OMaSgIa1/18EAWD6ebe7K7XJD2qN6Taa\n45FDJlOc\n-----END CERTIFICATE-----' + openssl verify -CAfile /tmp/tmp.i1hhairPLp/ca.crt /tmp/tmp.i1hhairPLp/tls.crt + echo 'OK: '\''some-name-ssl'\'' is signed by CA in '\''some-name-ca-cert'\''' OK: 'some-name-ssl' is signed by CA in 'some-name-ca-cert' + verify_cert_signed_by_ca some-name-ssl-internal some-name-ca-cert + local tls_secret_name=some-name-ssl-internal + local ca_secret_name=some-name-ca-cert + local ca_crt ++ kubectl_bin get secret some-name-ca-cert -o 'jsonpath={.data.ca\.crt}' ++ base64 -d +++ mktemp ++ local LAST_OUT=/tmp/tmp.RUu0eQk7Su +++ mktemp ++ local LAST_ERR=/tmp/tmp.Eqba5vTZtc ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get secret some-name-ca-cert -o 'jsonpath={.data.ca\.crt}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.RUu0eQk7Su ++ cat /tmp/tmp.Eqba5vTZtc ++ rm /tmp/tmp.RUu0eQk7Su /tmp/tmp.Eqba5vTZtc ++ return 0 + ca_crt=$'-----BEGIN CERTIFICATE-----\nMIIDHDCCAgSgAwIBAgIRAOKCbOotUMOQvcitnmwCqI8wDQYJKoZIhvcNAQELBQAw\nEjEQMA4GA1UEChMHUm9vdCBDQTAgFw0yNjA2MDkxMDQ2MDNaGA85OTk5MTIzMTIz\nNTk1OVowEjEQMA4GA1UEChMHUm9vdCBDQTCCASIwDQYJKoZIhvcNAQEBBQADggEP\nADCCAQoCggEBAKWfk5pyPbV1b6Ctsa64tAtZGxqxiAzOG9NSDB90nX+mSuYDZcZr\n4XsFD3uaCXP2zbdlOm28D67lt9n9ZP1y2gwzY2vaehHFMMB/ybJAfZ2ba9IDr8M9\njxqDapgOc82VMCrFTUPrfHI/ZeRKpcOYn8aLWeQJdsDCjfS82wf/B1cIdQ2haIvk\nTykjbKxA1CPe+ORR6655DxJlHSW+Kgo8XyMUDTxHY9B1JoogWfaNwYAG0C3mi3Rn\nrjC08gY+KGlEuCGEpI6c72klw7/FnWs/ZMQrW4DbKTkeSV+LzPGGrdpcANeQbnr4\nSQEShtVhXL5oRO2td3AZQ0vP7o2IOHnLrrsCAwEAAaNrMGkwDgYDVR0PAQH/BAQD\nAgIkMCcGA1UdJQQgMB4GCCsGAQUFBwMBBggrBgEFBQcDAgYIKwYBBQUHAwMwDwYD\nVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUHNwdIX9ky63IirIrX9VQ/AI2GIUwDQYJ\nKoZIhvcNAQELBQADggEBAJ+Tvunj7OmoyiiJfXojbqsZrc924jhJAnGJR3GzVcas\n1IQnS4bog54gfVqNf84InSM9WL38SBPJkRNE1PPlcMdy0Z+1Su/KP/LhKyVYN1ds\nBCnnMAelDQD+5h9XeH7QMkuOudgbnMXDQuoOiciv59dmVA9BoC42I/ljKCvB1EON\nEwIXjVMv05Lx4OuyThTyn8zded8hTiP9yRC0RKYB5LCdBie6g4LmpFQU4NWJRLhy\nvLtm7Mrq699SOOcIJAX3mFDjwol9EQuH/x95vPfeyQfYgAeaP/sS08W6nGcYSTGb\nCeTSH5KFnsOV3Z5+4xNs8hMyqJDTgMGk3Dw3aGoUTcU=\n-----END CERTIFICATE-----' + local tls_crt ++ kubectl_bin get secret some-name-ssl-internal -o 'jsonpath={.data.tls\.crt}' ++ base64 -d +++ mktemp ++ local LAST_OUT=/tmp/tmp.ZcZzqrltLZ +++ mktemp ++ local LAST_ERR=/tmp/tmp.Mm972C0ouh ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get secret some-name-ssl-internal -o 'jsonpath={.data.tls\.crt}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.ZcZzqrltLZ ++ cat /tmp/tmp.Mm972C0ouh ++ rm /tmp/tmp.ZcZzqrltLZ /tmp/tmp.Mm972C0ouh ++ return 0 + tls_crt=$'-----BEGIN CERTIFICATE-----\nMIIIczCCB1ugAwIBAgIRAKjs79PWN/yUHxrXczThMFAwDQYJKoZIhvcNAQELBQAw\nEjEQMA4GA1UEChMHUm9vdCBDQTAgFw0yNjA2MDkxMDQ2MDNaGA85OTk5MTIzMTIz\nNTk1OVowEDEOMAwGA1UEChMFUFNNREIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw\nggEKAoIBAQCedVULBFLto7t8pH9/gZp6IGYvv0Q/Mx7mIzJwdXZRJaEKkxlUnwx2\nbfFoAVo7W3cpXJvRXOtQCmYIn92EBYaC+urVw3AnnPxX+9ib03VyiimhAlZIK0Bd\n+WH2lbFLP6UU4gLB3dCNuUY4Hsct7/s6g6i2mxG5WmuoXfNLYlauaAhNjkRQUEgm\nriODN2oNUdW9H82nUXsbQ0zIMkYLsDhlY8uMc6ifF6KEa9mL++CYkVDA5WUHBPWf\nm35RHzw5vKR6s94ZYb/dTTaA6hrAeF6ThW/Q3UazYApvFCGdH9Baeo4YMY1Lr2hl\n713hOx853bgRoIOlLlbhvLYjvmP37oB1AgMBAAGjggXCMIIFvjAOBgNVHQ8BAf8E\nBAMCBaAwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMAwGA1UdEwEB/wQC\nMAAwHwYDVR0jBBgwFoAUHNwdIX9ky63IirIrX9VQ/AI2GIUwggVcBgNVHREEggVT\nMIIFT4IJbG9jYWxob3N0gg1zb21lLW5hbWUtcnMwgixzb21lLW5hbWUtcnMwLnNw\nbGl0LWhvcml6b24tbWFudWFsLXRscy0yNDMzNII+c29tZS1uYW1lLXJzMC5zcGxp\ndC1ob3Jpem9uLW1hbnVhbC10bHMtMjQzMzQuc3ZjLmNsdXN0ZXIubG9jYWyCDyou\nc29tZS1uYW1lLXJzMIIuKi5zb21lLW5hbWUtcnMwLnNwbGl0LWhvcml6b24tbWFu\ndWFsLXRscy0yNDMzNIJAKi5zb21lLW5hbWUtcnMwLnNwbGl0LWhvcml6b24tbWFu\ndWFsLXRscy0yNDMzNC5zdmMuY2x1c3Rlci5sb2NhbIJBc29tZS1uYW1lLXJzMC5z\ncGxpdC1ob3Jpem9uLW1hbnVhbC10bHMtMjQzMzQuc3ZjLmNsdXN0ZXJzZXQubG9j\nYWyCQyouc29tZS1uYW1lLXJzMC5zcGxpdC1ob3Jpem9uLW1hbnVhbC10bHMtMjQz\nMzQuc3ZjLmNsdXN0ZXJzZXQubG9jYWyCHXNvbWUtbmFtZS1yczAtMC5jbG91ZGRl\nbW8ueHl6gh1zb21lLW5hbWUtcnMwLTEuY2xvdWRkZW1vLnh5eoIdc29tZS1uYW1l\nLXJzMC0yLmNsb3VkZGVtby54eXqCNSouc3BsaXQtaG9yaXpvbi1tYW51YWwtdGxz\nLTI0MzM0LnN2Yy5jbHVzdGVyc2V0LmxvY2FsghBzb21lLW5hbWUtbW9uZ29zgi9z\nb21lLW5hbWUtbW9uZ29zLnNwbGl0LWhvcml6b24tbWFudWFsLXRscy0yNDMzNIJB\nc29tZS1uYW1lLW1vbmdvcy5zcGxpdC1ob3Jpem9uLW1hbnVhbC10bHMtMjQzMzQu\nc3ZjLmNsdXN0ZXIubG9jYWyCEiouc29tZS1uYW1lLW1vbmdvc4IxKi5zb21lLW5h\nbWUtbW9uZ29zLnNwbGl0LWhvcml6b24tbWFudWFsLXRscy0yNDMzNIJDKi5zb21l\nLW5hbWUtbW9uZ29zLnNwbGl0LWhvcml6b24tbWFudWFsLXRscy0yNDMzNC5zdmMu\nY2x1c3Rlci5sb2NhbIINc29tZS1uYW1lLWNmZ4Isc29tZS1uYW1lLWNmZy5zcGxp\ndC1ob3Jpem9uLW1hbnVhbC10bHMtMjQzMzSCPnNvbWUtbmFtZS1jZmcuc3BsaXQt\naG9yaXpvbi1tYW51YWwtdGxzLTI0MzM0LnN2Yy5jbHVzdGVyLmxvY2Fsgg8qLnNv\nbWUtbmFtZS1jZmeCLiouc29tZS1uYW1lLWNmZy5zcGxpdC1ob3Jpem9uLW1hbnVh\nbC10bHMtMjQzMzSCQCouc29tZS1uYW1lLWNmZy5zcGxpdC1ob3Jpem9uLW1hbnVh\nbC10bHMtMjQzMzQuc3ZjLmNsdXN0ZXIubG9jYWyCRHNvbWUtbmFtZS1tb25nb3Mu\nc3BsaXQtaG9yaXpvbi1tYW51YWwtdGxzLTI0MzM0LnN2Yy5jbHVzdGVyc2V0Lmxv\nY2FsgkYqLnNvbWUtbmFtZS1tb25nb3Muc3BsaXQtaG9yaXpvbi1tYW51YWwtdGxz\nLTI0MzM0LnN2Yy5jbHVzdGVyc2V0LmxvY2FsgkFzb21lLW5hbWUtY2ZnLnNwbGl0\nLWhvcml6b24tbWFudWFsLXRscy0yNDMzNC5zdmMuY2x1c3RlcnNldC5sb2NhbIJD\nKi5zb21lLW5hbWUtY2ZnLnNwbGl0LWhvcml6b24tbWFudWFsLXRscy0yNDMzNC5z\ndmMuY2x1c3RlcnNldC5sb2NhbDANBgkqhkiG9w0BAQsFAAOCAQEAIEA2fvFUXr2M\ngEAhbNrijuUs/kfjO8yujPAAkPhlGUGZa9JIMiRObtqKIv2pqnYPI06U+/TpDXyL\nsBVH8rQD4AHadKqA6vp0iCs7xfruMocwXUIXY1sG9nt6Q742hgCH5ImdNrZOhTw5\nwA9dEq2taXDGq0AlPnS65EW8Pn8tG7gM366lgTzrezNMFvav6rGdTFTXYYvErMJ4\nPaRp6OzMQ7S6UXcWJSkAUOBfRWfP1LFF9r2JVIePtFMvQ21OylZm18appXdOwkX2\nUt0JXVW3N8sj1OQP8RGVjqi0hZbEHoUs8uL9Y7JWo+WX9R+kbtFaa8YEm25SSwYO\n4bAksSaWmw==\n-----END CERTIFICATE-----' + echo $'-----BEGIN CERTIFICATE-----\nMIIDHDCCAgSgAwIBAgIRAOKCbOotUMOQvcitnmwCqI8wDQYJKoZIhvcNAQELBQAw\nEjEQMA4GA1UEChMHUm9vdCBDQTAgFw0yNjA2MDkxMDQ2MDNaGA85OTk5MTIzMTIz\nNTk1OVowEjEQMA4GA1UEChMHUm9vdCBDQTCCASIwDQYJKoZIhvcNAQEBBQADggEP\nADCCAQoCggEBAKWfk5pyPbV1b6Ctsa64tAtZGxqxiAzOG9NSDB90nX+mSuYDZcZr\n4XsFD3uaCXP2zbdlOm28D67lt9n9ZP1y2gwzY2vaehHFMMB/ybJAfZ2ba9IDr8M9\njxqDapgOc82VMCrFTUPrfHI/ZeRKpcOYn8aLWeQJdsDCjfS82wf/B1cIdQ2haIvk\nTykjbKxA1CPe+ORR6655DxJlHSW+Kgo8XyMUDTxHY9B1JoogWfaNwYAG0C3mi3Rn\nrjC08gY+KGlEuCGEpI6c72klw7/FnWs/ZMQrW4DbKTkeSV+LzPGGrdpcANeQbnr4\nSQEShtVhXL5oRO2td3AZQ0vP7o2IOHnLrrsCAwEAAaNrMGkwDgYDVR0PAQH/BAQD\nAgIkMCcGA1UdJQQgMB4GCCsGAQUFBwMBBggrBgEFBQcDAgYIKwYBBQUHAwMwDwYD\nVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUHNwdIX9ky63IirIrX9VQ/AI2GIUwDQYJ\nKoZIhvcNAQELBQADggEBAJ+Tvunj7OmoyiiJfXojbqsZrc924jhJAnGJR3GzVcas\n1IQnS4bog54gfVqNf84InSM9WL38SBPJkRNE1PPlcMdy0Z+1Su/KP/LhKyVYN1ds\nBCnnMAelDQD+5h9XeH7QMkuOudgbnMXDQuoOiciv59dmVA9BoC42I/ljKCvB1EON\nEwIXjVMv05Lx4OuyThTyn8zded8hTiP9yRC0RKYB5LCdBie6g4LmpFQU4NWJRLhy\nvLtm7Mrq699SOOcIJAX3mFDjwol9EQuH/x95vPfeyQfYgAeaP/sS08W6nGcYSTGb\nCeTSH5KFnsOV3Z5+4xNs8hMyqJDTgMGk3Dw3aGoUTcU=\n-----END CERTIFICATE-----' + echo $'-----BEGIN CERTIFICATE-----\nMIIIczCCB1ugAwIBAgIRAKjs79PWN/yUHxrXczThMFAwDQYJKoZIhvcNAQELBQAw\nEjEQMA4GA1UEChMHUm9vdCBDQTAgFw0yNjA2MDkxMDQ2MDNaGA85OTk5MTIzMTIz\nNTk1OVowEDEOMAwGA1UEChMFUFNNREIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw\nggEKAoIBAQCedVULBFLto7t8pH9/gZp6IGYvv0Q/Mx7mIzJwdXZRJaEKkxlUnwx2\nbfFoAVo7W3cpXJvRXOtQCmYIn92EBYaC+urVw3AnnPxX+9ib03VyiimhAlZIK0Bd\n+WH2lbFLP6UU4gLB3dCNuUY4Hsct7/s6g6i2mxG5WmuoXfNLYlauaAhNjkRQUEgm\nriODN2oNUdW9H82nUXsbQ0zIMkYLsDhlY8uMc6ifF6KEa9mL++CYkVDA5WUHBPWf\nm35RHzw5vKR6s94ZYb/dTTaA6hrAeF6ThW/Q3UazYApvFCGdH9Baeo4YMY1Lr2hl\n713hOx853bgRoIOlLlbhvLYjvmP37oB1AgMBAAGjggXCMIIFvjAOBgNVHQ8BAf8E\nBAMCBaAwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMAwGA1UdEwEB/wQC\nMAAwHwYDVR0jBBgwFoAUHNwdIX9ky63IirIrX9VQ/AI2GIUwggVcBgNVHREEggVT\nMIIFT4IJbG9jYWxob3N0gg1zb21lLW5hbWUtcnMwgixzb21lLW5hbWUtcnMwLnNw\nbGl0LWhvcml6b24tbWFudWFsLXRscy0yNDMzNII+c29tZS1uYW1lLXJzMC5zcGxp\ndC1ob3Jpem9uLW1hbnVhbC10bHMtMjQzMzQuc3ZjLmNsdXN0ZXIubG9jYWyCDyou\nc29tZS1uYW1lLXJzMIIuKi5zb21lLW5hbWUtcnMwLnNwbGl0LWhvcml6b24tbWFu\ndWFsLXRscy0yNDMzNIJAKi5zb21lLW5hbWUtcnMwLnNwbGl0LWhvcml6b24tbWFu\ndWFsLXRscy0yNDMzNC5zdmMuY2x1c3Rlci5sb2NhbIJBc29tZS1uYW1lLXJzMC5z\ncGxpdC1ob3Jpem9uLW1hbnVhbC10bHMtMjQzMzQuc3ZjLmNsdXN0ZXJzZXQubG9j\nYWyCQyouc29tZS1uYW1lLXJzMC5zcGxpdC1ob3Jpem9uLW1hbnVhbC10bHMtMjQz\nMzQuc3ZjLmNsdXN0ZXJzZXQubG9jYWyCHXNvbWUtbmFtZS1yczAtMC5jbG91ZGRl\nbW8ueHl6gh1zb21lLW5hbWUtcnMwLTEuY2xvdWRkZW1vLnh5eoIdc29tZS1uYW1l\nLXJzMC0yLmNsb3VkZGVtby54eXqCNSouc3BsaXQtaG9yaXpvbi1tYW51YWwtdGxz\nLTI0MzM0LnN2Yy5jbHVzdGVyc2V0LmxvY2FsghBzb21lLW5hbWUtbW9uZ29zgi9z\nb21lLW5hbWUtbW9uZ29zLnNwbGl0LWhvcml6b24tbWFudWFsLXRscy0yNDMzNIJB\nc29tZS1uYW1lLW1vbmdvcy5zcGxpdC1ob3Jpem9uLW1hbnVhbC10bHMtMjQzMzQu\nc3ZjLmNsdXN0ZXIubG9jYWyCEiouc29tZS1uYW1lLW1vbmdvc4IxKi5zb21lLW5h\nbWUtbW9uZ29zLnNwbGl0LWhvcml6b24tbWFudWFsLXRscy0yNDMzNIJDKi5zb21l\nLW5hbWUtbW9uZ29zLnNwbGl0LWhvcml6b24tbWFudWFsLXRscy0yNDMzNC5zdmMu\nY2x1c3Rlci5sb2NhbIINc29tZS1uYW1lLWNmZ4Isc29tZS1uYW1lLWNmZy5zcGxp\ndC1ob3Jpem9uLW1hbnVhbC10bHMtMjQzMzSCPnNvbWUtbmFtZS1jZmcuc3BsaXQt\naG9yaXpvbi1tYW51YWwtdGxzLTI0MzM0LnN2Yy5jbHVzdGVyLmxvY2Fsgg8qLnNv\nbWUtbmFtZS1jZmeCLiouc29tZS1uYW1lLWNmZy5zcGxpdC1ob3Jpem9uLW1hbnVh\nbC10bHMtMjQzMzSCQCouc29tZS1uYW1lLWNmZy5zcGxpdC1ob3Jpem9uLW1hbnVh\nbC10bHMtMjQzMzQuc3ZjLmNsdXN0ZXIubG9jYWyCRHNvbWUtbmFtZS1tb25nb3Mu\nc3BsaXQtaG9yaXpvbi1tYW51YWwtdGxzLTI0MzM0LnN2Yy5jbHVzdGVyc2V0Lmxv\nY2FsgkYqLnNvbWUtbmFtZS1tb25nb3Muc3BsaXQtaG9yaXpvbi1tYW51YWwtdGxz\nLTI0MzM0LnN2Yy5jbHVzdGVyc2V0LmxvY2FsgkFzb21lLW5hbWUtY2ZnLnNwbGl0\nLWhvcml6b24tbWFudWFsLXRscy0yNDMzNC5zdmMuY2x1c3RlcnNldC5sb2NhbIJD\nKi5zb21lLW5hbWUtY2ZnLnNwbGl0LWhvcml6b24tbWFudWFsLXRscy0yNDMzNC5z\ndmMuY2x1c3RlcnNldC5sb2NhbDANBgkqhkiG9w0BAQsFAAOCAQEAIEA2fvFUXr2M\ngEAhbNrijuUs/kfjO8yujPAAkPhlGUGZa9JIMiRObtqKIv2pqnYPI06U+/TpDXyL\nsBVH8rQD4AHadKqA6vp0iCs7xfruMocwXUIXY1sG9nt6Q742hgCH5ImdNrZOhTw5\nwA9dEq2taXDGq0AlPnS65EW8Pn8tG7gM366lgTzrezNMFvav6rGdTFTXYYvErMJ4\nPaRp6OzMQ7S6UXcWJSkAUOBfRWfP1LFF9r2JVIePtFMvQ21OylZm18appXdOwkX2\nUt0JXVW3N8sj1OQP8RGVjqi0hZbEHoUs8uL9Y7JWo+WX9R+kbtFaa8YEm25SSwYO\n4bAksSaWmw==\n-----END CERTIFICATE-----' + openssl verify -CAfile /tmp/tmp.i1hhairPLp/ca.crt /tmp/tmp.i1hhairPLp/tls.crt + echo 'OK: '\''some-name-ssl-internal'\'' is signed by CA in '\''some-name-ca-cert'\''' OK: 'some-name-ssl-internal' is signed by CA in 'some-name-ca-cert' + desc 'verify split-horizon DNS names are in certificate SANs' + set +o xtrace ----------------------------------------------------------------------------------- verify split-horizon DNS names are in certificate SANs ----------------------------------------------------------------------------------- + verify_cert_san some-name-ssl some-name-rs0-0.clouddemo.xyz + local secret_name=some-name-ssl + local expected_san=some-name-rs0-0.clouddemo.xyz + local san_list ++ kubectl_bin get secret some-name-ssl -o 'jsonpath={.data.tls\.crt}' ++ base64 -d +++ mktemp ++ openssl x509 -text -noout ++ grep DNS: ++ tr , '\n' ++ local LAST_OUT=/tmp/tmp.mhxGgW5C2p +++ mktemp ++ local LAST_ERR=/tmp/tmp.K0ApfxGDhZ ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get secret some-name-ssl -o 'jsonpath={.data.tls\.crt}' ++ xargs ++ sed 's/.*DNS://g' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.mhxGgW5C2p ++ cat /tmp/tmp.K0ApfxGDhZ ++ rm /tmp/tmp.mhxGgW5C2p /tmp/tmp.K0ApfxGDhZ ++ return 0 + san_list='localhost some-name-rs0 some-name-rs0.split-horizon-manual-tls-24334 some-name-rs0.split-horizon-manual-tls-24334.svc.cluster.local *.some-name-rs0 *.some-name-rs0.split-horizon-manual-tls-24334 *.some-name-rs0.split-horizon-manual-tls-24334.svc.cluster.local some-name-rs0.split-horizon-manual-tls-24334.svc.clusterset.local *.some-name-rs0.split-horizon-manual-tls-24334.svc.clusterset.local some-name-rs0-0.clouddemo.xyz some-name-rs0-1.clouddemo.xyz some-name-rs0-2.clouddemo.xyz *.split-horizon-manual-tls-24334.svc.clusterset.local some-name-mongos some-name-mongos.split-horizon-manual-tls-24334 some-name-mongos.split-horizon-manual-tls-24334.svc.cluster.local *.some-name-mongos *.some-name-mongos.split-horizon-manual-tls-24334 *.some-name-mongos.split-horizon-manual-tls-24334.svc.cluster.local some-name-cfg some-name-cfg.split-horizon-manual-tls-24334 some-name-cfg.split-horizon-manual-tls-24334.svc.cluster.local *.some-name-cfg *.some-name-cfg.split-horizon-manual-tls-24334 *.some-name-cfg.split-horizon-manual-tls-24334.svc.cluster.local some-name-mongos.split-horizon-manual-tls-24334.svc.clusterset.local *.some-name-mongos.split-horizon-manual-tls-24334.svc.clusterset.local some-name-cfg.split-horizon-manual-tls-24334.svc.clusterset.local *.some-name-cfg.split-horizon-manual-tls-24334.svc.clusterset.local' + echo 'localhost some-name-rs0 some-name-rs0.split-horizon-manual-tls-24334 some-name-rs0.split-horizon-manual-tls-24334.svc.cluster.local *.some-name-rs0 *.some-name-rs0.split-horizon-manual-tls-24334 *.some-name-rs0.split-horizon-manual-tls-24334.svc.cluster.local some-name-rs0.split-horizon-manual-tls-24334.svc.clusterset.local *.some-name-rs0.split-horizon-manual-tls-24334.svc.clusterset.local some-name-rs0-0.clouddemo.xyz some-name-rs0-1.clouddemo.xyz some-name-rs0-2.clouddemo.xyz *.split-horizon-manual-tls-24334.svc.clusterset.local some-name-mongos some-name-mongos.split-horizon-manual-tls-24334 some-name-mongos.split-horizon-manual-tls-24334.svc.cluster.local *.some-name-mongos *.some-name-mongos.split-horizon-manual-tls-24334 *.some-name-mongos.split-horizon-manual-tls-24334.svc.cluster.local some-name-cfg some-name-cfg.split-horizon-manual-tls-24334 some-name-cfg.split-horizon-manual-tls-24334.svc.cluster.local *.some-name-cfg *.some-name-cfg.split-horizon-manual-tls-24334 *.some-name-cfg.split-horizon-manual-tls-24334.svc.cluster.local some-name-mongos.split-horizon-manual-tls-24334.svc.clusterset.local *.some-name-mongos.split-horizon-manual-tls-24334.svc.clusterset.local some-name-cfg.split-horizon-manual-tls-24334.svc.clusterset.local *.some-name-cfg.split-horizon-manual-tls-24334.svc.clusterset.local' + grep -q some-name-rs0-0.clouddemo.xyz + echo 'OK: SAN '\''some-name-rs0-0.clouddemo.xyz'\'' found in secret '\''some-name-ssl'\''' OK: SAN 'some-name-rs0-0.clouddemo.xyz' found in secret 'some-name-ssl' + verify_cert_san some-name-ssl some-name-rs0-1.clouddemo.xyz + local secret_name=some-name-ssl + local expected_san=some-name-rs0-1.clouddemo.xyz + local san_list ++ kubectl_bin get secret some-name-ssl -o 'jsonpath={.data.tls\.crt}' ++ base64 -d ++ openssl x509 -text -noout ++ grep DNS: +++ mktemp ++ tr , '\n' ++ sed 's/.*DNS://g' ++ local LAST_OUT=/tmp/tmp.m7Gzq7LPTe ++ xargs +++ mktemp ++ local LAST_ERR=/tmp/tmp.ehZrGnNULg ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get secret some-name-ssl -o 'jsonpath={.data.tls\.crt}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.m7Gzq7LPTe ++ cat /tmp/tmp.ehZrGnNULg ++ rm /tmp/tmp.m7Gzq7LPTe /tmp/tmp.ehZrGnNULg ++ return 0 + san_list='localhost some-name-rs0 some-name-rs0.split-horizon-manual-tls-24334 some-name-rs0.split-horizon-manual-tls-24334.svc.cluster.local *.some-name-rs0 *.some-name-rs0.split-horizon-manual-tls-24334 *.some-name-rs0.split-horizon-manual-tls-24334.svc.cluster.local some-name-rs0.split-horizon-manual-tls-24334.svc.clusterset.local *.some-name-rs0.split-horizon-manual-tls-24334.svc.clusterset.local some-name-rs0-0.clouddemo.xyz some-name-rs0-1.clouddemo.xyz some-name-rs0-2.clouddemo.xyz *.split-horizon-manual-tls-24334.svc.clusterset.local some-name-mongos some-name-mongos.split-horizon-manual-tls-24334 some-name-mongos.split-horizon-manual-tls-24334.svc.cluster.local *.some-name-mongos *.some-name-mongos.split-horizon-manual-tls-24334 *.some-name-mongos.split-horizon-manual-tls-24334.svc.cluster.local some-name-cfg some-name-cfg.split-horizon-manual-tls-24334 some-name-cfg.split-horizon-manual-tls-24334.svc.cluster.local *.some-name-cfg *.some-name-cfg.split-horizon-manual-tls-24334 *.some-name-cfg.split-horizon-manual-tls-24334.svc.cluster.local some-name-mongos.split-horizon-manual-tls-24334.svc.clusterset.local *.some-name-mongos.split-horizon-manual-tls-24334.svc.clusterset.local some-name-cfg.split-horizon-manual-tls-24334.svc.clusterset.local *.some-name-cfg.split-horizon-manual-tls-24334.svc.clusterset.local' + echo 'localhost some-name-rs0 some-name-rs0.split-horizon-manual-tls-24334 some-name-rs0.split-horizon-manual-tls-24334.svc.cluster.local *.some-name-rs0 *.some-name-rs0.split-horizon-manual-tls-24334 *.some-name-rs0.split-horizon-manual-tls-24334.svc.cluster.local some-name-rs0.split-horizon-manual-tls-24334.svc.clusterset.local *.some-name-rs0.split-horizon-manual-tls-24334.svc.clusterset.local some-name-rs0-0.clouddemo.xyz some-name-rs0-1.clouddemo.xyz some-name-rs0-2.clouddemo.xyz *.split-horizon-manual-tls-24334.svc.clusterset.local some-name-mongos some-name-mongos.split-horizon-manual-tls-24334 some-name-mongos.split-horizon-manual-tls-24334.svc.cluster.local *.some-name-mongos *.some-name-mongos.split-horizon-manual-tls-24334 *.some-name-mongos.split-horizon-manual-tls-24334.svc.cluster.local some-name-cfg some-name-cfg.split-horizon-manual-tls-24334 some-name-cfg.split-horizon-manual-tls-24334.svc.cluster.local *.some-name-cfg *.some-name-cfg.split-horizon-manual-tls-24334 *.some-name-cfg.split-horizon-manual-tls-24334.svc.cluster.local some-name-mongos.split-horizon-manual-tls-24334.svc.clusterset.local *.some-name-mongos.split-horizon-manual-tls-24334.svc.clusterset.local some-name-cfg.split-horizon-manual-tls-24334.svc.clusterset.local *.some-name-cfg.split-horizon-manual-tls-24334.svc.clusterset.local' + grep -q some-name-rs0-1.clouddemo.xyz + echo 'OK: SAN '\''some-name-rs0-1.clouddemo.xyz'\'' found in secret '\''some-name-ssl'\''' OK: SAN 'some-name-rs0-1.clouddemo.xyz' found in secret 'some-name-ssl' + verify_cert_san some-name-ssl some-name-rs0-2.clouddemo.xyz + local secret_name=some-name-ssl + local expected_san=some-name-rs0-2.clouddemo.xyz + local san_list ++ kubectl_bin get secret some-name-ssl -o 'jsonpath={.data.tls\.crt}' ++ base64 -d ++ openssl x509 -text -noout +++ mktemp ++ grep DNS: ++ tr , '\n' ++ sed 's/.*DNS://g' ++ local LAST_OUT=/tmp/tmp.NndtfzzcIS +++ mktemp ++ xargs ++ local LAST_ERR=/tmp/tmp.hZhR0vne1v ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get secret some-name-ssl -o 'jsonpath={.data.tls\.crt}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.NndtfzzcIS ++ cat /tmp/tmp.hZhR0vne1v ++ rm /tmp/tmp.NndtfzzcIS /tmp/tmp.hZhR0vne1v ++ return 0 + san_list='localhost some-name-rs0 some-name-rs0.split-horizon-manual-tls-24334 some-name-rs0.split-horizon-manual-tls-24334.svc.cluster.local *.some-name-rs0 *.some-name-rs0.split-horizon-manual-tls-24334 *.some-name-rs0.split-horizon-manual-tls-24334.svc.cluster.local some-name-rs0.split-horizon-manual-tls-24334.svc.clusterset.local *.some-name-rs0.split-horizon-manual-tls-24334.svc.clusterset.local some-name-rs0-0.clouddemo.xyz some-name-rs0-1.clouddemo.xyz some-name-rs0-2.clouddemo.xyz *.split-horizon-manual-tls-24334.svc.clusterset.local some-name-mongos some-name-mongos.split-horizon-manual-tls-24334 some-name-mongos.split-horizon-manual-tls-24334.svc.cluster.local *.some-name-mongos *.some-name-mongos.split-horizon-manual-tls-24334 *.some-name-mongos.split-horizon-manual-tls-24334.svc.cluster.local some-name-cfg some-name-cfg.split-horizon-manual-tls-24334 some-name-cfg.split-horizon-manual-tls-24334.svc.cluster.local *.some-name-cfg *.some-name-cfg.split-horizon-manual-tls-24334 *.some-name-cfg.split-horizon-manual-tls-24334.svc.cluster.local some-name-mongos.split-horizon-manual-tls-24334.svc.clusterset.local *.some-name-mongos.split-horizon-manual-tls-24334.svc.clusterset.local some-name-cfg.split-horizon-manual-tls-24334.svc.clusterset.local *.some-name-cfg.split-horizon-manual-tls-24334.svc.clusterset.local' + echo 'localhost some-name-rs0 some-name-rs0.split-horizon-manual-tls-24334 some-name-rs0.split-horizon-manual-tls-24334.svc.cluster.local *.some-name-rs0 *.some-name-rs0.split-horizon-manual-tls-24334 *.some-name-rs0.split-horizon-manual-tls-24334.svc.cluster.local some-name-rs0.split-horizon-manual-tls-24334.svc.clusterset.local *.some-name-rs0.split-horizon-manual-tls-24334.svc.clusterset.local some-name-rs0-0.clouddemo.xyz some-name-rs0-1.clouddemo.xyz some-name-rs0-2.clouddemo.xyz *.split-horizon-manual-tls-24334.svc.clusterset.local some-name-mongos some-name-mongos.split-horizon-manual-tls-24334 some-name-mongos.split-horizon-manual-tls-24334.svc.cluster.local *.some-name-mongos *.some-name-mongos.split-horizon-manual-tls-24334 *.some-name-mongos.split-horizon-manual-tls-24334.svc.cluster.local some-name-cfg some-name-cfg.split-horizon-manual-tls-24334 some-name-cfg.split-horizon-manual-tls-24334.svc.cluster.local *.some-name-cfg *.some-name-cfg.split-horizon-manual-tls-24334 *.some-name-cfg.split-horizon-manual-tls-24334.svc.cluster.local some-name-mongos.split-horizon-manual-tls-24334.svc.clusterset.local *.some-name-mongos.split-horizon-manual-tls-24334.svc.clusterset.local some-name-cfg.split-horizon-manual-tls-24334.svc.clusterset.local *.some-name-cfg.split-horizon-manual-tls-24334.svc.clusterset.local' + grep -q some-name-rs0-2.clouddemo.xyz + echo 'OK: SAN '\''some-name-rs0-2.clouddemo.xyz'\'' found in secret '\''some-name-ssl'\''' OK: SAN 'some-name-rs0-2.clouddemo.xyz' found in secret 'some-name-ssl' + verify_cert_san some-name-ssl-internal some-name-rs0-0.clouddemo.xyz + local secret_name=some-name-ssl-internal + local expected_san=some-name-rs0-0.clouddemo.xyz + local san_list ++ kubectl_bin get secret some-name-ssl-internal -o 'jsonpath={.data.tls\.crt}' ++ base64 -d +++ mktemp ++ openssl x509 -text -noout ++ grep DNS: ++ tr , '\n' ++ local LAST_OUT=/tmp/tmp.L5wC0TaMi4 ++ sed 's/.*DNS://g' ++ xargs +++ mktemp ++ local LAST_ERR=/tmp/tmp.FvK3hMYL9f ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get secret some-name-ssl-internal -o 'jsonpath={.data.tls\.crt}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.L5wC0TaMi4 ++ cat /tmp/tmp.FvK3hMYL9f ++ rm /tmp/tmp.L5wC0TaMi4 /tmp/tmp.FvK3hMYL9f ++ return 0 + san_list='localhost some-name-rs0 some-name-rs0.split-horizon-manual-tls-24334 some-name-rs0.split-horizon-manual-tls-24334.svc.cluster.local *.some-name-rs0 *.some-name-rs0.split-horizon-manual-tls-24334 *.some-name-rs0.split-horizon-manual-tls-24334.svc.cluster.local some-name-rs0.split-horizon-manual-tls-24334.svc.clusterset.local *.some-name-rs0.split-horizon-manual-tls-24334.svc.clusterset.local some-name-rs0-0.clouddemo.xyz some-name-rs0-1.clouddemo.xyz some-name-rs0-2.clouddemo.xyz *.split-horizon-manual-tls-24334.svc.clusterset.local some-name-mongos some-name-mongos.split-horizon-manual-tls-24334 some-name-mongos.split-horizon-manual-tls-24334.svc.cluster.local *.some-name-mongos *.some-name-mongos.split-horizon-manual-tls-24334 *.some-name-mongos.split-horizon-manual-tls-24334.svc.cluster.local some-name-cfg some-name-cfg.split-horizon-manual-tls-24334 some-name-cfg.split-horizon-manual-tls-24334.svc.cluster.local *.some-name-cfg *.some-name-cfg.split-horizon-manual-tls-24334 *.some-name-cfg.split-horizon-manual-tls-24334.svc.cluster.local some-name-mongos.split-horizon-manual-tls-24334.svc.clusterset.local *.some-name-mongos.split-horizon-manual-tls-24334.svc.clusterset.local some-name-cfg.split-horizon-manual-tls-24334.svc.clusterset.local *.some-name-cfg.split-horizon-manual-tls-24334.svc.clusterset.local' + echo 'localhost some-name-rs0 some-name-rs0.split-horizon-manual-tls-24334 some-name-rs0.split-horizon-manual-tls-24334.svc.cluster.local *.some-name-rs0 *.some-name-rs0.split-horizon-manual-tls-24334 *.some-name-rs0.split-horizon-manual-tls-24334.svc.cluster.local some-name-rs0.split-horizon-manual-tls-24334.svc.clusterset.local *.some-name-rs0.split-horizon-manual-tls-24334.svc.clusterset.local some-name-rs0-0.clouddemo.xyz some-name-rs0-1.clouddemo.xyz some-name-rs0-2.clouddemo.xyz *.split-horizon-manual-tls-24334.svc.clusterset.local some-name-mongos some-name-mongos.split-horizon-manual-tls-24334 some-name-mongos.split-horizon-manual-tls-24334.svc.cluster.local *.some-name-mongos *.some-name-mongos.split-horizon-manual-tls-24334 *.some-name-mongos.split-horizon-manual-tls-24334.svc.cluster.local some-name-cfg some-name-cfg.split-horizon-manual-tls-24334 some-name-cfg.split-horizon-manual-tls-24334.svc.cluster.local *.some-name-cfg *.some-name-cfg.split-horizon-manual-tls-24334 *.some-name-cfg.split-horizon-manual-tls-24334.svc.cluster.local some-name-mongos.split-horizon-manual-tls-24334.svc.clusterset.local *.some-name-mongos.split-horizon-manual-tls-24334.svc.clusterset.local some-name-cfg.split-horizon-manual-tls-24334.svc.clusterset.local *.some-name-cfg.split-horizon-manual-tls-24334.svc.clusterset.local' + grep -q some-name-rs0-0.clouddemo.xyz + echo 'OK: SAN '\''some-name-rs0-0.clouddemo.xyz'\'' found in secret '\''some-name-ssl-internal'\''' OK: SAN 'some-name-rs0-0.clouddemo.xyz' found in secret 'some-name-ssl-internal' + desc 'save certificate fingerprint before horizon update' + set +o xtrace ----------------------------------------------------------------------------------- save certificate fingerprint before horizon update ----------------------------------------------------------------------------------- ++ save_cert_hash some-name-ssl ++ local secret_name=some-name-ssl ++ local output_var= ++ kubectl_bin get secret some-name-ssl -o 'jsonpath={.data.tls\.crt}' ++ base64 -d ++ openssl x509 -fingerprint -noout +++ mktemp ++ local LAST_OUT=/tmp/tmp.zDXEC1bX0D +++ mktemp ++ local LAST_ERR=/tmp/tmp.xCu7mBlPY1 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get secret some-name-ssl -o 'jsonpath={.data.tls\.crt}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.zDXEC1bX0D ++ cat /tmp/tmp.xCu7mBlPY1 ++ rm /tmp/tmp.zDXEC1bX0D /tmp/tmp.xCu7mBlPY1 ++ return 0 + cert_hash_before='SHA1 Fingerprint=58:E9:51:1A:90:00:9B:17:3B:15:F7:62:8D:2C:CF:40:55:17:2C:49' + configure_client_hostAliases + local 'hostAliasesJson=[]' ++ kubectl get svc ++ awk '{print $3 "|" $1}' ++ grep -E '^[0-9].*' + for svc in $(kubectl get svc | awk '{print $3 "|" $1}' | grep -E '^[0-9].*') ++ echo '34.118.234.8|some-name-rs0-0' ++ awk -F '|' '{print $2}' + hostname=some-name-rs0-0 ++ echo '34.118.234.8|some-name-rs0-0' ++ awk -F '|' '{print $1}' + ip=34.118.234.8 + hostAlias='{"ip": "34.118.234.8", "hostnames": ["some-name-rs0-0.clouddemo.xyz"]}' ++ echo '[]' ++ jq --argjson newAlias '{"ip": "34.118.234.8", "hostnames": ["some-name-rs0-0.clouddemo.xyz"]}' '. += [$newAlias]' + hostAliasesJson=$'[\n {\n "ip": "34.118.234.8",\n "hostnames": [\n "some-name-rs0-0.clouddemo.xyz"\n ]\n }\n]' + for svc in $(kubectl get svc | awk '{print $3 "|" $1}' | grep -E '^[0-9].*') ++ echo '34.118.229.29|some-name-rs0-1' ++ awk -F '|' '{print $2}' + hostname=some-name-rs0-1 ++ echo '34.118.229.29|some-name-rs0-1' ++ awk -F '|' '{print $1}' + ip=34.118.229.29 + hostAlias='{"ip": "34.118.229.29", "hostnames": ["some-name-rs0-1.clouddemo.xyz"]}' ++ echo $'[\n {\n "ip": "34.118.234.8",\n "hostnames": [\n "some-name-rs0-0.clouddemo.xyz"\n ]\n }\n]' ++ jq --argjson newAlias '{"ip": "34.118.229.29", "hostnames": ["some-name-rs0-1.clouddemo.xyz"]}' '. += [$newAlias]' + hostAliasesJson=$'[\n {\n "ip": "34.118.234.8",\n "hostnames": [\n "some-name-rs0-0.clouddemo.xyz"\n ]\n },\n {\n "ip": "34.118.229.29",\n "hostnames": [\n "some-name-rs0-1.clouddemo.xyz"\n ]\n }\n]' + for svc in $(kubectl get svc | awk '{print $3 "|" $1}' | grep -E '^[0-9].*') ++ echo '34.118.230.62|some-name-rs0-2' ++ awk -F '|' '{print $2}' + hostname=some-name-rs0-2 ++ echo '34.118.230.62|some-name-rs0-2' ++ awk -F '|' '{print $1}' + ip=34.118.230.62 + hostAlias='{"ip": "34.118.230.62", "hostnames": ["some-name-rs0-2.clouddemo.xyz"]}' ++ echo $'[\n {\n "ip": "34.118.234.8",\n "hostnames": [\n "some-name-rs0-0.clouddemo.xyz"\n ]\n },\n {\n "ip": "34.118.229.29",\n "hostnames": [\n "some-name-rs0-1.clouddemo.xyz"\n ]\n }\n]' ++ jq --argjson newAlias '{"ip": "34.118.230.62", "hostnames": ["some-name-rs0-2.clouddemo.xyz"]}' '. += [$newAlias]' + hostAliasesJson=$'[\n {\n "ip": "34.118.234.8",\n "hostnames": [\n "some-name-rs0-0.clouddemo.xyz"\n ]\n },\n {\n "ip": "34.118.229.29",\n "hostnames": [\n "some-name-rs0-1.clouddemo.xyz"\n ]\n },\n {\n "ip": "34.118.230.62",\n "hostnames": [\n "some-name-rs0-2.clouddemo.xyz"\n ]\n }\n]' + kubectl_bin patch deployment psmdb-client --type=json '-p=[{'\''op'\'': '\''replace'\'', '\''path'\'': '\''/spec/replicas'\'', '\''value'\'': 0}]' ++ mktemp + local LAST_OUT=/tmp/tmp.U6uVxdSuaM ++ mktemp + local LAST_ERR=/tmp/tmp.ci7YGKZth0 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl patch deployment psmdb-client --type=json '-p=[{'\''op'\'': '\''replace'\'', '\''path'\'': '\''/spec/replicas'\'', '\''value'\'': 0}]' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.U6uVxdSuaM deployment.apps/psmdb-client patched + cat /tmp/tmp.ci7YGKZth0 + rm /tmp/tmp.U6uVxdSuaM /tmp/tmp.ci7YGKZth0 + return 0 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.sX8OwTVYbS +++ mktemp ++ local LAST_ERR=/tmp/tmp.lPJEYnsbII ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.sX8OwTVYbS ++ cat /tmp/tmp.lPJEYnsbII ++ rm /tmp/tmp.sX8OwTVYbS /tmp/tmp.lPJEYnsbII ++ return 0 + wait_for_delete pod/psmdb-client-5649fbb65f-6bt47 + local res=pod/psmdb-client-5649fbb65f-6bt47 + local wait_time=60 + set +o xtrace waiting for pod/psmdb-client-5649fbb65f-6bt47 to be deleted.....Error from server (NotFound): pods "psmdb-client-5649fbb65f-6bt47" not found Error from server (NotFound): pods "psmdb-client-5649fbb65f-6bt47" not found Error from server (NotFound): pods "psmdb-client-5649fbb65f-6bt47" not found Error from server (NotFound): pods "psmdb-client-5649fbb65f-6bt47" not found + kubectl_bin patch deployment psmdb-client --type=json $'-p=[{\'op\': \'replace\', \'path\': \'/spec/template/spec/hostAliases\', \'value\': [\n {\n "ip": "34.118.234.8",\n "hostnames": [\n "some-name-rs0-0.clouddemo.xyz"\n ]\n },\n {\n "ip": "34.118.229.29",\n "hostnames": [\n "some-name-rs0-1.clouddemo.xyz"\n ]\n },\n {\n "ip": "34.118.230.62",\n "hostnames": [\n "some-name-rs0-2.clouddemo.xyz"\n ]\n }\n]}, {\'op\': \'replace\', \'path\': \'/spec/replicas\', \'value\': 1}]' ++ mktemp + local LAST_OUT=/tmp/tmp.oABeO9rIXF ++ mktemp + local LAST_ERR=/tmp/tmp.gLGAPOupij + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl patch deployment psmdb-client --type=json $'-p=[{\'op\': \'replace\', \'path\': \'/spec/template/spec/hostAliases\', \'value\': [\n {\n "ip": "34.118.234.8",\n "hostnames": [\n "some-name-rs0-0.clouddemo.xyz"\n ]\n },\n {\n "ip": "34.118.229.29",\n "hostnames": [\n "some-name-rs0-1.clouddemo.xyz"\n ]\n },\n {\n "ip": "34.118.230.62",\n "hostnames": [\n "some-name-rs0-2.clouddemo.xyz"\n ]\n }\n]}, {\'op\': \'replace\', \'path\': \'/spec/replicas\', \'value\': 1}]' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.oABeO9rIXF deployment.apps/psmdb-client patched + cat /tmp/tmp.gLGAPOupij + rm /tmp/tmp.oABeO9rIXF /tmp/tmp.gLGAPOupij + return 0 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.J3xPr8PRkZ +++ mktemp ++ local LAST_ERR=/tmp/tmp.Z36yHF7aSA ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.J3xPr8PRkZ ++ cat /tmp/tmp.Z36yHF7aSA ++ rm /tmp/tmp.J3xPr8PRkZ /tmp/tmp.Z36yHF7aSA ++ return 0 + wait_pod psmdb-client-6bd9478dff-xj848 + local pod=psmdb-client-6bd9478dff-xj848 + set +o xtrace waiting for pod/psmdb-client-6bd9478dff-xj848 to be ready.OK + sleep 10 + desc 'verify horizons via rs.conf()' + set +o xtrace ----------------------------------------------------------------------------------- verify horizons via rs.conf() ----------------------------------------------------------------------------------- + run_mongo_tls 'rs.conf().members.map(function(member) { return member.horizons }).sort((a, b) => a.external.localeCompare(b.external))' clusterAdmin:clusterAdmin123456@some-name-rs0-0.clouddemo.xyz,some-name-rs0-1.clouddemo.xyz,some-name-rs0-2.clouddemo.xyz mongodb '' --quiet + local 'command=rs.conf().members.map(function(member) { return member.horizons }).sort((a, b) => a.external.localeCompare(b.external))' + local uri=clusterAdmin:clusterAdmin123456@some-name-rs0-0.clouddemo.xyz,some-name-rs0-1.clouddemo.xyz,some-name-rs0-2.clouddemo.xyz + local driver=mongodb + local suffix=.svc.cluster.local + grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|does not match the remote host name' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.zNoJfn0oqs +++ mktemp ++ local LAST_ERR=/tmp/tmp.FAwtN3qBea ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.zNoJfn0oqs ++ cat /tmp/tmp.FAwtN3qBea ++ rm /tmp/tmp.zNoJfn0oqs /tmp/tmp.FAwtN3qBea ++ return 0 + local client_container=psmdb-client-6bd9478dff-xj848 + local mongo_flag=--quiet + local port=27017 ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 + [[ clusterAdmin:clusterAdmin123456@some-name-rs0-0.clouddemo.xyz,some-name-rs0-1.clouddemo.xyz,some-name-rs0-2.clouddemo.xyz == *cfg* ]] + replica_set=rs0 + kubectl_bin exec psmdb-client-6bd9478dff-xj848 -- bash -c 'printf '\''rs.conf().members.map(function(member) { return member.horizons }).sort((a, b) => a.external.localeCompare(b.external))\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@some-name-rs0-0.clouddemo.xyz,some-name-rs0-1.clouddemo.xyz,some-name-rs0-2.clouddemo.xyz.svc.cluster.local:27017/admin?replicaSet=rs0 --tls --tlsCAFile /etc/mongodb-ssl/ca.crt --tlsCertificateKeyFile /tmp/tls.pem --tlsAllowInvalidHostnames --quiet' ++ mktemp + local LAST_OUT=/tmp/tmp.vp2fkIO6vD ++ mktemp + local LAST_ERR=/tmp/tmp.TBZTgfnhhT + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-6bd9478dff-xj848 -- bash -c 'printf '\''rs.conf().members.map(function(member) { return member.horizons }).sort((a, b) => a.external.localeCompare(b.external))\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@some-name-rs0-0.clouddemo.xyz,some-name-rs0-1.clouddemo.xyz,some-name-rs0-2.clouddemo.xyz.svc.cluster.local:27017/admin?replicaSet=rs0 --tls --tlsCAFile /etc/mongodb-ssl/ca.crt --tlsCertificateKeyFile /tmp/tls.pem --tlsAllowInvalidHostnames --quiet' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.vp2fkIO6vD + cat /tmp/tmp.TBZTgfnhhT + rm /tmp/tmp.vp2fkIO6vD /tmp/tmp.TBZTgfnhhT + return 0 + diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/split-horizon-manual-tls/compare/horizons-3.json /tmp/tmp.i1hhairPLp/horizons-3.json + desc 'update to 5 horizons (triggers SAN change and cert re-signing)' + set +o xtrace ----------------------------------------------------------------------------------- update to 5 horizons (triggers SAN change and cert re-signing) ----------------------------------------------------------------------------------- + apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/split-horizon-manual-tls/conf/some-name-5horizons.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/split-horizon-manual-tls/conf/some-name-5horizons.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/split-horizon-manual-tls/conf/some-name-5horizons.yml ++ mktemp + yq eval '(.spec | select(.image == null)).image = "docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0"' + yq eval '(.spec | select(has("pmm"))).pmm.image = "docker.io/percona/pmm-client:2.44.1-1"' + yq eval '(.spec | select(has("initImage"))).initImage = "docker.io/perconalab/percona-server-mongodb-operator:PR-2058-fb8cc7187"' + yq eval '(.spec | select(has("backup"))).backup.image = "docker.io/perconalab/percona-server-mongodb-operator:main-backup"' + local LAST_OUT=/tmp/tmp.eoOYiHnchb + /usr/sbin/sed -e s/NAME_SPACE/split-horizon-manual-tls-24334/g ++ mktemp + yq eval '.spec.upgradeOptions.apply="Never"' + local LAST_ERR=/tmp/tmp.Fytkh88W0N + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.eoOYiHnchb perconaservermongodb.psmdb.percona.com/some-name configured + cat /tmp/tmp.Fytkh88W0N + rm /tmp/tmp.eoOYiHnchb /tmp/tmp.Fytkh88W0N + return 0 + wait_for_running some-name-rs0 3 + local name=some-name-rs0 + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-rs0-0 + local pod=some-name-rs0-0 + set +o xtrace waiting for pod/some-name-rs0-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-rs0-1 + local pod=some-name-rs0-1 + set +o xtrace waiting for pod/some-name-rs0-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.OEdgZdvnt8 +++ mktemp ++ local LAST_ERR=/tmp/tmp.WJXgimD4MM ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.OEdgZdvnt8 ++ cat /tmp/tmp.WJXgimD4MM ++ rm /tmp/tmp.OEdgZdvnt8 /tmp/tmp.WJXgimD4MM ++ return 0 + [[ '' == true ]] + wait_pod some-name-rs0-2 + local pod=some-name-rs0-2 + set +o xtrace waiting for pod/some-name-rs0-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.WlLrnmpEvk +++ mktemp ++ local LAST_ERR=/tmp/tmp.zDlBQc3Wfw ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.WlLrnmpEvk ++ cat /tmp/tmp.zDlBQc3Wfw ++ rm /tmp/tmp.WlLrnmpEvk /tmp/tmp.zDlBQc3Wfw ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Fufid1lT9D +++ mktemp ++ local LAST_ERR=/tmp/tmp.M4u0Q0RDWQ ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Fufid1lT9D ++ cat /tmp/tmp.M4u0Q0RDWQ ++ rm /tmp/tmp.Fufid1lT9D /tmp/tmp.M4u0Q0RDWQ ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster readyness................................................................. + wait_cluster_consistency some-name + local cluster_name=some-name + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster readyness' waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.JbRZiZefIa +++ mktemp ++ local LAST_ERR=/tmp/tmp.ngs6YSz4Wo ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.JbRZiZefIa ++ cat /tmp/tmp.ngs6YSz4Wo ++ rm /tmp/tmp.JbRZiZefIa /tmp/tmp.ngs6YSz4Wo ++ return 0 + [[ ready == ready ]] + echo .OK .OK + desc 'verify new horizon DNS names are in certificate SANs after re-signing' + set +o xtrace ----------------------------------------------------------------------------------- verify new horizon DNS names are in certificate SANs after re-signing ----------------------------------------------------------------------------------- + verify_cert_san some-name-ssl some-name-rs0-3.clouddemo.xyz + local secret_name=some-name-ssl + local expected_san=some-name-rs0-3.clouddemo.xyz + local san_list ++ kubectl_bin get secret some-name-ssl -o 'jsonpath={.data.tls\.crt}' ++ base64 -d ++ openssl x509 -text -noout ++ grep DNS: ++ tr , '\n' ++ sed 's/.*DNS://g' +++ mktemp ++ xargs ++ local LAST_OUT=/tmp/tmp.FfYimdNY94 +++ mktemp ++ local LAST_ERR=/tmp/tmp.NNkVj7QJ2N ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get secret some-name-ssl -o 'jsonpath={.data.tls\.crt}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.FfYimdNY94 ++ cat /tmp/tmp.NNkVj7QJ2N ++ rm /tmp/tmp.FfYimdNY94 /tmp/tmp.NNkVj7QJ2N ++ return 0 + san_list='localhost some-name-rs0 some-name-rs0.split-horizon-manual-tls-24334 some-name-rs0.split-horizon-manual-tls-24334.svc.cluster.local *.some-name-rs0 *.some-name-rs0.split-horizon-manual-tls-24334 *.some-name-rs0.split-horizon-manual-tls-24334.svc.cluster.local some-name-rs0.split-horizon-manual-tls-24334.svc.clusterset.local *.some-name-rs0.split-horizon-manual-tls-24334.svc.clusterset.local some-name-rs0-0.clouddemo.xyz some-name-rs0-1.clouddemo.xyz some-name-rs0-2.clouddemo.xyz some-name-rs0-3.clouddemo.xyz some-name-rs0-4.clouddemo.xyz *.split-horizon-manual-tls-24334.svc.clusterset.local some-name-mongos some-name-mongos.split-horizon-manual-tls-24334 some-name-mongos.split-horizon-manual-tls-24334.svc.cluster.local *.some-name-mongos *.some-name-mongos.split-horizon-manual-tls-24334 *.some-name-mongos.split-horizon-manual-tls-24334.svc.cluster.local some-name-cfg some-name-cfg.split-horizon-manual-tls-24334 some-name-cfg.split-horizon-manual-tls-24334.svc.cluster.local *.some-name-cfg *.some-name-cfg.split-horizon-manual-tls-24334 *.some-name-cfg.split-horizon-manual-tls-24334.svc.cluster.local some-name-mongos.split-horizon-manual-tls-24334.svc.clusterset.local *.some-name-mongos.split-horizon-manual-tls-24334.svc.clusterset.local some-name-cfg.split-horizon-manual-tls-24334.svc.clusterset.local *.some-name-cfg.split-horizon-manual-tls-24334.svc.clusterset.local' + echo 'localhost some-name-rs0 some-name-rs0.split-horizon-manual-tls-24334 some-name-rs0.split-horizon-manual-tls-24334.svc.cluster.local *.some-name-rs0 *.some-name-rs0.split-horizon-manual-tls-24334 *.some-name-rs0.split-horizon-manual-tls-24334.svc.cluster.local some-name-rs0.split-horizon-manual-tls-24334.svc.clusterset.local *.some-name-rs0.split-horizon-manual-tls-24334.svc.clusterset.local some-name-rs0-0.clouddemo.xyz some-name-rs0-1.clouddemo.xyz some-name-rs0-2.clouddemo.xyz some-name-rs0-3.clouddemo.xyz some-name-rs0-4.clouddemo.xyz *.split-horizon-manual-tls-24334.svc.clusterset.local some-name-mongos some-name-mongos.split-horizon-manual-tls-24334 some-name-mongos.split-horizon-manual-tls-24334.svc.cluster.local *.some-name-mongos *.some-name-mongos.split-horizon-manual-tls-24334 *.some-name-mongos.split-horizon-manual-tls-24334.svc.cluster.local some-name-cfg some-name-cfg.split-horizon-manual-tls-24334 some-name-cfg.split-horizon-manual-tls-24334.svc.cluster.local *.some-name-cfg *.some-name-cfg.split-horizon-manual-tls-24334 *.some-name-cfg.split-horizon-manual-tls-24334.svc.cluster.local some-name-mongos.split-horizon-manual-tls-24334.svc.clusterset.local *.some-name-mongos.split-horizon-manual-tls-24334.svc.clusterset.local some-name-cfg.split-horizon-manual-tls-24334.svc.clusterset.local *.some-name-cfg.split-horizon-manual-tls-24334.svc.clusterset.local' + grep -q some-name-rs0-3.clouddemo.xyz + echo 'OK: SAN '\''some-name-rs0-3.clouddemo.xyz'\'' found in secret '\''some-name-ssl'\''' OK: SAN 'some-name-rs0-3.clouddemo.xyz' found in secret 'some-name-ssl' + verify_cert_san some-name-ssl some-name-rs0-4.clouddemo.xyz + local secret_name=some-name-ssl + local expected_san=some-name-rs0-4.clouddemo.xyz + local san_list ++ kubectl_bin get secret some-name-ssl -o 'jsonpath={.data.tls\.crt}' ++ base64 -d ++ openssl x509 -text -noout ++ grep DNS: +++ mktemp ++ tr , '\n' ++ sed 's/.*DNS://g' ++ local LAST_OUT=/tmp/tmp.Q18RmMecjM ++ xargs +++ mktemp ++ local LAST_ERR=/tmp/tmp.7HlKyDNZcx ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get secret some-name-ssl -o 'jsonpath={.data.tls\.crt}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Q18RmMecjM ++ cat /tmp/tmp.7HlKyDNZcx ++ rm /tmp/tmp.Q18RmMecjM /tmp/tmp.7HlKyDNZcx ++ return 0 + san_list='localhost some-name-rs0 some-name-rs0.split-horizon-manual-tls-24334 some-name-rs0.split-horizon-manual-tls-24334.svc.cluster.local *.some-name-rs0 *.some-name-rs0.split-horizon-manual-tls-24334 *.some-name-rs0.split-horizon-manual-tls-24334.svc.cluster.local some-name-rs0.split-horizon-manual-tls-24334.svc.clusterset.local *.some-name-rs0.split-horizon-manual-tls-24334.svc.clusterset.local some-name-rs0-0.clouddemo.xyz some-name-rs0-1.clouddemo.xyz some-name-rs0-2.clouddemo.xyz some-name-rs0-3.clouddemo.xyz some-name-rs0-4.clouddemo.xyz *.split-horizon-manual-tls-24334.svc.clusterset.local some-name-mongos some-name-mongos.split-horizon-manual-tls-24334 some-name-mongos.split-horizon-manual-tls-24334.svc.cluster.local *.some-name-mongos *.some-name-mongos.split-horizon-manual-tls-24334 *.some-name-mongos.split-horizon-manual-tls-24334.svc.cluster.local some-name-cfg some-name-cfg.split-horizon-manual-tls-24334 some-name-cfg.split-horizon-manual-tls-24334.svc.cluster.local *.some-name-cfg *.some-name-cfg.split-horizon-manual-tls-24334 *.some-name-cfg.split-horizon-manual-tls-24334.svc.cluster.local some-name-mongos.split-horizon-manual-tls-24334.svc.clusterset.local *.some-name-mongos.split-horizon-manual-tls-24334.svc.clusterset.local some-name-cfg.split-horizon-manual-tls-24334.svc.clusterset.local *.some-name-cfg.split-horizon-manual-tls-24334.svc.clusterset.local' + echo 'localhost some-name-rs0 some-name-rs0.split-horizon-manual-tls-24334 some-name-rs0.split-horizon-manual-tls-24334.svc.cluster.local *.some-name-rs0 *.some-name-rs0.split-horizon-manual-tls-24334 *.some-name-rs0.split-horizon-manual-tls-24334.svc.cluster.local some-name-rs0.split-horizon-manual-tls-24334.svc.clusterset.local *.some-name-rs0.split-horizon-manual-tls-24334.svc.clusterset.local some-name-rs0-0.clouddemo.xyz some-name-rs0-1.clouddemo.xyz some-name-rs0-2.clouddemo.xyz some-name-rs0-3.clouddemo.xyz some-name-rs0-4.clouddemo.xyz *.split-horizon-manual-tls-24334.svc.clusterset.local some-name-mongos some-name-mongos.split-horizon-manual-tls-24334 some-name-mongos.split-horizon-manual-tls-24334.svc.cluster.local *.some-name-mongos *.some-name-mongos.split-horizon-manual-tls-24334 *.some-name-mongos.split-horizon-manual-tls-24334.svc.cluster.local some-name-cfg some-name-cfg.split-horizon-manual-tls-24334 some-name-cfg.split-horizon-manual-tls-24334.svc.cluster.local *.some-name-cfg *.some-name-cfg.split-horizon-manual-tls-24334 *.some-name-cfg.split-horizon-manual-tls-24334.svc.cluster.local some-name-mongos.split-horizon-manual-tls-24334.svc.clusterset.local *.some-name-mongos.split-horizon-manual-tls-24334.svc.clusterset.local some-name-cfg.split-horizon-manual-tls-24334.svc.clusterset.local *.some-name-cfg.split-horizon-manual-tls-24334.svc.clusterset.local' + grep -q some-name-rs0-4.clouddemo.xyz + echo 'OK: SAN '\''some-name-rs0-4.clouddemo.xyz'\'' found in secret '\''some-name-ssl'\''' OK: SAN 'some-name-rs0-4.clouddemo.xyz' found in secret 'some-name-ssl' + desc 'verify certificate was re-signed (fingerprint changed)' + set +o xtrace ----------------------------------------------------------------------------------- verify certificate was re-signed (fingerprint changed) ----------------------------------------------------------------------------------- ++ save_cert_hash some-name-ssl ++ local secret_name=some-name-ssl ++ local output_var= ++ kubectl_bin get secret some-name-ssl -o 'jsonpath={.data.tls\.crt}' ++ base64 -d ++ openssl x509 -fingerprint -noout +++ mktemp ++ local LAST_OUT=/tmp/tmp.A3j8LrVAxT +++ mktemp ++ local LAST_ERR=/tmp/tmp.UHXcAYxDtr ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get secret some-name-ssl -o 'jsonpath={.data.tls\.crt}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.A3j8LrVAxT ++ cat /tmp/tmp.UHXcAYxDtr ++ rm /tmp/tmp.A3j8LrVAxT /tmp/tmp.UHXcAYxDtr ++ return 0 + cert_hash_after='SHA1 Fingerprint=07:53:00:29:63:E0:F9:72:B4:92:99:F2:11:7E:4B:DF:A4:A8:B9:DE' + '[' 'SHA1 Fingerprint=58:E9:51:1A:90:00:9B:17:3B:15:F7:62:8D:2C:CF:40:55:17:2C:49' = 'SHA1 Fingerprint=07:53:00:29:63:E0:F9:72:B4:92:99:F2:11:7E:4B:DF:A4:A8:B9:DE' ']' + echo 'OK: certificate was re-signed after horizon update' OK: certificate was re-signed after horizon update + desc 'verify TLS secrets are still signed by the SAME CA' + set +o xtrace ----------------------------------------------------------------------------------- verify TLS secrets are still signed by the SAME CA ----------------------------------------------------------------------------------- + verify_cert_signed_by_ca some-name-ssl some-name-ca-cert + local tls_secret_name=some-name-ssl + local ca_secret_name=some-name-ca-cert + local ca_crt ++ kubectl_bin get secret some-name-ca-cert -o 'jsonpath={.data.ca\.crt}' ++ base64 -d +++ mktemp ++ local LAST_OUT=/tmp/tmp.nZMKGUn3He +++ mktemp ++ local LAST_ERR=/tmp/tmp.AaEXZrJn1o ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get secret some-name-ca-cert -o 'jsonpath={.data.ca\.crt}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.nZMKGUn3He ++ cat /tmp/tmp.AaEXZrJn1o ++ rm /tmp/tmp.nZMKGUn3He /tmp/tmp.AaEXZrJn1o ++ return 0 + ca_crt=$'-----BEGIN CERTIFICATE-----\nMIIDHDCCAgSgAwIBAgIRAOKCbOotUMOQvcitnmwCqI8wDQYJKoZIhvcNAQELBQAw\nEjEQMA4GA1UEChMHUm9vdCBDQTAgFw0yNjA2MDkxMDQ2MDNaGA85OTk5MTIzMTIz\nNTk1OVowEjEQMA4GA1UEChMHUm9vdCBDQTCCASIwDQYJKoZIhvcNAQEBBQADggEP\nADCCAQoCggEBAKWfk5pyPbV1b6Ctsa64tAtZGxqxiAzOG9NSDB90nX+mSuYDZcZr\n4XsFD3uaCXP2zbdlOm28D67lt9n9ZP1y2gwzY2vaehHFMMB/ybJAfZ2ba9IDr8M9\njxqDapgOc82VMCrFTUPrfHI/ZeRKpcOYn8aLWeQJdsDCjfS82wf/B1cIdQ2haIvk\nTykjbKxA1CPe+ORR6655DxJlHSW+Kgo8XyMUDTxHY9B1JoogWfaNwYAG0C3mi3Rn\nrjC08gY+KGlEuCGEpI6c72klw7/FnWs/ZMQrW4DbKTkeSV+LzPGGrdpcANeQbnr4\nSQEShtVhXL5oRO2td3AZQ0vP7o2IOHnLrrsCAwEAAaNrMGkwDgYDVR0PAQH/BAQD\nAgIkMCcGA1UdJQQgMB4GCCsGAQUFBwMBBggrBgEFBQcDAgYIKwYBBQUHAwMwDwYD\nVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUHNwdIX9ky63IirIrX9VQ/AI2GIUwDQYJ\nKoZIhvcNAQELBQADggEBAJ+Tvunj7OmoyiiJfXojbqsZrc924jhJAnGJR3GzVcas\n1IQnS4bog54gfVqNf84InSM9WL38SBPJkRNE1PPlcMdy0Z+1Su/KP/LhKyVYN1ds\nBCnnMAelDQD+5h9XeH7QMkuOudgbnMXDQuoOiciv59dmVA9BoC42I/ljKCvB1EON\nEwIXjVMv05Lx4OuyThTyn8zded8hTiP9yRC0RKYB5LCdBie6g4LmpFQU4NWJRLhy\nvLtm7Mrq699SOOcIJAX3mFDjwol9EQuH/x95vPfeyQfYgAeaP/sS08W6nGcYSTGb\nCeTSH5KFnsOV3Z5+4xNs8hMyqJDTgMGk3Dw3aGoUTcU=\n-----END CERTIFICATE-----' + local tls_crt ++ base64 -d ++ kubectl_bin get secret some-name-ssl -o 'jsonpath={.data.tls\.crt}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.FBcNEhL3WQ +++ mktemp ++ local LAST_ERR=/tmp/tmp.LrriVJWoUO ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get secret some-name-ssl -o 'jsonpath={.data.tls\.crt}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.FBcNEhL3WQ ++ cat /tmp/tmp.LrriVJWoUO ++ rm /tmp/tmp.FBcNEhL3WQ /tmp/tmp.LrriVJWoUO ++ return 0 + tls_crt=$'-----BEGIN CERTIFICATE-----\nMIIIsDCCB5igAwIBAgIQZFArD7dV3RMBxWEFjj1rXTANBgkqhkiG9w0BAQsFADAS\nMRAwDgYDVQQKEwdSb290IENBMCAXDTI2MDYwOTEwNDkxOVoYDzk5OTkxMjMxMjM1\nOTU5WjAQMQ4wDAYDVQQKEwVQU01EQjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC\nAQoCggEBANL0ZVwqzBWlm6MUo0MxW/teEiI9c/Y9KWD/LaEHKra8fJ8kVtIAXoCA\nK/wsNpjg9uqUtvyuYqzj8fkUfmhkNbcid7D8Rw+jfc002z2h1/Fl6jL2zxbQAOcB\njd/XNTijp3aGcWmJMeIBh36EKhGz2E6CtVlXbsGZMmZKv+bdu+roK64Dpw/ifFYX\n0lpjx6eSjv9USKCmWoJ7we3ldbDgfQ51Q/pTDXAKZsX45ZRb9i42vyBXrhm9bwLU\nsGX0kAgDF/ZF5uOkd+3eAGPmqPCnutTnc24END+okCB1WYLTyxWqCNlu7VHBCTx3\n8yS5RQum+9XtSLGDmtnpI5pAYbjjOiUCAwEAAaOCBgAwggX8MA4GA1UdDwEB/wQE\nAwIFoDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwDAYDVR0TAQH/BAIw\nADAfBgNVHSMEGDAWgBQc3B0hf2TLrciKsitf1VD8AjYYhTCCBZoGA1UdEQSCBZEw\nggWNgglsb2NhbGhvc3SCDXNvbWUtbmFtZS1yczCCLHNvbWUtbmFtZS1yczAuc3Bs\naXQtaG9yaXpvbi1tYW51YWwtdGxzLTI0MzM0gj5zb21lLW5hbWUtcnMwLnNwbGl0\nLWhvcml6b24tbWFudWFsLXRscy0yNDMzNC5zdmMuY2x1c3Rlci5sb2NhbIIPKi5z\nb21lLW5hbWUtcnMwgi4qLnNvbWUtbmFtZS1yczAuc3BsaXQtaG9yaXpvbi1tYW51\nYWwtdGxzLTI0MzM0gkAqLnNvbWUtbmFtZS1yczAuc3BsaXQtaG9yaXpvbi1tYW51\nYWwtdGxzLTI0MzM0LnN2Yy5jbHVzdGVyLmxvY2FsgkFzb21lLW5hbWUtcnMwLnNw\nbGl0LWhvcml6b24tbWFudWFsLXRscy0yNDMzNC5zdmMuY2x1c3RlcnNldC5sb2Nh\nbIJDKi5zb21lLW5hbWUtcnMwLnNwbGl0LWhvcml6b24tbWFudWFsLXRscy0yNDMz\nNC5zdmMuY2x1c3RlcnNldC5sb2NhbIIdc29tZS1uYW1lLXJzMC0wLmNsb3VkZGVt\nby54eXqCHXNvbWUtbmFtZS1yczAtMS5jbG91ZGRlbW8ueHl6gh1zb21lLW5hbWUt\ncnMwLTIuY2xvdWRkZW1vLnh5eoIdc29tZS1uYW1lLXJzMC0zLmNsb3VkZGVtby54\neXqCHXNvbWUtbmFtZS1yczAtNC5jbG91ZGRlbW8ueHl6gjUqLnNwbGl0LWhvcml6\nb24tbWFudWFsLXRscy0yNDMzNC5zdmMuY2x1c3RlcnNldC5sb2NhbIIQc29tZS1u\nYW1lLW1vbmdvc4Ivc29tZS1uYW1lLW1vbmdvcy5zcGxpdC1ob3Jpem9uLW1hbnVh\nbC10bHMtMjQzMzSCQXNvbWUtbmFtZS1tb25nb3Muc3BsaXQtaG9yaXpvbi1tYW51\nYWwtdGxzLTI0MzM0LnN2Yy5jbHVzdGVyLmxvY2FsghIqLnNvbWUtbmFtZS1tb25n\nb3OCMSouc29tZS1uYW1lLW1vbmdvcy5zcGxpdC1ob3Jpem9uLW1hbnVhbC10bHMt\nMjQzMzSCQyouc29tZS1uYW1lLW1vbmdvcy5zcGxpdC1ob3Jpem9uLW1hbnVhbC10\nbHMtMjQzMzQuc3ZjLmNsdXN0ZXIubG9jYWyCDXNvbWUtbmFtZS1jZmeCLHNvbWUt\nbmFtZS1jZmcuc3BsaXQtaG9yaXpvbi1tYW51YWwtdGxzLTI0MzM0gj5zb21lLW5h\nbWUtY2ZnLnNwbGl0LWhvcml6b24tbWFudWFsLXRscy0yNDMzNC5zdmMuY2x1c3Rl\nci5sb2NhbIIPKi5zb21lLW5hbWUtY2Zngi4qLnNvbWUtbmFtZS1jZmcuc3BsaXQt\naG9yaXpvbi1tYW51YWwtdGxzLTI0MzM0gkAqLnNvbWUtbmFtZS1jZmcuc3BsaXQt\naG9yaXpvbi1tYW51YWwtdGxzLTI0MzM0LnN2Yy5jbHVzdGVyLmxvY2FsgkRzb21l\nLW5hbWUtbW9uZ29zLnNwbGl0LWhvcml6b24tbWFudWFsLXRscy0yNDMzNC5zdmMu\nY2x1c3RlcnNldC5sb2NhbIJGKi5zb21lLW5hbWUtbW9uZ29zLnNwbGl0LWhvcml6\nb24tbWFudWFsLXRscy0yNDMzNC5zdmMuY2x1c3RlcnNldC5sb2NhbIJBc29tZS1u\nYW1lLWNmZy5zcGxpdC1ob3Jpem9uLW1hbnVhbC10bHMtMjQzMzQuc3ZjLmNsdXN0\nZXJzZXQubG9jYWyCQyouc29tZS1uYW1lLWNmZy5zcGxpdC1ob3Jpem9uLW1hbnVh\nbC10bHMtMjQzMzQuc3ZjLmNsdXN0ZXJzZXQubG9jYWwwDQYJKoZIhvcNAQELBQAD\nggEBAFQs0NM+SFf3rTOx23NQUnZIOBmtsC1R94Es8KwQDkwNQpbn6HP020A93SMD\nH6Q2SgtaqR7MMEWMUScHhAXrEg21NZMuJX1MB8E9evohs3xajGsTMDNVLpznfTAC\nP0Uj1g3tymp+u294f1bbFU0eMXNR/10t8x5BpS6xk8N4mFh86hdr3mD1k46pQ6EW\nG7Bh42K1eHJv+bLVrnkVYDDoFDxJWFOpn7yR0NQZBHRYQFpchrCujrr3VROESYdY\nzWyyeoo0/sEl8QhQdEfNuPtdSeC5fQfpFn2F4KFEkqbfSPpGewBJM/JWeeawa/AV\nqyZTTFhZ0pWCQinCxW9IZUJpd4E=\n-----END CERTIFICATE-----' + echo $'-----BEGIN CERTIFICATE-----\nMIIDHDCCAgSgAwIBAgIRAOKCbOotUMOQvcitnmwCqI8wDQYJKoZIhvcNAQELBQAw\nEjEQMA4GA1UEChMHUm9vdCBDQTAgFw0yNjA2MDkxMDQ2MDNaGA85OTk5MTIzMTIz\nNTk1OVowEjEQMA4GA1UEChMHUm9vdCBDQTCCASIwDQYJKoZIhvcNAQEBBQADggEP\nADCCAQoCggEBAKWfk5pyPbV1b6Ctsa64tAtZGxqxiAzOG9NSDB90nX+mSuYDZcZr\n4XsFD3uaCXP2zbdlOm28D67lt9n9ZP1y2gwzY2vaehHFMMB/ybJAfZ2ba9IDr8M9\njxqDapgOc82VMCrFTUPrfHI/ZeRKpcOYn8aLWeQJdsDCjfS82wf/B1cIdQ2haIvk\nTykjbKxA1CPe+ORR6655DxJlHSW+Kgo8XyMUDTxHY9B1JoogWfaNwYAG0C3mi3Rn\nrjC08gY+KGlEuCGEpI6c72klw7/FnWs/ZMQrW4DbKTkeSV+LzPGGrdpcANeQbnr4\nSQEShtVhXL5oRO2td3AZQ0vP7o2IOHnLrrsCAwEAAaNrMGkwDgYDVR0PAQH/BAQD\nAgIkMCcGA1UdJQQgMB4GCCsGAQUFBwMBBggrBgEFBQcDAgYIKwYBBQUHAwMwDwYD\nVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUHNwdIX9ky63IirIrX9VQ/AI2GIUwDQYJ\nKoZIhvcNAQELBQADggEBAJ+Tvunj7OmoyiiJfXojbqsZrc924jhJAnGJR3GzVcas\n1IQnS4bog54gfVqNf84InSM9WL38SBPJkRNE1PPlcMdy0Z+1Su/KP/LhKyVYN1ds\nBCnnMAelDQD+5h9XeH7QMkuOudgbnMXDQuoOiciv59dmVA9BoC42I/ljKCvB1EON\nEwIXjVMv05Lx4OuyThTyn8zded8hTiP9yRC0RKYB5LCdBie6g4LmpFQU4NWJRLhy\nvLtm7Mrq699SOOcIJAX3mFDjwol9EQuH/x95vPfeyQfYgAeaP/sS08W6nGcYSTGb\nCeTSH5KFnsOV3Z5+4xNs8hMyqJDTgMGk3Dw3aGoUTcU=\n-----END CERTIFICATE-----' + echo $'-----BEGIN CERTIFICATE-----\nMIIIsDCCB5igAwIBAgIQZFArD7dV3RMBxWEFjj1rXTANBgkqhkiG9w0BAQsFADAS\nMRAwDgYDVQQKEwdSb290IENBMCAXDTI2MDYwOTEwNDkxOVoYDzk5OTkxMjMxMjM1\nOTU5WjAQMQ4wDAYDVQQKEwVQU01EQjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC\nAQoCggEBANL0ZVwqzBWlm6MUo0MxW/teEiI9c/Y9KWD/LaEHKra8fJ8kVtIAXoCA\nK/wsNpjg9uqUtvyuYqzj8fkUfmhkNbcid7D8Rw+jfc002z2h1/Fl6jL2zxbQAOcB\njd/XNTijp3aGcWmJMeIBh36EKhGz2E6CtVlXbsGZMmZKv+bdu+roK64Dpw/ifFYX\n0lpjx6eSjv9USKCmWoJ7we3ldbDgfQ51Q/pTDXAKZsX45ZRb9i42vyBXrhm9bwLU\nsGX0kAgDF/ZF5uOkd+3eAGPmqPCnutTnc24END+okCB1WYLTyxWqCNlu7VHBCTx3\n8yS5RQum+9XtSLGDmtnpI5pAYbjjOiUCAwEAAaOCBgAwggX8MA4GA1UdDwEB/wQE\nAwIFoDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwDAYDVR0TAQH/BAIw\nADAfBgNVHSMEGDAWgBQc3B0hf2TLrciKsitf1VD8AjYYhTCCBZoGA1UdEQSCBZEw\nggWNgglsb2NhbGhvc3SCDXNvbWUtbmFtZS1yczCCLHNvbWUtbmFtZS1yczAuc3Bs\naXQtaG9yaXpvbi1tYW51YWwtdGxzLTI0MzM0gj5zb21lLW5hbWUtcnMwLnNwbGl0\nLWhvcml6b24tbWFudWFsLXRscy0yNDMzNC5zdmMuY2x1c3Rlci5sb2NhbIIPKi5z\nb21lLW5hbWUtcnMwgi4qLnNvbWUtbmFtZS1yczAuc3BsaXQtaG9yaXpvbi1tYW51\nYWwtdGxzLTI0MzM0gkAqLnNvbWUtbmFtZS1yczAuc3BsaXQtaG9yaXpvbi1tYW51\nYWwtdGxzLTI0MzM0LnN2Yy5jbHVzdGVyLmxvY2FsgkFzb21lLW5hbWUtcnMwLnNw\nbGl0LWhvcml6b24tbWFudWFsLXRscy0yNDMzNC5zdmMuY2x1c3RlcnNldC5sb2Nh\nbIJDKi5zb21lLW5hbWUtcnMwLnNwbGl0LWhvcml6b24tbWFudWFsLXRscy0yNDMz\nNC5zdmMuY2x1c3RlcnNldC5sb2NhbIIdc29tZS1uYW1lLXJzMC0wLmNsb3VkZGVt\nby54eXqCHXNvbWUtbmFtZS1yczAtMS5jbG91ZGRlbW8ueHl6gh1zb21lLW5hbWUt\ncnMwLTIuY2xvdWRkZW1vLnh5eoIdc29tZS1uYW1lLXJzMC0zLmNsb3VkZGVtby54\neXqCHXNvbWUtbmFtZS1yczAtNC5jbG91ZGRlbW8ueHl6gjUqLnNwbGl0LWhvcml6\nb24tbWFudWFsLXRscy0yNDMzNC5zdmMuY2x1c3RlcnNldC5sb2NhbIIQc29tZS1u\nYW1lLW1vbmdvc4Ivc29tZS1uYW1lLW1vbmdvcy5zcGxpdC1ob3Jpem9uLW1hbnVh\nbC10bHMtMjQzMzSCQXNvbWUtbmFtZS1tb25nb3Muc3BsaXQtaG9yaXpvbi1tYW51\nYWwtdGxzLTI0MzM0LnN2Yy5jbHVzdGVyLmxvY2FsghIqLnNvbWUtbmFtZS1tb25n\nb3OCMSouc29tZS1uYW1lLW1vbmdvcy5zcGxpdC1ob3Jpem9uLW1hbnVhbC10bHMt\nMjQzMzSCQyouc29tZS1uYW1lLW1vbmdvcy5zcGxpdC1ob3Jpem9uLW1hbnVhbC10\nbHMtMjQzMzQuc3ZjLmNsdXN0ZXIubG9jYWyCDXNvbWUtbmFtZS1jZmeCLHNvbWUt\nbmFtZS1jZmcuc3BsaXQtaG9yaXpvbi1tYW51YWwtdGxzLTI0MzM0gj5zb21lLW5h\nbWUtY2ZnLnNwbGl0LWhvcml6b24tbWFudWFsLXRscy0yNDMzNC5zdmMuY2x1c3Rl\nci5sb2NhbIIPKi5zb21lLW5hbWUtY2Zngi4qLnNvbWUtbmFtZS1jZmcuc3BsaXQt\naG9yaXpvbi1tYW51YWwtdGxzLTI0MzM0gkAqLnNvbWUtbmFtZS1jZmcuc3BsaXQt\naG9yaXpvbi1tYW51YWwtdGxzLTI0MzM0LnN2Yy5jbHVzdGVyLmxvY2FsgkRzb21l\nLW5hbWUtbW9uZ29zLnNwbGl0LWhvcml6b24tbWFudWFsLXRscy0yNDMzNC5zdmMu\nY2x1c3RlcnNldC5sb2NhbIJGKi5zb21lLW5hbWUtbW9uZ29zLnNwbGl0LWhvcml6\nb24tbWFudWFsLXRscy0yNDMzNC5zdmMuY2x1c3RlcnNldC5sb2NhbIJBc29tZS1u\nYW1lLWNmZy5zcGxpdC1ob3Jpem9uLW1hbnVhbC10bHMtMjQzMzQuc3ZjLmNsdXN0\nZXJzZXQubG9jYWyCQyouc29tZS1uYW1lLWNmZy5zcGxpdC1ob3Jpem9uLW1hbnVh\nbC10bHMtMjQzMzQuc3ZjLmNsdXN0ZXJzZXQubG9jYWwwDQYJKoZIhvcNAQELBQAD\nggEBAFQs0NM+SFf3rTOx23NQUnZIOBmtsC1R94Es8KwQDkwNQpbn6HP020A93SMD\nH6Q2SgtaqR7MMEWMUScHhAXrEg21NZMuJX1MB8E9evohs3xajGsTMDNVLpznfTAC\nP0Uj1g3tymp+u294f1bbFU0eMXNR/10t8x5BpS6xk8N4mFh86hdr3mD1k46pQ6EW\nG7Bh42K1eHJv+bLVrnkVYDDoFDxJWFOpn7yR0NQZBHRYQFpchrCujrr3VROESYdY\nzWyyeoo0/sEl8QhQdEfNuPtdSeC5fQfpFn2F4KFEkqbfSPpGewBJM/JWeeawa/AV\nqyZTTFhZ0pWCQinCxW9IZUJpd4E=\n-----END CERTIFICATE-----' + openssl verify -CAfile /tmp/tmp.i1hhairPLp/ca.crt /tmp/tmp.i1hhairPLp/tls.crt + echo 'OK: '\''some-name-ssl'\'' is signed by CA in '\''some-name-ca-cert'\''' OK: 'some-name-ssl' is signed by CA in 'some-name-ca-cert' + verify_cert_signed_by_ca some-name-ssl-internal some-name-ca-cert + local tls_secret_name=some-name-ssl-internal + local ca_secret_name=some-name-ca-cert + local ca_crt ++ kubectl_bin get secret some-name-ca-cert -o 'jsonpath={.data.ca\.crt}' ++ base64 -d +++ mktemp ++ local LAST_OUT=/tmp/tmp.BLJL4UJDKI +++ mktemp ++ local LAST_ERR=/tmp/tmp.rdodugzrDk ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get secret some-name-ca-cert -o 'jsonpath={.data.ca\.crt}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.BLJL4UJDKI ++ cat /tmp/tmp.rdodugzrDk ++ rm /tmp/tmp.BLJL4UJDKI /tmp/tmp.rdodugzrDk ++ return 0 + ca_crt=$'-----BEGIN CERTIFICATE-----\nMIIDHDCCAgSgAwIBAgIRAOKCbOotUMOQvcitnmwCqI8wDQYJKoZIhvcNAQELBQAw\nEjEQMA4GA1UEChMHUm9vdCBDQTAgFw0yNjA2MDkxMDQ2MDNaGA85OTk5MTIzMTIz\nNTk1OVowEjEQMA4GA1UEChMHUm9vdCBDQTCCASIwDQYJKoZIhvcNAQEBBQADggEP\nADCCAQoCggEBAKWfk5pyPbV1b6Ctsa64tAtZGxqxiAzOG9NSDB90nX+mSuYDZcZr\n4XsFD3uaCXP2zbdlOm28D67lt9n9ZP1y2gwzY2vaehHFMMB/ybJAfZ2ba9IDr8M9\njxqDapgOc82VMCrFTUPrfHI/ZeRKpcOYn8aLWeQJdsDCjfS82wf/B1cIdQ2haIvk\nTykjbKxA1CPe+ORR6655DxJlHSW+Kgo8XyMUDTxHY9B1JoogWfaNwYAG0C3mi3Rn\nrjC08gY+KGlEuCGEpI6c72klw7/FnWs/ZMQrW4DbKTkeSV+LzPGGrdpcANeQbnr4\nSQEShtVhXL5oRO2td3AZQ0vP7o2IOHnLrrsCAwEAAaNrMGkwDgYDVR0PAQH/BAQD\nAgIkMCcGA1UdJQQgMB4GCCsGAQUFBwMBBggrBgEFBQcDAgYIKwYBBQUHAwMwDwYD\nVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUHNwdIX9ky63IirIrX9VQ/AI2GIUwDQYJ\nKoZIhvcNAQELBQADggEBAJ+Tvunj7OmoyiiJfXojbqsZrc924jhJAnGJR3GzVcas\n1IQnS4bog54gfVqNf84InSM9WL38SBPJkRNE1PPlcMdy0Z+1Su/KP/LhKyVYN1ds\nBCnnMAelDQD+5h9XeH7QMkuOudgbnMXDQuoOiciv59dmVA9BoC42I/ljKCvB1EON\nEwIXjVMv05Lx4OuyThTyn8zded8hTiP9yRC0RKYB5LCdBie6g4LmpFQU4NWJRLhy\nvLtm7Mrq699SOOcIJAX3mFDjwol9EQuH/x95vPfeyQfYgAeaP/sS08W6nGcYSTGb\nCeTSH5KFnsOV3Z5+4xNs8hMyqJDTgMGk3Dw3aGoUTcU=\n-----END CERTIFICATE-----' + local tls_crt ++ kubectl_bin get secret some-name-ssl-internal -o 'jsonpath={.data.tls\.crt}' ++ base64 -d +++ mktemp ++ local LAST_OUT=/tmp/tmp.MvXa9Vhmkv +++ mktemp ++ local LAST_ERR=/tmp/tmp.a6SmOJr2In ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get secret some-name-ssl-internal -o 'jsonpath={.data.tls\.crt}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.MvXa9Vhmkv ++ cat /tmp/tmp.a6SmOJr2In ++ rm /tmp/tmp.MvXa9Vhmkv /tmp/tmp.a6SmOJr2In ++ return 0 + tls_crt=$'-----BEGIN CERTIFICATE-----\nMIIIsTCCB5mgAwIBAgIRAPqIXVzRwUKjp/8xq4Fl4NMwDQYJKoZIhvcNAQELBQAw\nEjEQMA4GA1UEChMHUm9vdCBDQTAgFw0yNjA2MDkxMDQ5MTlaGA85OTk5MTIzMTIz\nNTk1OVowEDEOMAwGA1UEChMFUFNNREIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw\nggEKAoIBAQCyrcGL1BcAawQBRwk2rmrlSwl31pWQlZqx5Aqvl1ar/oELoEulx2mk\nxz1oJsA7Q2ul04Jygdm+841Yh0JORE5YMqaokWfLsuwL9wq9p1VNUMK0rmLXJyjn\nMEWaZXU/pB/in+fZKl83EUJN43HmI1upSS19Tgb+PhP42m8wH+FQxzwdSd6ITDZf\n2CGzWo6VMyHo/ms/Vq4G+iOKcNUTcmHT8PoKjHoey3blaUuHUIxtt4AzxX74hhbI\nQAos1c4if4FiigBrB2qizSru4Vyi1zaLDym19bgw0gYLrIwSdgrTiAz1Olw/gTC7\nb/QOsbEgPL1fZtfpzrhTYuDEjRZzw0jxAgMBAAGjggYAMIIF/DAOBgNVHQ8BAf8E\nBAMCBaAwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMAwGA1UdEwEB/wQC\nMAAwHwYDVR0jBBgwFoAUHNwdIX9ky63IirIrX9VQ/AI2GIUwggWaBgNVHREEggWR\nMIIFjYIJbG9jYWxob3N0gg1zb21lLW5hbWUtcnMwgixzb21lLW5hbWUtcnMwLnNw\nbGl0LWhvcml6b24tbWFudWFsLXRscy0yNDMzNII+c29tZS1uYW1lLXJzMC5zcGxp\ndC1ob3Jpem9uLW1hbnVhbC10bHMtMjQzMzQuc3ZjLmNsdXN0ZXIubG9jYWyCDyou\nc29tZS1uYW1lLXJzMIIuKi5zb21lLW5hbWUtcnMwLnNwbGl0LWhvcml6b24tbWFu\ndWFsLXRscy0yNDMzNIJAKi5zb21lLW5hbWUtcnMwLnNwbGl0LWhvcml6b24tbWFu\ndWFsLXRscy0yNDMzNC5zdmMuY2x1c3Rlci5sb2NhbIJBc29tZS1uYW1lLXJzMC5z\ncGxpdC1ob3Jpem9uLW1hbnVhbC10bHMtMjQzMzQuc3ZjLmNsdXN0ZXJzZXQubG9j\nYWyCQyouc29tZS1uYW1lLXJzMC5zcGxpdC1ob3Jpem9uLW1hbnVhbC10bHMtMjQz\nMzQuc3ZjLmNsdXN0ZXJzZXQubG9jYWyCHXNvbWUtbmFtZS1yczAtMC5jbG91ZGRl\nbW8ueHl6gh1zb21lLW5hbWUtcnMwLTEuY2xvdWRkZW1vLnh5eoIdc29tZS1uYW1l\nLXJzMC0yLmNsb3VkZGVtby54eXqCHXNvbWUtbmFtZS1yczAtMy5jbG91ZGRlbW8u\neHl6gh1zb21lLW5hbWUtcnMwLTQuY2xvdWRkZW1vLnh5eoI1Ki5zcGxpdC1ob3Jp\nem9uLW1hbnVhbC10bHMtMjQzMzQuc3ZjLmNsdXN0ZXJzZXQubG9jYWyCEHNvbWUt\nbmFtZS1tb25nb3OCL3NvbWUtbmFtZS1tb25nb3Muc3BsaXQtaG9yaXpvbi1tYW51\nYWwtdGxzLTI0MzM0gkFzb21lLW5hbWUtbW9uZ29zLnNwbGl0LWhvcml6b24tbWFu\ndWFsLXRscy0yNDMzNC5zdmMuY2x1c3Rlci5sb2NhbIISKi5zb21lLW5hbWUtbW9u\nZ29zgjEqLnNvbWUtbmFtZS1tb25nb3Muc3BsaXQtaG9yaXpvbi1tYW51YWwtdGxz\nLTI0MzM0gkMqLnNvbWUtbmFtZS1tb25nb3Muc3BsaXQtaG9yaXpvbi1tYW51YWwt\ndGxzLTI0MzM0LnN2Yy5jbHVzdGVyLmxvY2Fsgg1zb21lLW5hbWUtY2Zngixzb21l\nLW5hbWUtY2ZnLnNwbGl0LWhvcml6b24tbWFudWFsLXRscy0yNDMzNII+c29tZS1u\nYW1lLWNmZy5zcGxpdC1ob3Jpem9uLW1hbnVhbC10bHMtMjQzMzQuc3ZjLmNsdXN0\nZXIubG9jYWyCDyouc29tZS1uYW1lLWNmZ4IuKi5zb21lLW5hbWUtY2ZnLnNwbGl0\nLWhvcml6b24tbWFudWFsLXRscy0yNDMzNIJAKi5zb21lLW5hbWUtY2ZnLnNwbGl0\nLWhvcml6b24tbWFudWFsLXRscy0yNDMzNC5zdmMuY2x1c3Rlci5sb2NhbIJEc29t\nZS1uYW1lLW1vbmdvcy5zcGxpdC1ob3Jpem9uLW1hbnVhbC10bHMtMjQzMzQuc3Zj\nLmNsdXN0ZXJzZXQubG9jYWyCRiouc29tZS1uYW1lLW1vbmdvcy5zcGxpdC1ob3Jp\nem9uLW1hbnVhbC10bHMtMjQzMzQuc3ZjLmNsdXN0ZXJzZXQubG9jYWyCQXNvbWUt\nbmFtZS1jZmcuc3BsaXQtaG9yaXpvbi1tYW51YWwtdGxzLTI0MzM0LnN2Yy5jbHVz\ndGVyc2V0LmxvY2FsgkMqLnNvbWUtbmFtZS1jZmcuc3BsaXQtaG9yaXpvbi1tYW51\nYWwtdGxzLTI0MzM0LnN2Yy5jbHVzdGVyc2V0LmxvY2FsMA0GCSqGSIb3DQEBCwUA\nA4IBAQBvqD8tbcuQhD37EYBL/nKBMLBqMcrC7hZOvk6+dcEL1mi2jIG7kmIQKNTb\nbjlbUG5+zgSQOX0LEa8JkbSKxmRPaoHwgQXqaadcbrxH0Shq0QT2dOtzJjfg4LBz\noCKLXS1K7Ga7w/gmsURt6jra3ve+Zw2puiBD6IbIjsjvwPMBVLovwhPr1bsRHDv6\nvwApIPbvKbgwarm2bCphOrqliYgHJo+2ri3dN42ONIILRnp7KTf5YpEZU7XnhpjC\nPHyedit01FClhT8rJkbmddABrp041o6jkxTf3AwngaYdjXVH4nI4lWLiU0hRll9v\n4HeRnJYnqoznJ+eJdLSIR+lv7gW9\n-----END CERTIFICATE-----' + echo $'-----BEGIN CERTIFICATE-----\nMIIDHDCCAgSgAwIBAgIRAOKCbOotUMOQvcitnmwCqI8wDQYJKoZIhvcNAQELBQAw\nEjEQMA4GA1UEChMHUm9vdCBDQTAgFw0yNjA2MDkxMDQ2MDNaGA85OTk5MTIzMTIz\nNTk1OVowEjEQMA4GA1UEChMHUm9vdCBDQTCCASIwDQYJKoZIhvcNAQEBBQADggEP\nADCCAQoCggEBAKWfk5pyPbV1b6Ctsa64tAtZGxqxiAzOG9NSDB90nX+mSuYDZcZr\n4XsFD3uaCXP2zbdlOm28D67lt9n9ZP1y2gwzY2vaehHFMMB/ybJAfZ2ba9IDr8M9\njxqDapgOc82VMCrFTUPrfHI/ZeRKpcOYn8aLWeQJdsDCjfS82wf/B1cIdQ2haIvk\nTykjbKxA1CPe+ORR6655DxJlHSW+Kgo8XyMUDTxHY9B1JoogWfaNwYAG0C3mi3Rn\nrjC08gY+KGlEuCGEpI6c72klw7/FnWs/ZMQrW4DbKTkeSV+LzPGGrdpcANeQbnr4\nSQEShtVhXL5oRO2td3AZQ0vP7o2IOHnLrrsCAwEAAaNrMGkwDgYDVR0PAQH/BAQD\nAgIkMCcGA1UdJQQgMB4GCCsGAQUFBwMBBggrBgEFBQcDAgYIKwYBBQUHAwMwDwYD\nVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUHNwdIX9ky63IirIrX9VQ/AI2GIUwDQYJ\nKoZIhvcNAQELBQADggEBAJ+Tvunj7OmoyiiJfXojbqsZrc924jhJAnGJR3GzVcas\n1IQnS4bog54gfVqNf84InSM9WL38SBPJkRNE1PPlcMdy0Z+1Su/KP/LhKyVYN1ds\nBCnnMAelDQD+5h9XeH7QMkuOudgbnMXDQuoOiciv59dmVA9BoC42I/ljKCvB1EON\nEwIXjVMv05Lx4OuyThTyn8zded8hTiP9yRC0RKYB5LCdBie6g4LmpFQU4NWJRLhy\nvLtm7Mrq699SOOcIJAX3mFDjwol9EQuH/x95vPfeyQfYgAeaP/sS08W6nGcYSTGb\nCeTSH5KFnsOV3Z5+4xNs8hMyqJDTgMGk3Dw3aGoUTcU=\n-----END CERTIFICATE-----' + echo $'-----BEGIN CERTIFICATE-----\nMIIIsTCCB5mgAwIBAgIRAPqIXVzRwUKjp/8xq4Fl4NMwDQYJKoZIhvcNAQELBQAw\nEjEQMA4GA1UEChMHUm9vdCBDQTAgFw0yNjA2MDkxMDQ5MTlaGA85OTk5MTIzMTIz\nNTk1OVowEDEOMAwGA1UEChMFUFNNREIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw\nggEKAoIBAQCyrcGL1BcAawQBRwk2rmrlSwl31pWQlZqx5Aqvl1ar/oELoEulx2mk\nxz1oJsA7Q2ul04Jygdm+841Yh0JORE5YMqaokWfLsuwL9wq9p1VNUMK0rmLXJyjn\nMEWaZXU/pB/in+fZKl83EUJN43HmI1upSS19Tgb+PhP42m8wH+FQxzwdSd6ITDZf\n2CGzWo6VMyHo/ms/Vq4G+iOKcNUTcmHT8PoKjHoey3blaUuHUIxtt4AzxX74hhbI\nQAos1c4if4FiigBrB2qizSru4Vyi1zaLDym19bgw0gYLrIwSdgrTiAz1Olw/gTC7\nb/QOsbEgPL1fZtfpzrhTYuDEjRZzw0jxAgMBAAGjggYAMIIF/DAOBgNVHQ8BAf8E\nBAMCBaAwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMAwGA1UdEwEB/wQC\nMAAwHwYDVR0jBBgwFoAUHNwdIX9ky63IirIrX9VQ/AI2GIUwggWaBgNVHREEggWR\nMIIFjYIJbG9jYWxob3N0gg1zb21lLW5hbWUtcnMwgixzb21lLW5hbWUtcnMwLnNw\nbGl0LWhvcml6b24tbWFudWFsLXRscy0yNDMzNII+c29tZS1uYW1lLXJzMC5zcGxp\ndC1ob3Jpem9uLW1hbnVhbC10bHMtMjQzMzQuc3ZjLmNsdXN0ZXIubG9jYWyCDyou\nc29tZS1uYW1lLXJzMIIuKi5zb21lLW5hbWUtcnMwLnNwbGl0LWhvcml6b24tbWFu\ndWFsLXRscy0yNDMzNIJAKi5zb21lLW5hbWUtcnMwLnNwbGl0LWhvcml6b24tbWFu\ndWFsLXRscy0yNDMzNC5zdmMuY2x1c3Rlci5sb2NhbIJBc29tZS1uYW1lLXJzMC5z\ncGxpdC1ob3Jpem9uLW1hbnVhbC10bHMtMjQzMzQuc3ZjLmNsdXN0ZXJzZXQubG9j\nYWyCQyouc29tZS1uYW1lLXJzMC5zcGxpdC1ob3Jpem9uLW1hbnVhbC10bHMtMjQz\nMzQuc3ZjLmNsdXN0ZXJzZXQubG9jYWyCHXNvbWUtbmFtZS1yczAtMC5jbG91ZGRl\nbW8ueHl6gh1zb21lLW5hbWUtcnMwLTEuY2xvdWRkZW1vLnh5eoIdc29tZS1uYW1l\nLXJzMC0yLmNsb3VkZGVtby54eXqCHXNvbWUtbmFtZS1yczAtMy5jbG91ZGRlbW8u\neHl6gh1zb21lLW5hbWUtcnMwLTQuY2xvdWRkZW1vLnh5eoI1Ki5zcGxpdC1ob3Jp\nem9uLW1hbnVhbC10bHMtMjQzMzQuc3ZjLmNsdXN0ZXJzZXQubG9jYWyCEHNvbWUt\nbmFtZS1tb25nb3OCL3NvbWUtbmFtZS1tb25nb3Muc3BsaXQtaG9yaXpvbi1tYW51\nYWwtdGxzLTI0MzM0gkFzb21lLW5hbWUtbW9uZ29zLnNwbGl0LWhvcml6b24tbWFu\ndWFsLXRscy0yNDMzNC5zdmMuY2x1c3Rlci5sb2NhbIISKi5zb21lLW5hbWUtbW9u\nZ29zgjEqLnNvbWUtbmFtZS1tb25nb3Muc3BsaXQtaG9yaXpvbi1tYW51YWwtdGxz\nLTI0MzM0gkMqLnNvbWUtbmFtZS1tb25nb3Muc3BsaXQtaG9yaXpvbi1tYW51YWwt\ndGxzLTI0MzM0LnN2Yy5jbHVzdGVyLmxvY2Fsgg1zb21lLW5hbWUtY2Zngixzb21l\nLW5hbWUtY2ZnLnNwbGl0LWhvcml6b24tbWFudWFsLXRscy0yNDMzNII+c29tZS1u\nYW1lLWNmZy5zcGxpdC1ob3Jpem9uLW1hbnVhbC10bHMtMjQzMzQuc3ZjLmNsdXN0\nZXIubG9jYWyCDyouc29tZS1uYW1lLWNmZ4IuKi5zb21lLW5hbWUtY2ZnLnNwbGl0\nLWhvcml6b24tbWFudWFsLXRscy0yNDMzNIJAKi5zb21lLW5hbWUtY2ZnLnNwbGl0\nLWhvcml6b24tbWFudWFsLXRscy0yNDMzNC5zdmMuY2x1c3Rlci5sb2NhbIJEc29t\nZS1uYW1lLW1vbmdvcy5zcGxpdC1ob3Jpem9uLW1hbnVhbC10bHMtMjQzMzQuc3Zj\nLmNsdXN0ZXJzZXQubG9jYWyCRiouc29tZS1uYW1lLW1vbmdvcy5zcGxpdC1ob3Jp\nem9uLW1hbnVhbC10bHMtMjQzMzQuc3ZjLmNsdXN0ZXJzZXQubG9jYWyCQXNvbWUt\nbmFtZS1jZmcuc3BsaXQtaG9yaXpvbi1tYW51YWwtdGxzLTI0MzM0LnN2Yy5jbHVz\ndGVyc2V0LmxvY2FsgkMqLnNvbWUtbmFtZS1jZmcuc3BsaXQtaG9yaXpvbi1tYW51\nYWwtdGxzLTI0MzM0LnN2Yy5jbHVzdGVyc2V0LmxvY2FsMA0GCSqGSIb3DQEBCwUA\nA4IBAQBvqD8tbcuQhD37EYBL/nKBMLBqMcrC7hZOvk6+dcEL1mi2jIG7kmIQKNTb\nbjlbUG5+zgSQOX0LEa8JkbSKxmRPaoHwgQXqaadcbrxH0Shq0QT2dOtzJjfg4LBz\noCKLXS1K7Ga7w/gmsURt6jra3ve+Zw2puiBD6IbIjsjvwPMBVLovwhPr1bsRHDv6\nvwApIPbvKbgwarm2bCphOrqliYgHJo+2ri3dN42ONIILRnp7KTf5YpEZU7XnhpjC\nPHyedit01FClhT8rJkbmddABrp041o6jkxTf3AwngaYdjXVH4nI4lWLiU0hRll9v\n4HeRnJYnqoznJ+eJdLSIR+lv7gW9\n-----END CERTIFICATE-----' + openssl verify -CAfile /tmp/tmp.i1hhairPLp/ca.crt /tmp/tmp.i1hhairPLp/tls.crt + echo 'OK: '\''some-name-ssl-internal'\'' is signed by CA in '\''some-name-ca-cert'\''' OK: 'some-name-ssl-internal' is signed by CA in 'some-name-ca-cert' + desc 'scale up to 5 members' + set +o xtrace ----------------------------------------------------------------------------------- scale up to 5 members ----------------------------------------------------------------------------------- + kubectl_bin patch psmdb some-name --type=json '-p=[{"op": "replace", "path": "/spec/replsets/0/size", "value": 5}]' ++ mktemp + local LAST_OUT=/tmp/tmp.1RlveNmqFV ++ mktemp + local LAST_ERR=/tmp/tmp.HMgGtw7lbA + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl patch psmdb some-name --type=json '-p=[{"op": "replace", "path": "/spec/replsets/0/size", "value": 5}]' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.1RlveNmqFV perconaservermongodb.psmdb.percona.com/some-name patched + cat /tmp/tmp.HMgGtw7lbA + rm /tmp/tmp.1RlveNmqFV /tmp/tmp.HMgGtw7lbA + return 0 + wait_for_running some-name-rs0 5 + local name=some-name-rs0 + let last_pod=4 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=some-name ++ seq 0 4 + for i in $(seq 0 $last_pod) + [[ 0 -eq 4 ]] + wait_pod some-name-rs0-0 + local pod=some-name-rs0-0 + set +o xtrace waiting for pod/some-name-rs0-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 4 ]] + wait_pod some-name-rs0-1 + local pod=some-name-rs0-1 + set +o xtrace waiting for pod/some-name-rs0-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 4 ]] + wait_pod some-name-rs0-2 + local pod=some-name-rs0-2 + set +o xtrace waiting for pod/some-name-rs0-2 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 3 -eq 4 ]] + wait_pod some-name-rs0-3 + local pod=some-name-rs0-3 + set +o xtrace waiting for pod/some-name-rs0-3 to be ready.................OK + for i in $(seq 0 $last_pod) + [[ 4 -eq 4 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.swy0d2ahQ1 +++ mktemp ++ local LAST_ERR=/tmp/tmp.BEjGXri4hm ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.swy0d2ahQ1 ++ cat /tmp/tmp.BEjGXri4hm ++ rm /tmp/tmp.swy0d2ahQ1 /tmp/tmp.BEjGXri4hm ++ return 0 + [[ '' == true ]] + wait_pod some-name-rs0-4 + local pod=some-name-rs0-4 + set +o xtrace waiting for pod/some-name-rs0-4 to be ready.............OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.REdGZ5HsmU +++ mktemp ++ local LAST_ERR=/tmp/tmp.xLs2BdW1tk ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.REdGZ5HsmU ++ cat /tmp/tmp.xLs2BdW1tk ++ rm /tmp/tmp.REdGZ5HsmU /tmp/tmp.xLs2BdW1tk ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.larfjjoK62 +++ mktemp ++ local LAST_ERR=/tmp/tmp.RIGIdFn2ZE ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.larfjjoK62 ++ cat /tmp/tmp.RIGIdFn2ZE ++ rm /tmp/tmp.larfjjoK62 /tmp/tmp.RIGIdFn2ZE ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster readyness + wait_cluster_consistency some-name + local cluster_name=some-name + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster readyness' waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.us37Z7MIeO +++ mktemp ++ local LAST_ERR=/tmp/tmp.8cl6d1NHiG ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.us37Z7MIeO ++ cat /tmp/tmp.8cl6d1NHiG ++ rm /tmp/tmp.us37Z7MIeO /tmp/tmp.8cl6d1NHiG ++ return 0 + [[ ready == ready ]] + echo .OK .OK + desc 'verify horizons after scale up' + set +o xtrace ----------------------------------------------------------------------------------- verify horizons after scale up ----------------------------------------------------------------------------------- + run_mongo_tls 'rs.conf().members.map(function(member) { return member.horizons }).sort((a, b) => a.external.localeCompare(b.external))' clusterAdmin:clusterAdmin123456@some-name-rs0-0.clouddemo.xyz,some-name-rs0-1.clouddemo.xyz,some-name-rs0-2.clouddemo.xyz mongodb '' --quiet + local 'command=rs.conf().members.map(function(member) { return member.horizons }).sort((a, b) => a.external.localeCompare(b.external))' + grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|does not match the remote host name' + local uri=clusterAdmin:clusterAdmin123456@some-name-rs0-0.clouddemo.xyz,some-name-rs0-1.clouddemo.xyz,some-name-rs0-2.clouddemo.xyz + local driver=mongodb + local suffix=.svc.cluster.local ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.dAaqArG6sL +++ mktemp ++ local LAST_ERR=/tmp/tmp.DfcnLC5rPv ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.dAaqArG6sL ++ cat /tmp/tmp.DfcnLC5rPv ++ rm /tmp/tmp.dAaqArG6sL /tmp/tmp.DfcnLC5rPv ++ return 0 + local client_container=psmdb-client-6bd9478dff-xj848 + local mongo_flag=--quiet + local port=27017 ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 + [[ clusterAdmin:clusterAdmin123456@some-name-rs0-0.clouddemo.xyz,some-name-rs0-1.clouddemo.xyz,some-name-rs0-2.clouddemo.xyz == *cfg* ]] + replica_set=rs0 + kubectl_bin exec psmdb-client-6bd9478dff-xj848 -- bash -c 'printf '\''rs.conf().members.map(function(member) { return member.horizons }).sort((a, b) => a.external.localeCompare(b.external))\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@some-name-rs0-0.clouddemo.xyz,some-name-rs0-1.clouddemo.xyz,some-name-rs0-2.clouddemo.xyz.svc.cluster.local:27017/admin?replicaSet=rs0 --tls --tlsCAFile /etc/mongodb-ssl/ca.crt --tlsCertificateKeyFile /tmp/tls.pem --tlsAllowInvalidHostnames --quiet' ++ mktemp + local LAST_OUT=/tmp/tmp.XjhyiYaOP7 ++ mktemp + local LAST_ERR=/tmp/tmp.Ys0Wn89863 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-6bd9478dff-xj848 -- bash -c 'printf '\''rs.conf().members.map(function(member) { return member.horizons }).sort((a, b) => a.external.localeCompare(b.external))\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@some-name-rs0-0.clouddemo.xyz,some-name-rs0-1.clouddemo.xyz,some-name-rs0-2.clouddemo.xyz.svc.cluster.local:27017/admin?replicaSet=rs0 --tls --tlsCAFile /etc/mongodb-ssl/ca.crt --tlsCertificateKeyFile /tmp/tls.pem --tlsAllowInvalidHostnames --quiet' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.XjhyiYaOP7 + cat /tmp/tmp.Ys0Wn89863 + rm /tmp/tmp.XjhyiYaOP7 /tmp/tmp.Ys0Wn89863 + return 0 + diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/split-horizon-manual-tls/compare/horizons-5.json /tmp/tmp.i1hhairPLp/horizons-5.json + desc 'scale down to 3 members' + set +o xtrace ----------------------------------------------------------------------------------- scale down to 3 members ----------------------------------------------------------------------------------- + kubectl_bin patch psmdb some-name --type=json '-p=[{"op": "replace", "path": "/spec/replsets/0/size", "value": 3}]' ++ mktemp + local LAST_OUT=/tmp/tmp.lVposAzqfZ ++ mktemp + local LAST_ERR=/tmp/tmp.wqjcb7WpWk + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl patch psmdb some-name --type=json '-p=[{"op": "replace", "path": "/spec/replsets/0/size", "value": 3}]' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.lVposAzqfZ perconaservermongodb.psmdb.percona.com/some-name patched + cat /tmp/tmp.wqjcb7WpWk + rm /tmp/tmp.lVposAzqfZ /tmp/tmp.wqjcb7WpWk + return 0 + wait_for_running some-name-rs0 3 + local name=some-name-rs0 + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-rs0-0 + local pod=some-name-rs0-0 + set +o xtrace waiting for pod/some-name-rs0-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-rs0-1 + local pod=some-name-rs0-1 + set +o xtrace waiting for pod/some-name-rs0-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.raT7uzNRwY +++ mktemp ++ local LAST_ERR=/tmp/tmp.VGYB739sFc ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.raT7uzNRwY ++ cat /tmp/tmp.VGYB739sFc ++ rm /tmp/tmp.raT7uzNRwY /tmp/tmp.VGYB739sFc ++ return 0 + [[ '' == true ]] + wait_pod some-name-rs0-2 + local pod=some-name-rs0-2 + set +o xtrace waiting for pod/some-name-rs0-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.X4MrAbo4i1 +++ mktemp ++ local LAST_ERR=/tmp/tmp.i7q543zK9I ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.X4MrAbo4i1 ++ cat /tmp/tmp.i7q543zK9I ++ rm /tmp/tmp.X4MrAbo4i1 /tmp/tmp.i7q543zK9I ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.v7wGeELwZS +++ mktemp ++ local LAST_ERR=/tmp/tmp.7XgKfLtNY0 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.v7wGeELwZS ++ cat /tmp/tmp.7XgKfLtNY0 ++ rm /tmp/tmp.v7wGeELwZS /tmp/tmp.7XgKfLtNY0 ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster readyness.. + wait_cluster_consistency some-name + local cluster_name=some-name + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster readyness' waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.b32C2JhC5R +++ mktemp ++ local LAST_ERR=/tmp/tmp.KwCwZ1yBAl ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.b32C2JhC5R ++ cat /tmp/tmp.KwCwZ1yBAl ++ rm /tmp/tmp.b32C2JhC5R /tmp/tmp.KwCwZ1yBAl ++ return 0 + [[ ready == ready ]] + echo .OK .OK + desc 'verify horizons after scale down' + set +o xtrace ----------------------------------------------------------------------------------- verify horizons after scale down ----------------------------------------------------------------------------------- + run_mongo_tls 'rs.conf().members.map(function(member) { return member.horizons }).sort((a, b) => a.external.localeCompare(b.external))' clusterAdmin:clusterAdmin123456@some-name-rs0-0.clouddemo.xyz,some-name-rs0-1.clouddemo.xyz,some-name-rs0-2.clouddemo.xyz mongodb '' --quiet + local 'command=rs.conf().members.map(function(member) { return member.horizons }).sort((a, b) => a.external.localeCompare(b.external))' + grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|does not match the remote host name' + local uri=clusterAdmin:clusterAdmin123456@some-name-rs0-0.clouddemo.xyz,some-name-rs0-1.clouddemo.xyz,some-name-rs0-2.clouddemo.xyz + local driver=mongodb + local suffix=.svc.cluster.local ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.WqbXdu6YLi +++ mktemp ++ local LAST_ERR=/tmp/tmp.vvUIgr78Lr ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.WqbXdu6YLi ++ cat /tmp/tmp.vvUIgr78Lr ++ rm /tmp/tmp.WqbXdu6YLi /tmp/tmp.vvUIgr78Lr ++ return 0 + local client_container=psmdb-client-6bd9478dff-xj848 + local mongo_flag=--quiet + local port=27017 ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 + [[ clusterAdmin:clusterAdmin123456@some-name-rs0-0.clouddemo.xyz,some-name-rs0-1.clouddemo.xyz,some-name-rs0-2.clouddemo.xyz == *cfg* ]] + replica_set=rs0 + kubectl_bin exec psmdb-client-6bd9478dff-xj848 -- bash -c 'printf '\''rs.conf().members.map(function(member) { return member.horizons }).sort((a, b) => a.external.localeCompare(b.external))\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@some-name-rs0-0.clouddemo.xyz,some-name-rs0-1.clouddemo.xyz,some-name-rs0-2.clouddemo.xyz.svc.cluster.local:27017/admin?replicaSet=rs0 --tls --tlsCAFile /etc/mongodb-ssl/ca.crt --tlsCertificateKeyFile /tmp/tls.pem --tlsAllowInvalidHostnames --quiet' ++ mktemp + local LAST_OUT=/tmp/tmp.VY9QIQXC8X ++ mktemp + local LAST_ERR=/tmp/tmp.sOPhBuXRIU + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-6bd9478dff-xj848 -- bash -c 'printf '\''rs.conf().members.map(function(member) { return member.horizons }).sort((a, b) => a.external.localeCompare(b.external))\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@some-name-rs0-0.clouddemo.xyz,some-name-rs0-1.clouddemo.xyz,some-name-rs0-2.clouddemo.xyz.svc.cluster.local:27017/admin?replicaSet=rs0 --tls --tlsCAFile /etc/mongodb-ssl/ca.crt --tlsCertificateKeyFile /tmp/tls.pem --tlsAllowInvalidHostnames --quiet' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.VY9QIQXC8X + cat /tmp/tmp.sOPhBuXRIU + rm /tmp/tmp.VY9QIQXC8X /tmp/tmp.sOPhBuXRIU + return 0 + diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/split-horizon-manual-tls/compare/horizons-3.json /tmp/tmp.i1hhairPLp/horizons.json + desc 'remove horizon configuration' + set +o xtrace ----------------------------------------------------------------------------------- remove horizon configuration ----------------------------------------------------------------------------------- + apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/split-horizon-manual-tls/conf/some-name.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/split-horizon-manual-tls/conf/some-name.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/split-horizon-manual-tls/conf/some-name.yml ++ mktemp + yq eval '(.spec | select(.image == null)).image = "docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0"' + yq eval '(.spec | select(has("pmm"))).pmm.image = "docker.io/percona/pmm-client:2.44.1-1"' + yq eval '(.spec | select(has("initImage"))).initImage = "docker.io/perconalab/percona-server-mongodb-operator:PR-2058-fb8cc7187"' + yq eval '(.spec | select(has("backup"))).backup.image = "docker.io/perconalab/percona-server-mongodb-operator:main-backup"' + local LAST_OUT=/tmp/tmp.87HN2MeKuN + /usr/sbin/sed -e s/NAME_SPACE/split-horizon-manual-tls-24334/g ++ mktemp + yq eval '.spec.upgradeOptions.apply="Never"' + local LAST_ERR=/tmp/tmp.LKqtpTcZDa + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.87HN2MeKuN perconaservermongodb.psmdb.percona.com/some-name configured + cat /tmp/tmp.LKqtpTcZDa + rm /tmp/tmp.87HN2MeKuN /tmp/tmp.LKqtpTcZDa + return 0 + wait_for_running some-name-rs0 3 + local name=some-name-rs0 + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-rs0-0 + local pod=some-name-rs0-0 + set +o xtrace waiting for pod/some-name-rs0-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-rs0-1 + local pod=some-name-rs0-1 + set +o xtrace waiting for pod/some-name-rs0-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.FzqBTKXy0G +++ mktemp ++ local LAST_ERR=/tmp/tmp.a4uzFotswr ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.FzqBTKXy0G ++ cat /tmp/tmp.a4uzFotswr ++ rm /tmp/tmp.FzqBTKXy0G /tmp/tmp.a4uzFotswr ++ return 0 + [[ '' == true ]] + wait_pod some-name-rs0-2 + local pod=some-name-rs0-2 + set +o xtrace waiting for pod/some-name-rs0-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.31b0fiwm0i +++ mktemp ++ local LAST_ERR=/tmp/tmp.jopTliBmer ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.31b0fiwm0i ++ cat /tmp/tmp.jopTliBmer ++ rm /tmp/tmp.31b0fiwm0i /tmp/tmp.jopTliBmer ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.DL12t30Lt4 +++ mktemp ++ local LAST_ERR=/tmp/tmp.UWHb71RaiL ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.DL12t30Lt4 ++ cat /tmp/tmp.UWHb71RaiL ++ rm /tmp/tmp.DL12t30Lt4 /tmp/tmp.UWHb71RaiL ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster readyness..................................................... + wait_cluster_consistency some-name + local cluster_name=some-name + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster readyness' waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.2pOBLMMrLo +++ mktemp ++ local LAST_ERR=/tmp/tmp.ONosdxtQES ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.2pOBLMMrLo ++ cat /tmp/tmp.ONosdxtQES ++ rm /tmp/tmp.2pOBLMMrLo /tmp/tmp.ONosdxtQES ++ return 0 + [[ ready == ready ]] + echo .OK .OK + destroy split-horizon-manual-tls-24334 + local namespace=split-horizon-manual-tls-24334 + local ignore_logs=true + [[ 1 == 1 ]] + echo 'SKIP_DELETE=1, not destroying split-horizon-manual-tls-24334' SKIP_DELETE=1, not destroying split-horizon-manual-tls-24334 + return + desc 'test passed' + set +o xtrace ----------------------------------------------------------------------------------- test passed -----------------------------------------------------------------------------------