Log: /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/logs/tls-issue-cert-manager-8-0.log Warning: version difference between client (1.37) and server (1.34) exceeds the supported minor version skew of +/-1 Warning: version difference between client (1.37) and server (1.34) exceeds the supported minor version skew of +/-1 + main + create_infra tls-issue-cert-manager-14810 + local ns=tls-issue-cert-manager-14810 + '[' -n pxc-operator ']' + read -r pns name + kubectl_bin get pxc --all-namespaces -o 'jsonpath={range .items[*]}{.metadata.namespace} {.metadata.name}{"\n"}{end}' + read -r pns name + kubectl_bin get pxc-backup --all-namespaces -o 'jsonpath={range .items[*]}{.metadata.namespace} {.metadata.name}{"\n"}{end}' + timeout 300 kubectl delete pxc --all --all-namespaces --wait=false --ignore-not-found No resources found + timeout 300 kubectl delete pxc-backup --all --all-namespaces --wait=false --ignore-not-found No resources found + timeout 300 kubectl delete pxc-restore --all --all-namespaces --wait=false --ignore-not-found No resources found + create_namespace pxc-operator + local namespace=pxc-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ sed s/NAMESPACE// ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ awk '-F ' '{print $2}' + local chaos_mesh_ns= + '[' -n '' ']' ++ awk '{print $1}' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ awk '{print $1}' ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get crd ++ grep chaos-mesh.org + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get clusterrolebinding + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + '[' -n '' ']' + desc 'cleaned up old namespaces pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin delete namespace pxc-operator + awk '{print$1}' ++ mktemp + grep -E -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME' + xargs kubectl delete ns + local LAST_OUT=/tmp/tmp.S7dLmZE4B5 + kubectl_bin get ns ++ mktemp ++ mktemp + local LAST_ERR=/tmp/tmp.sDudzN6z8c + local exit_status=0 ++ seq 0 2 + local LAST_OUT=/tmp/tmp.TrJlAN7v2A + for i in $(seq 0 2) + set +e + kubectl delete namespace pxc-operator ++ mktemp + local LAST_ERR=/tmp/tmp.0z45nl6lu0 + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get ns + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in $(seq 0 2) + set +e + kubectl delete namespace pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.TrJlAN7v2A + cat /tmp/tmp.0z45nl6lu0 + rm /tmp/tmp.TrJlAN7v2A /tmp/tmp.0z45nl6lu0 + return 0 error: resource(s) were provided, but no name was specified + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in $(seq 0 2) + set +e + kubectl delete namespace pxc-operator + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.S7dLmZE4B5 + cat /tmp/tmp.sDudzN6z8c Error from server (NotFound): namespaces "pxc-operator" not found + rm /tmp/tmp.S7dLmZE4B5 /tmp/tmp.sDudzN6z8c + return 1 + : + wait_for_delete namespace/pxc-operator + local res=namespace/pxc-operator + echo -n 'waiting for namespace/pxc-operator to be deleted' waiting for namespace/pxc-operator to be deleted+ set +o xtrace Error from server (NotFound): namespaces "pxc-operator" not found + desc 'create namespace pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- create namespace pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin create namespace pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.FUytAya8qX ++ mktemp + local LAST_ERR=/tmp/tmp.e1Z7GJNnWr + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.FUytAya8qX namespace/pxc-operator created + cat /tmp/tmp.e1Z7GJNnWr + rm /tmp/tmp.FUytAya8qX /tmp/tmp.e1Z7GJNnWr + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.4RnyZkEzmU +++ mktemp ++ local LAST_ERR=/tmp/tmp.b7hHhYs72j ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.4RnyZkEzmU ++ cat /tmp/tmp.b7hHhYs72j ++ rm /tmp/tmp.4RnyZkEzmU /tmp/tmp.b7hHhYs72j ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2546-caf25420-2-cluster9 --namespace=pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.IcpBr9orRP ++ mktemp + local LAST_ERR=/tmp/tmp.VNJtt1bToF + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2546-caf25420-2-cluster9 --namespace=pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.IcpBr9orRP Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2546-caf25420-2-cluster9" modified. + cat /tmp/tmp.VNJtt1bToF + rm /tmp/tmp.IcpBr9orRP /tmp/tmp.VNJtt1bToF + return 0 + deploy_operator + desc 'start PXC operator' + set +o xtrace ----------------------------------------------------------------------------------- start PXC operator ----------------------------------------------------------------------------------- + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.z07oX88prQ ++ mktemp + local LAST_ERR=/tmp/tmp.TKYxuaBSdv + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.z07oX88prQ customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com serverside-applied + cat /tmp/tmp.TKYxuaBSdv + rm /tmp/tmp.z07oX88prQ /tmp/tmp.TKYxuaBSdv + return 0 + kubectl_bin wait --for=condition=Established --timeout=120s crd/perconaxtradbclusters.pxc.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.Q3te2osGFF ++ mktemp + local LAST_ERR=/tmp/tmp.HFr7tbtbw1 + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=condition=Established --timeout=120s crd/perconaxtradbclusters.pxc.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Q3te2osGFF customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com condition met + cat /tmp/tmp.HFr7tbtbw1 + rm /tmp/tmp.Q3te2osGFF /tmp/tmp.HFr7tbtbw1 + return 0 + kubectl_bin wait --for=condition=Established --timeout=120s crd/perconaxtradbclusterbackups.pxc.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.uwx6HRppZ2 ++ mktemp + local LAST_ERR=/tmp/tmp.RlNfgjh5G0 + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=condition=Established --timeout=120s crd/perconaxtradbclusterbackups.pxc.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.uwx6HRppZ2 customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com condition met + cat /tmp/tmp.RlNfgjh5G0 + rm /tmp/tmp.uwx6HRppZ2 /tmp/tmp.RlNfgjh5G0 + return 0 + kubectl_bin wait --for=condition=Established --timeout=120s crd/perconaxtradbclusterrestores.pxc.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.TTULhMefuW ++ mktemp + local LAST_ERR=/tmp/tmp.MTuffDIQxL + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=condition=Established --timeout=120s crd/perconaxtradbclusterrestores.pxc.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.TTULhMefuW customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com condition met + cat /tmp/tmp.MTuffDIQxL + rm /tmp/tmp.TTULhMefuW /tmp/tmp.MTuffDIQxL + return 0 + '[' -n pxc-operator ']' + apply_rbac cw-rbac + local operator_namespace=pxc-operator + local rbac=cw-rbac + sed -e 's^namespace: .*^namespace: pxc-operator^' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/deploy/cw-rbac.yaml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.OOxwjrpv2N ++ mktemp + local LAST_ERR=/tmp/tmp.9prCLvNXeR + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.OOxwjrpv2N clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged serviceaccount/percona-xtradb-cluster-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged + cat /tmp/tmp.9prCLvNXeR + rm /tmp/tmp.OOxwjrpv2N /tmp/tmp.9prCLvNXeR + return 0 + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/deploy/cw-operator.yaml + sed -e 's^image: .*^image: perconalab/percona-xtradb-cluster-operator:PR-2546-caf25420^' + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"' - + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "PXCO_FEATURE_GATES").value) = ""' - + kubectl_bin apply -f - + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "LOG_LEVEL").value) = "VERBOSE"' - + sed -e 's^failureThreshold: .*^failureThreshold: 10^' ++ mktemp + local LAST_OUT=/tmp/tmp.kx2gadOfhr ++ mktemp + local LAST_ERR=/tmp/tmp.DCI3fYu6qq + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.kx2gadOfhr deployment.apps/percona-xtradb-cluster-operator created service/percona-xtradb-cluster-operator created + cat /tmp/tmp.DCI3fYu6qq + rm /tmp/tmp.kx2gadOfhr /tmp/tmp.DCI3fYu6qq + return 0 + sleep 10 + kubectl_bin wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s ++ mktemp + local LAST_OUT=/tmp/tmp.hpppkl2ys2 ++ mktemp + local LAST_ERR=/tmp/tmp.3YBF83yOrU + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.hpppkl2ys2 pod/percona-xtradb-cluster-operator-658fcf4b98-4pqtj condition met + cat /tmp/tmp.3YBF83yOrU + rm /tmp/tmp.hpppkl2ys2 /tmp/tmp.3YBF83yOrU + return 0 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ grep -c percona-xtradb-cluster-operator +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name' ++ head -1 ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.bk2S8DXFcx +++ mktemp ++ local LAST_ERR=/tmp/tmp.IqJWBBYmaH ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.bk2S8DXFcx ++ cat /tmp/tmp.IqJWBBYmaH ++ rm /tmp/tmp.bk2S8DXFcx /tmp/tmp.IqJWBBYmaH ++ return 0 + wait_pod percona-xtradb-cluster-operator-658fcf4b98-4pqtj 480 pxc-operator + local pod=percona-xtradb-cluster-operator-658fcf4b98-4pqtj + local max_retry=480 + local ns=pxc-operator ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo percona-xtradb-cluster-operator-658fcf4b98-4pqtj ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/percona-xtradb-cluster-operator-658fcf4b98-4pqtj condition met waiting for pod/percona-xtradb-cluster-operator-658fcf4b98-4pqtj to become Ready.Ok + sleep 3 + create_namespace tls-issue-cert-manager-14810 + local namespace=tls-issue-cert-manager-14810 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ sed s/NAMESPACE// ++ awk '-F ' '{print $2}' + local chaos_mesh_ns= + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + '[' -n '' ']' + desc 'cleaned up old namespaces tls-issue-cert-manager-14810' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces tls-issue-cert-manager-14810 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace tls-issue-cert-manager-14810 + grep -E -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME' + kubectl_bin get ns ++ mktemp + local LAST_OUT=/tmp/tmp.Uj4LJbyXI2 + xargs kubectl delete ns ++ mktemp + local LAST_OUT=/tmp/tmp.L2lV47kH15 + awk '{print$1}' ++ mktemp ++ mktemp + local LAST_ERR=/tmp/tmp.AcG0NegHQJ + local exit_status=0 + local LAST_ERR=/tmp/tmp.renCIObzch + local exit_status=0 ++ seq 0 2 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get ns + for i in $(seq 0 2) + set +e + kubectl delete namespace tls-issue-cert-manager-14810 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Uj4LJbyXI2 + cat /tmp/tmp.AcG0NegHQJ + rm /tmp/tmp.Uj4LJbyXI2 /tmp/tmp.AcG0NegHQJ + return 0 error: resource(s) were provided, but no name was specified + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in $(seq 0 2) + set +e + kubectl delete namespace tls-issue-cert-manager-14810 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in $(seq 0 2) + set +e + kubectl delete namespace tls-issue-cert-manager-14810 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.L2lV47kH15 + cat /tmp/tmp.renCIObzch Error from server (NotFound): namespaces "tls-issue-cert-manager-14810" not found + rm /tmp/tmp.L2lV47kH15 /tmp/tmp.renCIObzch + return 1 + : + wait_for_delete namespace/tls-issue-cert-manager-14810 + local res=namespace/tls-issue-cert-manager-14810 + echo -n 'waiting for namespace/tls-issue-cert-manager-14810 to be deleted' waiting for namespace/tls-issue-cert-manager-14810 to be deleted+ set +o xtrace Error from server (NotFound): namespaces "tls-issue-cert-manager-14810" not found + desc 'create namespace tls-issue-cert-manager-14810' + set +o xtrace ----------------------------------------------------------------------------------- create namespace tls-issue-cert-manager-14810 ----------------------------------------------------------------------------------- + kubectl_bin create namespace tls-issue-cert-manager-14810 ++ mktemp + local LAST_OUT=/tmp/tmp.yJO8iZB5aP ++ mktemp + local LAST_ERR=/tmp/tmp.CiaRwVXgaF + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace tls-issue-cert-manager-14810 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.yJO8iZB5aP namespace/tls-issue-cert-manager-14810 created + cat /tmp/tmp.CiaRwVXgaF + rm /tmp/tmp.yJO8iZB5aP /tmp/tmp.CiaRwVXgaF + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.fvNPIqxisy +++ mktemp ++ local LAST_ERR=/tmp/tmp.iFcw56dp5v ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.fvNPIqxisy ++ cat /tmp/tmp.iFcw56dp5v ++ rm /tmp/tmp.fvNPIqxisy /tmp/tmp.iFcw56dp5v ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2546-caf25420-2-cluster9 --namespace=tls-issue-cert-manager-14810 ++ mktemp + local LAST_OUT=/tmp/tmp.iQaqJ4DvP6 ++ mktemp + local LAST_ERR=/tmp/tmp.8kRhXs0JWv + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2546-caf25420-2-cluster9 --namespace=tls-issue-cert-manager-14810 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.iQaqJ4DvP6 Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2546-caf25420-2-cluster9" modified. + cat /tmp/tmp.8kRhXs0JWv + rm /tmp/tmp.iQaqJ4DvP6 /tmp/tmp.8kRhXs0JWv + return 0 + apply_secrets + desc 'create secrets for cloud storages' + set +o xtrace ----------------------------------------------------------------------------------- create secrets for cloud storages ----------------------------------------------------------------------------------- + '[' -z '' ']' + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/cloud-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.nzc9NLTJcd ++ mktemp + local LAST_ERR=/tmp/tmp.3sDEUV30wI + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/cloud-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.nzc9NLTJcd secret/minio-secret created secret/aws-s3-secret created secret/do-spaces-secret created secret/gcp-cs-secret created secret/azure-secret created + cat /tmp/tmp.3sDEUV30wI + rm /tmp/tmp.nzc9NLTJcd /tmp/tmp.3sDEUV30wI + return 0 + cluster=some-name-tls-issue + deploy_cert_manager + desc 'deploy cert manager' + set +o xtrace ----------------------------------------------------------------------------------- deploy cert manager ----------------------------------------------------------------------------------- + kubectl_bin create namespace cert-manager ++ mktemp + local LAST_OUT=/tmp/tmp.HY2tYO2xKr ++ mktemp + local LAST_ERR=/tmp/tmp.rYAEDc5VH5 + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace cert-manager + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.HY2tYO2xKr namespace/cert-manager created + cat /tmp/tmp.rYAEDc5VH5 + rm /tmp/tmp.HY2tYO2xKr /tmp/tmp.rYAEDc5VH5 + return 0 + kubectl_bin label namespace cert-manager certmanager.k8s.io/disable-validation=true ++ mktemp + local LAST_OUT=/tmp/tmp.xPWC1it7ba ++ mktemp + local LAST_ERR=/tmp/tmp.2NUEU94ihu + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl label namespace cert-manager certmanager.k8s.io/disable-validation=true + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.xPWC1it7ba namespace/cert-manager labeled + cat /tmp/tmp.2NUEU94ihu + rm /tmp/tmp.xPWC1it7ba /tmp/tmp.2NUEU94ihu + return 0 + kubectl_bin apply -f https://github.com/jetstack/cert-manager/releases/download/v1.20.2/cert-manager.yaml --validate=false ++ mktemp + local LAST_OUT=/tmp/tmp.1ObL8OqIbB ++ mktemp + local LAST_ERR=/tmp/tmp.BoJOanVP9k + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/v1.20.2/cert-manager.yaml --validate=false + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.1ObL8OqIbB namespace/cert-manager configured customresourcedefinition.apiextensions.k8s.io/challenges.acme.cert-manager.io created customresourcedefinition.apiextensions.k8s.io/orders.acme.cert-manager.io created customresourcedefinition.apiextensions.k8s.io/certificaterequests.cert-manager.io created customresourcedefinition.apiextensions.k8s.io/certificates.cert-manager.io created customresourcedefinition.apiextensions.k8s.io/clusterissuers.cert-manager.io created customresourcedefinition.apiextensions.k8s.io/issuers.cert-manager.io created serviceaccount/cert-manager-cainjector created serviceaccount/cert-manager created serviceaccount/cert-manager-webhook created clusterrole.rbac.authorization.k8s.io/cert-manager-cainjector created clusterrole.rbac.authorization.k8s.io/cert-manager-controller-issuers created clusterrole.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers created clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificates created clusterrole.rbac.authorization.k8s.io/cert-manager-controller-orders created clusterrole.rbac.authorization.k8s.io/cert-manager-controller-challenges created clusterrole.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim created clusterrole.rbac.authorization.k8s.io/cert-manager-cluster-view created clusterrole.rbac.authorization.k8s.io/cert-manager-view created clusterrole.rbac.authorization.k8s.io/cert-manager-edit created clusterrole.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io created clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests created clusterrole.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews created clusterrolebinding.rbac.authorization.k8s.io/cert-manager-cainjector created clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-issuers created clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers created clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificates created clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-orders created clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-challenges created clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim created clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io created clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests created clusterrolebinding.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews created role.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection created role.rbac.authorization.k8s.io/cert-manager:leaderelection created role.rbac.authorization.k8s.io/cert-manager-tokenrequest created role.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created rolebinding.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection created rolebinding.rbac.authorization.k8s.io/cert-manager:leaderelection created rolebinding.rbac.authorization.k8s.io/cert-manager-tokenrequest created rolebinding.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created service/cert-manager-cainjector created service/cert-manager created service/cert-manager-webhook created deployment.apps/cert-manager-cainjector created deployment.apps/cert-manager created deployment.apps/cert-manager-webhook created mutatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook created validatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook created + cat /tmp/tmp.BoJOanVP9k Warning: resource namespaces/cert-manager is missing the kubectl.kubernetes.io/last-applied-configuration annotation which is required by kubectl apply. kubectl apply should only be used on resources created declaratively by either kubectl create --save-config or kubectl apply. The missing annotation will be patched automatically. + rm /tmp/tmp.1ObL8OqIbB /tmp/tmp.BoJOanVP9k + return 0 + '[' '' == 4.10 ']' + kubectl_bin -n cert-manager rollout status deployment/cert-manager --timeout=120s ++ mktemp + local LAST_OUT=/tmp/tmp.9FOO1JZJEv ++ mktemp + local LAST_ERR=/tmp/tmp.ZtmasFX7kU + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl -n cert-manager rollout status deployment/cert-manager --timeout=120s + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.9FOO1JZJEv Waiting for deployment "cert-manager" rollout to finish: 0 of 1 updated replicas are available... deployment "cert-manager" successfully rolled out + cat /tmp/tmp.ZtmasFX7kU + rm /tmp/tmp.9FOO1JZJEv /tmp/tmp.ZtmasFX7kU + return 0 + kubectl_bin -n cert-manager rollout status deployment/cert-manager-cainjector --timeout=120s ++ mktemp + local LAST_OUT=/tmp/tmp.BmnG8utR3U ++ mktemp + local LAST_ERR=/tmp/tmp.C5JKJFGyRK + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl -n cert-manager rollout status deployment/cert-manager-cainjector --timeout=120s + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.BmnG8utR3U deployment "cert-manager-cainjector" successfully rolled out + cat /tmp/tmp.C5JKJFGyRK + rm /tmp/tmp.BmnG8utR3U /tmp/tmp.C5JKJFGyRK + return 0 + kubectl_bin -n cert-manager rollout status deployment/cert-manager-webhook --timeout=120s ++ mktemp + local LAST_OUT=/tmp/tmp.mxC1WQO17r ++ mktemp + local LAST_ERR=/tmp/tmp.cZRMnnAU0n + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl -n cert-manager rollout status deployment/cert-manager-webhook --timeout=120s + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.mxC1WQO17r Waiting for deployment "cert-manager-webhook" rollout to finish: 0 of 1 updated replicas are available... deployment "cert-manager-webhook" successfully rolled out + cat /tmp/tmp.cZRMnnAU0n + rm /tmp/tmp.mxC1WQO17r /tmp/tmp.cZRMnnAU0n + return 0 + log 'waiting for cert-manager webhook caBundle to be injected' ++ date +%Y-%m-%dT%H:%M:%S%z + echo '[2026-10-05T14:28:01+0000]' waiting for cert-manager webhook caBundle to be injected [2026-10-05T14:28:01+0000] waiting for cert-manager webhook caBundle to be injected ++ date +%s + local deadline=1791210601 + kubectl_bin get validatingwebhookconfigurations cert-manager-webhook -o 'jsonpath={.webhooks[0].clientConfig.caBundle}' + grep -q . + log 'probing cert-manager webhook' ++ date +%Y-%m-%dT%H:%M:%S%z + echo '[2026-10-05T14:28:02+0000]' probing cert-manager webhook [2026-10-05T14:28:02+0000] probing cert-manager webhook + local probe_file ++ mktemp + probe_file=/tmp/tmp.YeW3rX1Cfs + cat + local retries=0 + kubectl_bin apply -f /tmp/tmp.YeW3rX1Cfs --dry-run=server + rm -f /tmp/tmp.YeW3rX1Cfs + desc 'wait for cert-manager to be ready' + set +o xtrace ----------------------------------------------------------------------------------- wait for cert-manager to be ready ----------------------------------------------------------------------------------- + kubectl_bin -n cert-manager wait --for=condition=Available deployment/cert-manager --timeout=180s ++ mktemp + local LAST_OUT=/tmp/tmp.dhWbYfCcIJ ++ mktemp + local LAST_ERR=/tmp/tmp.vv3apiA9am + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl -n cert-manager wait --for=condition=Available deployment/cert-manager --timeout=180s + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.dhWbYfCcIJ deployment.apps/cert-manager condition met + cat /tmp/tmp.vv3apiA9am + rm /tmp/tmp.dhWbYfCcIJ /tmp/tmp.vv3apiA9am + return 0 + kubectl_bin -n cert-manager wait --for=condition=Available deployment/cert-manager-cainjector --timeout=180s ++ mktemp + local LAST_OUT=/tmp/tmp.JMzDxIDD8t ++ mktemp + local LAST_ERR=/tmp/tmp.Zoh6fffVCd + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl -n cert-manager wait --for=condition=Available deployment/cert-manager-cainjector --timeout=180s + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.JMzDxIDD8t deployment.apps/cert-manager-cainjector condition met + cat /tmp/tmp.Zoh6fffVCd + rm /tmp/tmp.JMzDxIDD8t /tmp/tmp.Zoh6fffVCd + return 0 + kubectl_bin -n cert-manager wait --for=condition=Available deployment/cert-manager-webhook --timeout=180s ++ mktemp + local LAST_OUT=/tmp/tmp.DvKRl17s5q ++ mktemp + local LAST_ERR=/tmp/tmp.kPoKTDywx0 + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl -n cert-manager wait --for=condition=Available deployment/cert-manager-webhook --timeout=180s + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.DvKRl17s5q deployment.apps/cert-manager-webhook condition met + cat /tmp/tmp.kPoKTDywx0 + rm /tmp/tmp.DvKRl17s5q /tmp/tmp.kPoKTDywx0 + return 0 + local cm_ready=0 ++ seq 1 30 + for cm_try in $(seq 1 30) + echo '{"apiVersion":"cert-manager.io/v1","kind":"Issuer","metadata":{"name":"cert-manager-readiness-check"},"spec":{"selfSigned":{}}}' + kubectl_bin apply -f - issuer.cert-manager.io/cert-manager-readiness-check created + kubectl_bin delete issuer cert-manager-readiness-check --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.qtEsoTYiVi ++ mktemp + local LAST_ERR=/tmp/tmp.KMkg0z7LLR + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete issuer cert-manager-readiness-check --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.qtEsoTYiVi issuer.cert-manager.io "cert-manager-readiness-check" deleted from tls-issue-cert-manager-14810 namespace + cat /tmp/tmp.KMkg0z7LLR + rm /tmp/tmp.qtEsoTYiVi /tmp/tmp.KMkg0z7LLR + return 0 + cm_ready=1 + break + '[' 1 -ne 1 ']' + desc 'create pxc cluster' + set +o xtrace ----------------------------------------------------------------------------------- create pxc cluster ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/secrets_without_tls.yml ++ mktemp + local LAST_OUT=/tmp/tmp.AUpr5TRm9n ++ mktemp + local LAST_ERR=/tmp/tmp.Yo1ZocPiZc + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/secrets_without_tls.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.AUpr5TRm9n secret/my-cluster-secrets created + cat /tmp/tmp.Yo1ZocPiZc + rm /tmp/tmp.AUpr5TRm9n /tmp/tmp.Yo1ZocPiZc + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/tls-issue-cert-manager/conf/client.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/tls-issue-cert-manager/conf/client.yml + local pvc_name= + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/tls-issue-cert-manager/conf/client.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/tls-issue-cert-manager/conf/client.yml + local pvc_name= + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/tls-issue-cert-manager/conf/client.yml ++ mktemp + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2546-caf25420#' + local LAST_OUT=/tmp/tmp.HFqXtHLUwl + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' ++ mktemp + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + local LAST_ERR=/tmp/tmp.3DuGWbPUjF + local exit_status=0 + /usr/bin/sed -e s~minio-service.#namespace~minio-service.tls-issue-cert-manager-14810~ ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.HFqXtHLUwl deployment.apps/pxc-client created + cat /tmp/tmp.3DuGWbPUjF + rm /tmp/tmp.HFqXtHLUwl /tmp/tmp.3DuGWbPUjF + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/tls-issue-cert-manager/conf/some-name-tls-issue.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/tls-issue-cert-manager/conf/some-name-tls-issue.yml + local pvc_name= + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/tls-issue-cert-manager/conf/some-name-tls-issue.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/tls-issue-cert-manager/conf/some-name-tls-issue.yml + local pvc_name= + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/tls-issue-cert-manager/conf/some-name-tls-issue.yml ++ mktemp + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + local LAST_OUT=/tmp/tmp.nanTFDWRuG ++ mktemp + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2546-caf25420#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + local LAST_ERR=/tmp/tmp.KepV7AtE47 + local exit_status=0 + /usr/bin/sed -e s~minio-service.#namespace~minio-service.tls-issue-cert-manager-14810~ + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.nanTFDWRuG perconaxtradbcluster.pxc.percona.com/some-name-tls-issue created + cat /tmp/tmp.KepV7AtE47 + rm /tmp/tmp.nanTFDWRuG /tmp/tmp.KepV7AtE47 + return 0 + desc 'wait for cluster to be ready' + set +o xtrace ----------------------------------------------------------------------------------- wait for cluster to be ready ----------------------------------------------------------------------------------- + wait_for_running some-name-tls-issue-haproxy 1 + local name=some-name-tls-issue-haproxy + local size=1 + local last_pod + local max_retry=480 + [[ -z some-name-tls-issue-haproxy ]] + [[ 1 -le 0 ]] + last_pod=0 + desc 'Waiting for 1 running pods for some-name-tls-issue-haproxy' + set +o xtrace ----------------------------------------------------------------------------------- Waiting for 1 running pods for some-name-tls-issue-haproxy ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in $(seq 0 $last_pod) + wait_pod some-name-tls-issue-haproxy-0 480 + local pod=some-name-tls-issue-haproxy-0 + local max_retry=480 + local ns= ++ echo some-name-tls-issue-haproxy-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace Error from server (NotFound): pods "some-name-tls-issue-haproxy-0" not found waiting for pod/some-name-tls-issue-haproxy-0 to become Ready.....................Ok + wait_for_running some-name-tls-issue-pxc 3 + local name=some-name-tls-issue-pxc + local size=3 + local last_pod + local max_retry=480 + [[ -z some-name-tls-issue-pxc ]] + [[ 3 -le 0 ]] + last_pod=2 + desc 'Waiting for 3 running pods for some-name-tls-issue-pxc' + set +o xtrace ----------------------------------------------------------------------------------- Waiting for 3 running pods for some-name-tls-issue-pxc ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in $(seq 0 $last_pod) + wait_pod some-name-tls-issue-pxc-0 480 + local pod=some-name-tls-issue-pxc-0 + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ echo some-name-tls-issue-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/some-name-tls-issue-pxc-0 condition met waiting for pod/some-name-tls-issue-pxc-0 to become Ready.Ok + for i in $(seq 0 $last_pod) + wait_pod some-name-tls-issue-pxc-1 480 + local pod=some-name-tls-issue-pxc-1 + local max_retry=480 + local ns= ++ echo some-name-tls-issue-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/some-name-tls-issue-pxc-1 condition met waiting for pod/some-name-tls-issue-pxc-1 to become Ready.Ok + for i in $(seq 0 $last_pod) + wait_pod some-name-tls-issue-pxc-2 480 + local pod=some-name-tls-issue-pxc-2 + local max_retry=480 + local ns= ++ echo some-name-tls-issue-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/some-name-tls-issue-pxc-2 condition met waiting for pod/some-name-tls-issue-pxc-2 to become Ready.Ok + wait_cluster_consistency some-name-tls-issue 3 2 + local cluster_name=some-name-tls-issue + local pxc_size=3 + local proxy_size=2 + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local max_seconds=1500 + local delay=5 + local retries=300 + local pxc_ready + local proxy_ready + sleep 7 + echo -n 'waiting for pxc/some-name-tls-issue to be ready' waiting for pxc/some-name-tls-issue to be ready++ seq 1 300 + for _ in $(seq 1 "${retries}") ++ kubectl_bin get pxc some-name-tls-issue -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.o7i0LZHPyH +++ mktemp ++ local LAST_ERR=/tmp/tmp.UhbGhgXBrX ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc some-name-tls-issue -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.o7i0LZHPyH ++ cat /tmp/tmp.UhbGhgXBrX ++ rm /tmp/tmp.o7i0LZHPyH /tmp/tmp.UhbGhgXBrX ++ return 0 + state=ready ++ kubectl_bin get pxc some-name-tls-issue -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.vLWsDckYq1 +++ mktemp ++ local LAST_ERR=/tmp/tmp.lLdHz5fqYf ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc some-name-tls-issue -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.vLWsDckYq1 ++ cat /tmp/tmp.lLdHz5fqYf ++ rm /tmp/tmp.vLWsDckYq1 /tmp/tmp.lLdHz5fqYf ++ return 0 + pxc_ready=3 + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine some-name-tls-issue +++ local cluster_name=some-name-tls-issue ++++ get_proxy some-name-tls-issue ++++ local target_cluster=some-name-tls-issue +++++ kubectl_bin get pxc some-name-tls-issue -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.FwEQraoSyj ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.lYKfizJdMD +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get pxc some-name-tls-issue -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.FwEQraoSyj +++++ cat /tmp/tmp.lYKfizJdMD +++++ rm /tmp/tmp.FwEQraoSyj /tmp/tmp.lYKfizJdMD +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo some-name-tls-issue-haproxy ++++ return +++ local cluster_proxy=some-name-tls-issue-haproxy +++ echo haproxy ++ kubectl_bin get pxc some-name-tls-issue -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.EPoJ08chlP +++ mktemp ++ local LAST_ERR=/tmp/tmp.culdEE9NfN ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc some-name-tls-issue -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.EPoJ08chlP ++ cat /tmp/tmp.culdEE9NfN ++ rm /tmp/tmp.EPoJ08chlP /tmp/tmp.culdEE9NfN ++ return 0 + proxy_ready=2 + [[ ready == \r\e\a\d\y ]] + [[ 3 == \3 ]] + [[ 2 == \2 ]] + echo + return + desc 'check if certificates issued with certmanager' + set +o xtrace ----------------------------------------------------------------------------------- check if certificates issued with certmanager ----------------------------------------------------------------------------------- + tlsSecretsShouldExist some-name-tls-issue-ssl + local secretName=some-name-tls-issue-ssl + checkTLSSecret some-name-tls-issue-ssl ca.crt + local secretName=some-name-tls-issue-ssl + local dataKey=ca.crt ++ kubectl_bin get secrets/some-name-tls-issue-ssl -o json +++ mktemp ++ local LAST_OUT=/tmp/tmp.WgV8dWvNKm +++ mktemp ++ local LAST_ERR=/tmp/tmp.Oq1YJjIXqu ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get secrets/some-name-tls-issue-ssl -o json ++ jq '.data["ca.crt"]' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.WgV8dWvNKm ++ cat /tmp/tmp.Oq1YJjIXqu ++ rm /tmp/tmp.WgV8dWvNKm /tmp/tmp.Oq1YJjIXqu ++ return 0 + local 'secretData="LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURFakNDQWZxZ0F3SUJBZ0lVQmVwNytoT1dxd2ZlSDdxeVRjUFl5amt2NGQ0d0RRWUpLb1pJaHZjTkFRRUwKQlFBd0lURWZNQjBHQTFVRUF4TVdjMjl0WlMxdVlXMWxMWFJzY3kxcGMzTjFaUzFqWVRBZUZ3MHlOakV3TURVeApOREk0TVRWYUZ3MHlPREE0TVRNeU1qSTRNVFZhTUNFeEh6QWRCZ05WQkFNVEZuTnZiV1V0Ym1GdFpTMTBiSE10CmFYTnpkV1V0WTJFd2dnRWlNQTBHQ1NxR1NJYjNEUUVCQVFVQUE0SUJEd0F3Z2dFS0FvSUJBUUM4L2t3Z2pPR2MKdGRQRGVzZ0hIbTVRcFpMcE9nYkhLd1M1d095SnArYk9xTW04azlQS0F6N1MvWHB0Tm9NNWowQTB6VUJjb0poTgpmcStYR1dhUHVBS1ZXWFVnMjVtWFZSSWxHK29HSithNkhCWnJickRSb0lwOCtqTUUzblF1ZXhGdWJqakVocDF5Cm05RWl1QVg4SlJIYkRUZm9rTUpUbEJCMlNmTmVGZmUxM3Jady9TbXk2VjZjOHZqVEJCeFVTVTR2blhFQVZHaUgKVG54ajUxMGI0MVJJWnVVSVRKY25IYUZqM0sxdzhoeERmcExhZk1Ub1ZsMEw5YitHYTJRUEtncnhObHBmMERwdQpWQk5QakpxWC9VcEhzTlRqMzk3NGVxb0lyS29ycDh6OXhaTmFwT2pmWUJObHh1bHdLVzcveWEyazJFWnhMTW0yClA1RVZaSURBZm1TSkFnTUJBQUdqUWpCQU1BNEdBMVVkRHdFQi93UUVBd0lDcERBUEJnTlZIUk1CQWY4RUJUQUQKQVFIL01CMEdBMVVkRGdRV0JCVE5NTHhTVGp3dGo3cFA4dmJVbzF2YjhpWFFTekFOQmdrcWhraUc5dzBCQVFzRgpBQU9DQVFFQWFHOEpYRjNjTVIxQXc1MUFJbW5rRkQ4UlgzUW5VUkFvTXE5dDZSK1E4aFczWE5aaU15OWNKbWcwCkgwTDdQNzhQTm54V20wcm5qS1VlOFFFS0ZjcyttVTU5eGxGNkY5Y1pvdXBPV0xCMTYvWEFJSXp5Tmh4SWtNN1EKR2hBd0doM3E0UXpCaW9rbktWSlAxRHFydFl3UkduQXg3UW9aRFNZT2FyVkVremRxR1JjUmdOTVNOYlQ5OHg0eApSb1M4cFZWNERlbXJYbXExWUZLM3p6WTY2SUFsM1lvLzBwaVVSRjdSQkQ3UXhiL0JVbzJRZmhvdTNSUkhlRURRCjk2QW01MDNFL0YvRS9PL294ZzNkL1k4ZE9jQ1ZFNTc3YW03dnNjWWFIYnVaeHA5UGtFK2NpaFF1TG52dGxYY0EKb3A4UGpOYU5kREpBZTN3QnM1S0JKS1d2T1UvNTNRPT0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo="' + '[' -z '"LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURFakNDQWZxZ0F3SUJBZ0lVQmVwNytoT1dxd2ZlSDdxeVRjUFl5amt2NGQ0d0RRWUpLb1pJaHZjTkFRRUwKQlFBd0lURWZNQjBHQTFVRUF4TVdjMjl0WlMxdVlXMWxMWFJzY3kxcGMzTjFaUzFqWVRBZUZ3MHlOakV3TURVeApOREk0TVRWYUZ3MHlPREE0TVRNeU1qSTRNVFZhTUNFeEh6QWRCZ05WQkFNVEZuTnZiV1V0Ym1GdFpTMTBiSE10CmFYTnpkV1V0WTJFd2dnRWlNQTBHQ1NxR1NJYjNEUUVCQVFVQUE0SUJEd0F3Z2dFS0FvSUJBUUM4L2t3Z2pPR2MKdGRQRGVzZ0hIbTVRcFpMcE9nYkhLd1M1d095SnArYk9xTW04azlQS0F6N1MvWHB0Tm9NNWowQTB6VUJjb0poTgpmcStYR1dhUHVBS1ZXWFVnMjVtWFZSSWxHK29HSithNkhCWnJickRSb0lwOCtqTUUzblF1ZXhGdWJqakVocDF5Cm05RWl1QVg4SlJIYkRUZm9rTUpUbEJCMlNmTmVGZmUxM3Jady9TbXk2VjZjOHZqVEJCeFVTVTR2blhFQVZHaUgKVG54ajUxMGI0MVJJWnVVSVRKY25IYUZqM0sxdzhoeERmcExhZk1Ub1ZsMEw5YitHYTJRUEtncnhObHBmMERwdQpWQk5QakpxWC9VcEhzTlRqMzk3NGVxb0lyS29ycDh6OXhaTmFwT2pmWUJObHh1bHdLVzcveWEyazJFWnhMTW0yClA1RVZaSURBZm1TSkFnTUJBQUdqUWpCQU1BNEdBMVVkRHdFQi93UUVBd0lDcERBUEJnTlZIUk1CQWY4RUJUQUQKQVFIL01CMEdBMVVkRGdRV0JCVE5NTHhTVGp3dGo3cFA4dmJVbzF2YjhpWFFTekFOQmdrcWhraUc5dzBCQVFzRgpBQU9DQVFFQWFHOEpYRjNjTVIxQXc1MUFJbW5rRkQ4UlgzUW5VUkFvTXE5dDZSK1E4aFczWE5aaU15OWNKbWcwCkgwTDdQNzhQTm54V20wcm5qS1VlOFFFS0ZjcyttVTU5eGxGNkY5Y1pvdXBPV0xCMTYvWEFJSXp5Tmh4SWtNN1EKR2hBd0doM3E0UXpCaW9rbktWSlAxRHFydFl3UkduQXg3UW9aRFNZT2FyVkVremRxR1JjUmdOTVNOYlQ5OHg0eApSb1M4cFZWNERlbXJYbXExWUZLM3p6WTY2SUFsM1lvLzBwaVVSRjdSQkQ3UXhiL0JVbzJRZmhvdTNSUkhlRURRCjk2QW01MDNFL0YvRS9PL294ZzNkL1k4ZE9jQ1ZFNTc3YW03dnNjWWFIYnVaeHA5UGtFK2NpaFF1TG52dGxYY0EKb3A4UGpOYU5kREpBZTN3QnM1S0JKS1d2T1UvNTNRPT0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo="' ']' + checkTLSSecret some-name-tls-issue-ssl tls.crt + local secretName=some-name-tls-issue-ssl + local dataKey=tls.crt ++ kubectl_bin get secrets/some-name-tls-issue-ssl -o json ++ jq '.data["tls.crt"]' +++ mktemp ++ local LAST_OUT=/tmp/tmp.uXNFtd9B6E +++ mktemp ++ local LAST_ERR=/tmp/tmp.lM7api9smt ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get secrets/some-name-tls-issue-ssl -o json ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.uXNFtd9B6E ++ cat /tmp/tmp.lM7api9smt ++ rm /tmp/tmp.uXNFtd9B6E /tmp/tmp.lM7api9smt ++ return 0 + local 'secretData="LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURvVENDQW9tZ0F3SUJBZ0lVRk1Gbk0raGhJckxJR0p5SjNOc1dTSGRmcXFFd0RRWUpLb1pJaHZjTkFRRUwKQlFBd0lURWZNQjBHQTFVRUF4TVdjMjl0WlMxdVlXMWxMWFJzY3kxcGMzTjFaUzFqWVRBZUZ3MHlOakV3TURVeApOREk0TVRoYUZ3MHlOakV4TWpJeU1qSTRNVGhhTUNjeEpUQWpCZ05WQkFNVEhITnZiV1V0Ym1GdFpTMTBiSE10CmFYTnpkV1V0Y0hKdmVIbHpjV3d3Z2dFaU1BMEdDU3FHU0liM0RRRUJBUVVBQTRJQkR3QXdnZ0VLQW9JQkFRRFUKZk93SzE3eXBLSXpFZWRnUkU1aHZvMUR4dGVPU3phWXh1YlBscVdndTdqUE1yWDFzVGd1WDdKOWVxSGJ0ZmgzRwpsQVNmdUxqVm5NemRlRVp5U01ycVhhU25jbks0b0hKcUFGTXlES3Ira0JpTXpqeVlnRDcxRFBBZERpRWZBcnVrCkZjcTJiRDhUUTQ5enpmVC9yTjBYcWZwTWY1M04yeXMvbEFWVmNtTUJWSWlRNmFzeFZUaloxazhsa1RGSEVKUjkKZkVGbFZQQnUreXlBbW90QkFONXA5TW1odVpOWDl4YzlCVCtsaS94T2lXMzBZek1ydGwySTFxS2xScHdTeWsxTgpiaUdoOE5tcUN4VXlRZEovbmdOaDJxN3FWTUhmT0srcGRZRzg1cUNxakNiaDFXL1J0UlZ1bjVELzQ1ZCtrajhuClpFV21CaldCb0J4eTg5SFRHeTQvQWdNQkFBR2pnY293Z2Njd0RnWURWUjBQQVFIL0JBUURBZ1dnTUF3R0ExVWQKRXdFQi93UUNNQUF3SHdZRFZSMGpCQmd3Rm9BVXpUQzhVazQ4TFkrNlQvTDIxS05iMi9JbDBFc3dnWVVHQTFVZApFUVIrTUh5Q0YzTnZiV1V0Ym1GdFpTMTBiSE10YVhOemRXVXRjSGhqZ2h4emIyMWxMVzVoYldVdGRHeHpMV2x6CmMzVmxMWEJ5YjNoNWMzRnNnaGtxTG5OdmJXVXRibUZ0WlMxMGJITXRhWE56ZFdVdGNIaGpnaDRxTG5OdmJXVXQKYm1GdFpTMTBiSE10YVhOemRXVXRjSEp2ZUhsemNXeUNDSFJsYzNRdVkyOXRNQTBHQ1NxR1NJYjNEUUVCQ3dVQQpBNElCQVFBVTFMeXhZYWsrSnNoSjJaMXRhbUgxMGduOFFISHJTdktHSUU2clR1WmtPd1EyQkg2ZGhCSmFPdXoyCmhZNmFHTXdvREhvQ0R6a1EvUjJFQWxVbFNkUFB1VjFTTVlJZlFLZk5yS2ZTU0ZBVGZuK2JZSlExMkVuRktTcmcKUXBkVzVhZG12YTZmejd3YkRSU05raXh2QjJ6dnRZcFpWa0FjWVJBdGpEM0Q0L2dzbGlSTitmazlTU1lNdUZoVgpQOUVzazNPUnEvcEdiN1pNTkFWUThqckRWdk5qZVd4SjRzb2U5NFV2T0V5V2E5NXZZR3NkTWVyR1BiS0JrUWNLCjVkNU9Nc3Y1QTV4aWdWalZlbHQ5RE1HU2diUVNTUE9oUFgzajNBK09FZDByR0E3WEVudDZiMU1xMzl1LzdHeEwKb0tGWllPLzE2TnFWVDlIMlY5NVh1TUszcElvegotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg=="' + '[' -z '"LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURvVENDQW9tZ0F3SUJBZ0lVRk1Gbk0raGhJckxJR0p5SjNOc1dTSGRmcXFFd0RRWUpLb1pJaHZjTkFRRUwKQlFBd0lURWZNQjBHQTFVRUF4TVdjMjl0WlMxdVlXMWxMWFJzY3kxcGMzTjFaUzFqWVRBZUZ3MHlOakV3TURVeApOREk0TVRoYUZ3MHlOakV4TWpJeU1qSTRNVGhhTUNjeEpUQWpCZ05WQkFNVEhITnZiV1V0Ym1GdFpTMTBiSE10CmFYTnpkV1V0Y0hKdmVIbHpjV3d3Z2dFaU1BMEdDU3FHU0liM0RRRUJBUVVBQTRJQkR3QXdnZ0VLQW9JQkFRRFUKZk93SzE3eXBLSXpFZWRnUkU1aHZvMUR4dGVPU3phWXh1YlBscVdndTdqUE1yWDFzVGd1WDdKOWVxSGJ0ZmgzRwpsQVNmdUxqVm5NemRlRVp5U01ycVhhU25jbks0b0hKcUFGTXlES3Ira0JpTXpqeVlnRDcxRFBBZERpRWZBcnVrCkZjcTJiRDhUUTQ5enpmVC9yTjBYcWZwTWY1M04yeXMvbEFWVmNtTUJWSWlRNmFzeFZUaloxazhsa1RGSEVKUjkKZkVGbFZQQnUreXlBbW90QkFONXA5TW1odVpOWDl4YzlCVCtsaS94T2lXMzBZek1ydGwySTFxS2xScHdTeWsxTgpiaUdoOE5tcUN4VXlRZEovbmdOaDJxN3FWTUhmT0srcGRZRzg1cUNxakNiaDFXL1J0UlZ1bjVELzQ1ZCtrajhuClpFV21CaldCb0J4eTg5SFRHeTQvQWdNQkFBR2pnY293Z2Njd0RnWURWUjBQQVFIL0JBUURBZ1dnTUF3R0ExVWQKRXdFQi93UUNNQUF3SHdZRFZSMGpCQmd3Rm9BVXpUQzhVazQ4TFkrNlQvTDIxS05iMi9JbDBFc3dnWVVHQTFVZApFUVIrTUh5Q0YzTnZiV1V0Ym1GdFpTMTBiSE10YVhOemRXVXRjSGhqZ2h4emIyMWxMVzVoYldVdGRHeHpMV2x6CmMzVmxMWEJ5YjNoNWMzRnNnaGtxTG5OdmJXVXRibUZ0WlMxMGJITXRhWE56ZFdVdGNIaGpnaDRxTG5OdmJXVXQKYm1GdFpTMTBiSE10YVhOemRXVXRjSEp2ZUhsemNXeUNDSFJsYzNRdVkyOXRNQTBHQ1NxR1NJYjNEUUVCQ3dVQQpBNElCQVFBVTFMeXhZYWsrSnNoSjJaMXRhbUgxMGduOFFISHJTdktHSUU2clR1WmtPd1EyQkg2ZGhCSmFPdXoyCmhZNmFHTXdvREhvQ0R6a1EvUjJFQWxVbFNkUFB1VjFTTVlJZlFLZk5yS2ZTU0ZBVGZuK2JZSlExMkVuRktTcmcKUXBkVzVhZG12YTZmejd3YkRSU05raXh2QjJ6dnRZcFpWa0FjWVJBdGpEM0Q0L2dzbGlSTitmazlTU1lNdUZoVgpQOUVzazNPUnEvcEdiN1pNTkFWUThqckRWdk5qZVd4SjRzb2U5NFV2T0V5V2E5NXZZR3NkTWVyR1BiS0JrUWNLCjVkNU9Nc3Y1QTV4aWdWalZlbHQ5RE1HU2diUVNTUE9oUFgzajNBK09FZDByR0E3WEVudDZiMU1xMzl1LzdHeEwKb0tGWllPLzE2TnFWVDlIMlY5NVh1TUszcElvegotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg=="' ']' + checkTLSSecret some-name-tls-issue-ssl tls.key + local secretName=some-name-tls-issue-ssl + local dataKey=tls.key ++ kubectl_bin get secrets/some-name-tls-issue-ssl -o json ++ jq '.data["tls.key"]' +++ mktemp ++ local LAST_OUT=/tmp/tmp.6OMIyoi40G +++ mktemp ++ local LAST_ERR=/tmp/tmp.2iIZ5rT2En ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get secrets/some-name-tls-issue-ssl -o json ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.6OMIyoi40G ++ cat /tmp/tmp.2iIZ5rT2En ++ rm /tmp/tmp.6OMIyoi40G /tmp/tmp.2iIZ5rT2En ++ return 0 + local 'secretData="LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb3dJQkFBS0NBUUVBMUh6c0N0ZThxU2lNeEhuWUVST1liNk5ROGJYamtzMm1NYm16NWFsb0x1NHp6SzE5CmJFNExsK3lmWHFoMjdYNGR4cFFFbjdpNDFaek0zWGhHY2tqSzZsMmtwM0p5dUtCeWFnQlRNZ3lxL3BBWWpNNDgKbUlBKzlRendIUTRoSHdLN3BCWEt0bXcvRTBPUGM4MzAvNnpkRjZuNlRIK2R6ZHNyUDVRRlZYSmpBVlNJa09tcgpNVlU0MmRaUEpaRXhSeENVZlh4QlpWVHdidnNzZ0pxTFFRRGVhZlRKb2JtVFYvY1hQUVUvcFl2OFRvbHQ5R016Cks3WmRpTmFpcFVhY0VzcE5UVzRob2ZEWnFnc1ZNa0hTZjU0RFlkcXU2bFRCM3ppdnFYV0J2T2FncW93bTRkVnYKMGJVVmJwK1EvK09YZnBJL0oyUkZwZ1kxZ2FBY2N2UFIweHN1UHdJREFRQUJBb0lCQUVQNi8yMXNuZkFraC95bgoycWVEOUdYckF3K0JtS0F6NlBMSnJiTlJJb0s1Tkp0TCtlOTNmS0FMVk9WczBqZWkvSlcyYjFCR1FlUWRlVEYxCnRGd0Z5OTExUGlOUUswSEVHemFzaWZGZU9pcWRJUzJUSWJITWZkZmFYc3Q4NUwzT0d3Z2c3S3kyTHV2VDFvWS8KSXgzVlNScTBCOThIaVEzK0MzVVl2eWdhSTNDZDdHRjVHc1ltTVd6aGl4dmQ0ZEFNMjZ6ajZXRzFpdWdreXlEeQo5UzhHMWdUaDVrbjJCMUozSUVockFGc1JaczJ3S0plcDA2aTA0RmRHWHAvblhmUGRpdWhmK21LTVVva3p4K1BnCnBKdWduQlZERyt2MkJHdkpkYkN6VnlzOHpOei84TjlLNVF1V1ZTRUd6bEdtcDJiU0F0VUhlR0M0VmpFRlR4bGwKeDMxcjc3a0NnWUVBOTBVSkV5Zm81VzVFYlZ0MERqNFdmbnNJSHQ4SlhrRlkzSS9PbGxlejVzKzZUdzEyY0t4awpoMkxZbE1ueVZUR0lkRUU1WG81WXNURWl2UVAvVklnd1BEYUh3ZnJZMFlWeTJIRXpodHFYak1RWkxTdGEvSmpoCmkzSDIxaHFuRC9nNFc3SUpOYXcxaHNqZXBGTTVWeE1sRzUxaHowSzFVMCtVWjFzZDB6ZFNuaE1DZ1lFQTIvMkMKZ05LNWFTREpPbi82RXhvdk1Hd1NnTW11WnBUVmNXa1NzcVpSZGJVUTlzRE1tYjY1VjhkaFhma0VLTVJBa25VRgo5NERmelovUkI4MHZPbW5HdFVMM1NYWVpuamU4VGJTUEVKdU1RcWRLRHJTb3hGeUczUHZRU0F0eWQxQ3RSSVlEClZtR3MwbWdldWlVeS9LeTAwM1NYRmZFR1dEdnhFSCtMMkw1eEJLVUNnWUF4YURycTkvT0NQR2t4eks5QTFVa2IKVnF3RnR4MGhGeitST3E3VnZZcDEyUlhxQzgrWnJlaWNZTFRUN2hJYzBidnBOc3NteUp5U016WTB2R0s5V2tVQgpFd2hIU3o4ZGhtT3ZPZGRJMzg2ZVV1U1E4UFBLT3lYOS9mU0xUV0lLUDlmTEFxVHljNGx5ZUJwOGpTQ005SWRhCm9EVFYvQWROTVQ1KzlHYmVPRlNxdndLQmdGZEkvOFhEVm1PYTZaeTNlRzZwM2YxVmRBWDdmNklkL1k1UkJXVXkKem5Gb2xVR0NmK3VrbS85ekVYUHF3TGE0SDNLODBkQkJKd3Q4Rk1FR3JXMTgyS1hFMno4Zmg4eWFDckcrS1dHSwpoaDFRS1BjcXdTZWdHcmFGaHo5K296MHFHb1Zzc3RoTVN2ajA3UFFvbkZGYTZTaFpRelVtZThENGo0NWZQUXFUCmpWNHRBb0dCQU9nWGtPTGlMMGxURGlTMTQ5TDJRSm9ZcE96VVdwMDI3UUd2MmsrbTk1UTQxRmxWNG4wWllyeEsKcmErRGx3TCs0ek8wYTJXZmd6blEyejBKUDJSS0VxOGFNTG95V0ZWUFNKeStBczJGQmtacitZZFVzSnVUbHVLdgpnbjBpT2xYVzNRVFQvcUZpdjdsU2RMa21NcytwQ1VGNzl0YTl4dDF1bVl5dW1HYmgzZUZkCi0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCg=="' + '[' -z '"LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb3dJQkFBS0NBUUVBMUh6c0N0ZThxU2lNeEhuWUVST1liNk5ROGJYamtzMm1NYm16NWFsb0x1NHp6SzE5CmJFNExsK3lmWHFoMjdYNGR4cFFFbjdpNDFaek0zWGhHY2tqSzZsMmtwM0p5dUtCeWFnQlRNZ3lxL3BBWWpNNDgKbUlBKzlRendIUTRoSHdLN3BCWEt0bXcvRTBPUGM4MzAvNnpkRjZuNlRIK2R6ZHNyUDVRRlZYSmpBVlNJa09tcgpNVlU0MmRaUEpaRXhSeENVZlh4QlpWVHdidnNzZ0pxTFFRRGVhZlRKb2JtVFYvY1hQUVUvcFl2OFRvbHQ5R016Cks3WmRpTmFpcFVhY0VzcE5UVzRob2ZEWnFnc1ZNa0hTZjU0RFlkcXU2bFRCM3ppdnFYV0J2T2FncW93bTRkVnYKMGJVVmJwK1EvK09YZnBJL0oyUkZwZ1kxZ2FBY2N2UFIweHN1UHdJREFRQUJBb0lCQUVQNi8yMXNuZkFraC95bgoycWVEOUdYckF3K0JtS0F6NlBMSnJiTlJJb0s1Tkp0TCtlOTNmS0FMVk9WczBqZWkvSlcyYjFCR1FlUWRlVEYxCnRGd0Z5OTExUGlOUUswSEVHemFzaWZGZU9pcWRJUzJUSWJITWZkZmFYc3Q4NUwzT0d3Z2c3S3kyTHV2VDFvWS8KSXgzVlNScTBCOThIaVEzK0MzVVl2eWdhSTNDZDdHRjVHc1ltTVd6aGl4dmQ0ZEFNMjZ6ajZXRzFpdWdreXlEeQo5UzhHMWdUaDVrbjJCMUozSUVockFGc1JaczJ3S0plcDA2aTA0RmRHWHAvblhmUGRpdWhmK21LTVVva3p4K1BnCnBKdWduQlZERyt2MkJHdkpkYkN6VnlzOHpOei84TjlLNVF1V1ZTRUd6bEdtcDJiU0F0VUhlR0M0VmpFRlR4bGwKeDMxcjc3a0NnWUVBOTBVSkV5Zm81VzVFYlZ0MERqNFdmbnNJSHQ4SlhrRlkzSS9PbGxlejVzKzZUdzEyY0t4awpoMkxZbE1ueVZUR0lkRUU1WG81WXNURWl2UVAvVklnd1BEYUh3ZnJZMFlWeTJIRXpodHFYak1RWkxTdGEvSmpoCmkzSDIxaHFuRC9nNFc3SUpOYXcxaHNqZXBGTTVWeE1sRzUxaHowSzFVMCtVWjFzZDB6ZFNuaE1DZ1lFQTIvMkMKZ05LNWFTREpPbi82RXhvdk1Hd1NnTW11WnBUVmNXa1NzcVpSZGJVUTlzRE1tYjY1VjhkaFhma0VLTVJBa25VRgo5NERmelovUkI4MHZPbW5HdFVMM1NYWVpuamU4VGJTUEVKdU1RcWRLRHJTb3hGeUczUHZRU0F0eWQxQ3RSSVlEClZtR3MwbWdldWlVeS9LeTAwM1NYRmZFR1dEdnhFSCtMMkw1eEJLVUNnWUF4YURycTkvT0NQR2t4eks5QTFVa2IKVnF3RnR4MGhGeitST3E3VnZZcDEyUlhxQzgrWnJlaWNZTFRUN2hJYzBidnBOc3NteUp5U016WTB2R0s5V2tVQgpFd2hIU3o4ZGhtT3ZPZGRJMzg2ZVV1U1E4UFBLT3lYOS9mU0xUV0lLUDlmTEFxVHljNGx5ZUJwOGpTQ005SWRhCm9EVFYvQWROTVQ1KzlHYmVPRlNxdndLQmdGZEkvOFhEVm1PYTZaeTNlRzZwM2YxVmRBWDdmNklkL1k1UkJXVXkKem5Gb2xVR0NmK3VrbS85ekVYUHF3TGE0SDNLODBkQkJKd3Q4Rk1FR3JXMTgyS1hFMno4Zmg4eWFDckcrS1dHSwpoaDFRS1BjcXdTZWdHcmFGaHo5K296MHFHb1Zzc3RoTVN2ajA3UFFvbkZGYTZTaFpRelVtZThENGo0NWZQUXFUCmpWNHRBb0dCQU9nWGtPTGlMMGxURGlTMTQ5TDJRSm9ZcE96VVdwMDI3UUd2MmsrbTk1UTQxRmxWNG4wWllyeEsKcmErRGx3TCs0ek8wYTJXZmd6blEyejBKUDJSS0VxOGFNTG95V0ZWUFNKeStBczJGQmtacitZZFVzSnVUbHVLdgpnbjBpT2xYVzNRVFQvcUZpdjdsU2RMa21NcytwQ1VGNzl0YTl4dDF1bVl5dW1HYmgzZUZkCi0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCg=="' ']' + desc 'check if CA issuer created' + set +o xtrace ----------------------------------------------------------------------------------- check if CA issuer created ----------------------------------------------------------------------------------- + compare_kubectl issuer/some-name-tls-issue-pxc-ca-issuer + local resource=issuer/some-name-tls-issue-pxc-ca-issuer + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/tls-issue-cert-manager/compare/issuer_some-name-tls-issue-pxc-ca-issuer.yml + local new_result=/tmp/tmp.vlgOWXkFpe/issuer_some-name-tls-issue-pxc-ca-issuer.yml + desc 'compare issuer/some-name-tls-issue-pxc-ca-issuer-' + set +o xtrace ----------------------------------------------------------------------------------- compare issuer/some-name-tls-issue-pxc-ca-issuer- ----------------------------------------------------------------------------------- + '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/tls-issue-cert-manager/compare/issuer_some-name-tls-issue-pxc-ca-issuer-eks.yml ']' + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/tls-issue-cert-manager/compare/issuer_some-name-tls-issue-pxc-ca-issuer-80.yml ']' + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + version_gt 1.33 ++ echo '1.34 >= 1.33' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/tls-issue-cert-manager/compare/issuer_some-name-tls-issue-pxc-ca-issuer-k133.yml ']' + version_gt 1.29 ++ echo '1.34 >= 1.29' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/tls-issue-cert-manager/compare/issuer_some-name-tls-issue-pxc-ca-issuer-k129.yml ']' + version_gt 1.27 ++ echo '1.34 >= 1.27' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/tls-issue-cert-manager/compare/issuer_some-name-tls-issue-pxc-ca-issuer-k127.yml ']' + version_gt 1.24 ++ echo '1.34 >= 1.24' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/tls-issue-cert-manager/compare/issuer_some-name-tls-issue-pxc-ca-issuer-k124.yml ']' + version_gt 1.22 ++ echo '1.34 >= 1.22' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/tls-issue-cert-manager/compare/issuer_some-name-tls-issue-pxc-ca-issuer-k122.yml ']' + version_gt 1.21 ++ echo '1.34 >= 1.21' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/tls-issue-cert-manager/compare/issuer_some-name-tls-issue-pxc-ca-issuer-k121.yml ']' + '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/tls-issue-cert-manager/compare/issuer_some-name-tls-issue-pxc-ca-issuer-oc.yml ']' + version_gt 1.29 ++ echo '1.34 >= 1.29' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/tls-issue-cert-manager/compare/issuer_some-name-tls-issue-pxc-ca-issuer-k129-oc.yml ']' + '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/tls-issue-cert-manager/compare/issuer_some-name-tls-issue-pxc-ca-issuer-eks.yml ']' + '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/tls-issue-cert-manager/compare/issuer_some-name-tls-issue-pxc-ca-issuer-aks.yml ']' + kubectl_bin get -o yaml issuer/some-name-tls-issue-pxc-ca-issuer + yq eval ' del(.metadata.managedFields) | del(.. | select(has("creationTimestamp")).creationTimestamp) | del(.. | select(has("namespace")).namespace) | del(.. | select(has("uid")).uid) | del(.metadata.resourceVersion) | del(.spec.template.spec.containers[].env[] | select(.name == "CLUSTER_HASH")) | del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_PATH")) | del(.spec.template.spec.containers[].env[] | select(.name == "BACKUP_PATH")) | del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_URL")) | del(.spec.template.spec.containers[].env[] | select(.name == "AZURE_CONTAINER_NAME")) | del(.spec.template.spec.containers[].env[] | select(.name == "XTRABACKUP_ENABLED")) | del(.metadata.selfLink) | del(.metadata.deletionTimestamp) | del(.metadata.labels."batch.kubernetes.io/job-name") | del(.metadata.labels."job-name") | del(.metadata.labels."topology.kubernetes.io/region") | del(.metadata.labels."topology.kubernetes.io/zone") | del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") | del(.metadata.annotations."kubernetes.io/psp") | del(.metadata.annotations."batch.kubernetes.io/job-tracking") | del(.metadata.annotations."networking.gke.io/target-pool") | del(.metadata.annotations."cloud.google.com/neg") | del(.metadata.annotations."k8s.v1.cni.cncf.io*") | del(.metadata.annotations."k8s.ovn.org/pod-networks") | del(.metadata.annotations."kubernetes.digitalocean.com/load-balancer-id") | del(.metadata.annotations."service.beta.kubernetes.io/do-loadbalancer-type") | del(.spec.template.metadata.annotations."last-applied-secret") | del(.spec.template.metadata.labels."batch.kubernetes.io/job-name") | del(.spec.template.metadata.labels."job-name") | del(.. | select(has("batch.kubernetes.io/controller-uid"))."batch.kubernetes.io/controller-uid") | del(.. | select(has("image")).image) | del(.. | select(has("clusterIP")).clusterIP) | del(.. | select(has("clusterIPs")).clusterIPs) | del(.. | select(has("dataSource")).dataSource) | del(.. | select(has("procMount")).procMount) | del(.. | select(has("storageClassName")).storageClassName) | del(.. | select(has("finalizers")).finalizers) | del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") | del(.. | select(has("volumeName")).volumeName) | del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") | del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") | del(.spec.volumeMode) | del(.spec.nodeName) | del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") | del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") | del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") | del(.. | select(has("percona.com/env-secret-config-hash"))."percona.com/env-secret-config-hash") | del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") | del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") | del(.. | select(has("kubectl.kubernetes.io/default-container"))."kubectl.kubernetes.io/default-container") | del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) | del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) | del(.. | select(has("nodePort")).nodePort) | del(.. | select(has("imagePullSecrets")).imagePullSecrets) | del(.. | select(has("enableServiceLinks")).enableServiceLinks) | del(.status) | del(.spec.volumeClaimTemplates[].apiVersion) | del(.spec.volumeClaimTemplates[].kind) | del(.metadata.ownerReferences[].apiVersion) | del(.. | select(has("controller-uid")).controller-uid) | del(.. | select(has("preemptionPolicy")).preemptionPolicy) | del(.spec.ipFamilies) | del(.spec.ipFamilyPolicy) | del(.spec.updateStrategy.rollingUpdate.maxUnavailable) | (.. | select(. == "policy/v1beta1")) = "policy/v1" | del(.. | select(has("kubernetes.io/hostname"))."kubernetes.io/hostname") | (.. | select(tag == "!!str")) |= sub("tls-issue-cert-manager-14810", "namespace") | (.. | select(tag == "!!str")) |= sub("kube-api-access-.*", "kube-api-access") | del(.. | select(has("annotations")).annotations | select(length==0)) | ++ mktemp del(.spec.crVersion) | del(.. | select(.[] == "percona-xtradb-cluster-operator-workload-token*"))' - + local LAST_OUT=/tmp/tmp.WUsygmS2Ti ++ mktemp + local LAST_ERR=/tmp/tmp.7CWuc8dpvq + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get -o yaml issuer/some-name-tls-issue-pxc-ca-issuer + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.WUsygmS2Ti + cat /tmp/tmp.7CWuc8dpvq + rm /tmp/tmp.WUsygmS2Ti /tmp/tmp.7CWuc8dpvq + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/tls-issue-cert-manager/compare/issuer_some-name-tls-issue-pxc-ca-issuer.yml /tmp/tmp.vlgOWXkFpe/issuer_some-name-tls-issue-pxc-ca-issuer.yml + log 'compare_kubectl: issuer/some-name-tls-issue-pxc-ca-issuer OK' ++ date +%Y-%m-%dT%H:%M:%S%z + echo '[2026-10-05T14:32:13+0000]' compare_kubectl: issuer/some-name-tls-issue-pxc-ca-issuer OK [2026-10-05T14:32:13+0000] compare_kubectl: issuer/some-name-tls-issue-pxc-ca-issuer OK + desc 'check if issuer created' + set +o xtrace ----------------------------------------------------------------------------------- check if issuer created ----------------------------------------------------------------------------------- + compare_kubectl issuer/some-name-tls-issue-pxc-issuer + local resource=issuer/some-name-tls-issue-pxc-issuer + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/tls-issue-cert-manager/compare/issuer_some-name-tls-issue-pxc-issuer.yml + local new_result=/tmp/tmp.vlgOWXkFpe/issuer_some-name-tls-issue-pxc-issuer.yml + desc 'compare issuer/some-name-tls-issue-pxc-issuer-' + set +o xtrace ----------------------------------------------------------------------------------- compare issuer/some-name-tls-issue-pxc-issuer- ----------------------------------------------------------------------------------- + '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/tls-issue-cert-manager/compare/issuer_some-name-tls-issue-pxc-issuer-eks.yml ']' + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/tls-issue-cert-manager/compare/issuer_some-name-tls-issue-pxc-issuer-80.yml ']' + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + version_gt 1.33 ++ bc -l ++ echo '1.34 >= 1.33' + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/tls-issue-cert-manager/compare/issuer_some-name-tls-issue-pxc-issuer-k133.yml ']' + version_gt 1.29 ++ echo '1.34 >= 1.29' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/tls-issue-cert-manager/compare/issuer_some-name-tls-issue-pxc-issuer-k129.yml ']' + version_gt 1.27 ++ echo '1.34 >= 1.27' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/tls-issue-cert-manager/compare/issuer_some-name-tls-issue-pxc-issuer-k127.yml ']' + version_gt 1.24 ++ echo '1.34 >= 1.24' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/tls-issue-cert-manager/compare/issuer_some-name-tls-issue-pxc-issuer-k124.yml ']' + version_gt 1.22 ++ echo '1.34 >= 1.22' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/tls-issue-cert-manager/compare/issuer_some-name-tls-issue-pxc-issuer-k122.yml ']' + version_gt 1.21 ++ echo '1.34 >= 1.21' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/tls-issue-cert-manager/compare/issuer_some-name-tls-issue-pxc-issuer-k121.yml ']' + '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/tls-issue-cert-manager/compare/issuer_some-name-tls-issue-pxc-issuer-oc.yml ']' + version_gt 1.29 ++ bc -l ++ echo '1.34 >= 1.29' + '[' 1 -eq 1 ']' + return 0 + '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/tls-issue-cert-manager/compare/issuer_some-name-tls-issue-pxc-issuer-k129-oc.yml ']' + '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/tls-issue-cert-manager/compare/issuer_some-name-tls-issue-pxc-issuer-eks.yml ']' + '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/tls-issue-cert-manager/compare/issuer_some-name-tls-issue-pxc-issuer-aks.yml ']' + kubectl_bin get -o yaml issuer/some-name-tls-issue-pxc-issuer + yq eval ' del(.metadata.managedFields) | del(.. | select(has("creationTimestamp")).creationTimestamp) | del(.. | select(has("namespace")).namespace) | del(.. | select(has("uid")).uid) | del(.metadata.resourceVersion) | del(.spec.template.spec.containers[].env[] | select(.name == "CLUSTER_HASH")) | del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_PATH")) | del(.spec.template.spec.containers[].env[] | select(.name == "BACKUP_PATH")) | del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_URL")) | del(.spec.template.spec.containers[].env[] | select(.name == "AZURE_CONTAINER_NAME")) | del(.spec.template.spec.containers[].env[] | select(.name == "XTRABACKUP_ENABLED")) | del(.metadata.selfLink) | del(.metadata.deletionTimestamp) | del(.metadata.labels."batch.kubernetes.io/job-name") | del(.metadata.labels."job-name") | del(.metadata.labels."topology.kubernetes.io/region") | del(.metadata.labels."topology.kubernetes.io/zone") | del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") | del(.metadata.annotations."kubernetes.io/psp") | del(.metadata.annotations."batch.kubernetes.io/job-tracking") | del(.metadata.annotations."networking.gke.io/target-pool") | del(.metadata.annotations."cloud.google.com/neg") | del(.metadata.annotations."k8s.v1.cni.cncf.io*") | del(.metadata.annotations."k8s.ovn.org/pod-networks") | del(.metadata.annotations."kubernetes.digitalocean.com/load-balancer-id") | del(.metadata.annotations."service.beta.kubernetes.io/do-loadbalancer-type") | del(.spec.template.metadata.annotations."last-applied-secret") | del(.spec.template.metadata.labels."batch.kubernetes.io/job-name") | del(.spec.template.metadata.labels."job-name") | del(.. | select(has("batch.kubernetes.io/controller-uid"))."batch.kubernetes.io/controller-uid") | del(.. | select(has("image")).image) | del(.. | select(has("clusterIP")).clusterIP) | del(.. | select(has("clusterIPs")).clusterIPs) | del(.. | select(has("dataSource")).dataSource) | del(.. | select(has("procMount")).procMount) | del(.. | select(has("storageClassName")).storageClassName) | del(.. | select(has("finalizers")).finalizers) | del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") | del(.. | select(has("volumeName")).volumeName) | del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") | del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") | del(.spec.volumeMode) | del(.spec.nodeName) | del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") | del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") | del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") | del(.. | select(has("percona.com/env-secret-config-hash"))."percona.com/env-secret-config-hash") | del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") | del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") | del(.. | select(has("kubectl.kubernetes.io/default-container"))."kubectl.kubernetes.io/default-container") | del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) | del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) | del(.. | select(has("nodePort")).nodePort) | del(.. | select(has("imagePullSecrets")).imagePullSecrets) | del(.. | select(has("enableServiceLinks")).enableServiceLinks) | del(.status) | del(.spec.volumeClaimTemplates[].apiVersion) | del(.spec.volumeClaimTemplates[].kind) | del(.metadata.ownerReferences[].apiVersion) | del(.. | select(has("controller-uid")).controller-uid) | del(.. | select(has("preemptionPolicy")).preemptionPolicy) | del(.spec.ipFamilies) | del(.spec.ipFamilyPolicy) | del(.spec.updateStrategy.rollingUpdate.maxUnavailable) | (.. | select(. == "policy/v1beta1")) = "policy/v1" | del(.. | select(has("kubernetes.io/hostname"))."kubernetes.io/hostname") | (.. | select(tag == "!!str")) |= sub("tls-issue-cert-manager-14810", "namespace") | (.. | select(tag == "!!str")) |= sub("kube-api-access-.*", "kube-api-access") | del(.. | select(has("annotations")).annotations | select(length==0)) | del(.spec.crVersion) | del(.. | select(.[] == "percona-xtradb-cluster-operator-workload-token*"))' - ++ mktemp + local LAST_OUT=/tmp/tmp.LQE4oCoizi ++ mktemp + local LAST_ERR=/tmp/tmp.3RFTQSu6vu + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get -o yaml issuer/some-name-tls-issue-pxc-issuer + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.LQE4oCoizi + cat /tmp/tmp.3RFTQSu6vu + rm /tmp/tmp.LQE4oCoizi /tmp/tmp.3RFTQSu6vu + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/tls-issue-cert-manager/compare/issuer_some-name-tls-issue-pxc-issuer.yml /tmp/tmp.vlgOWXkFpe/issuer_some-name-tls-issue-pxc-issuer.yml + log 'compare_kubectl: issuer/some-name-tls-issue-pxc-issuer OK' ++ date +%Y-%m-%dT%H:%M:%S%z + echo '[2026-10-05T14:32:14+0000]' compare_kubectl: issuer/some-name-tls-issue-pxc-issuer OK [2026-10-05T14:32:14+0000] compare_kubectl: issuer/some-name-tls-issue-pxc-issuer OK + desc 'check if certificate issued' + set +o xtrace ----------------------------------------------------------------------------------- check if certificate issued ----------------------------------------------------------------------------------- + compare_kubectl certificate/some-name-tls-issue-ssl + local resource=certificate/some-name-tls-issue-ssl + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/tls-issue-cert-manager/compare/certificate_some-name-tls-issue-ssl.yml + local new_result=/tmp/tmp.vlgOWXkFpe/certificate_some-name-tls-issue-ssl.yml + desc 'compare certificate/some-name-tls-issue-ssl-' + set +o xtrace ----------------------------------------------------------------------------------- compare certificate/some-name-tls-issue-ssl- ----------------------------------------------------------------------------------- + '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/tls-issue-cert-manager/compare/certificate_some-name-tls-issue-ssl-eks.yml ']' + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/tls-issue-cert-manager/compare/certificate_some-name-tls-issue-ssl-80.yml ']' + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + version_gt 1.33 ++ echo '1.34 >= 1.33' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/tls-issue-cert-manager/compare/certificate_some-name-tls-issue-ssl-k133.yml ']' + version_gt 1.29 ++ echo '1.34 >= 1.29' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/tls-issue-cert-manager/compare/certificate_some-name-tls-issue-ssl-k129.yml ']' + version_gt 1.27 ++ echo '1.34 >= 1.27' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/tls-issue-cert-manager/compare/certificate_some-name-tls-issue-ssl-k127.yml ']' + version_gt 1.24 ++ echo '1.34 >= 1.24' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/tls-issue-cert-manager/compare/certificate_some-name-tls-issue-ssl-k124.yml ']' + version_gt 1.22 ++ echo '1.34 >= 1.22' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/tls-issue-cert-manager/compare/certificate_some-name-tls-issue-ssl-k122.yml ']' + version_gt 1.21 ++ echo '1.34 >= 1.21' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/tls-issue-cert-manager/compare/certificate_some-name-tls-issue-ssl-k121.yml ']' + '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/tls-issue-cert-manager/compare/certificate_some-name-tls-issue-ssl-oc.yml ']' + version_gt 1.29 ++ echo '1.34 >= 1.29' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/tls-issue-cert-manager/compare/certificate_some-name-tls-issue-ssl-k129-oc.yml ']' + '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/tls-issue-cert-manager/compare/certificate_some-name-tls-issue-ssl-eks.yml ']' + '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/tls-issue-cert-manager/compare/certificate_some-name-tls-issue-ssl-aks.yml ']' + kubectl_bin get -o yaml certificate/some-name-tls-issue-ssl + yq eval ' del(.metadata.managedFields) | del(.. | select(has("creationTimestamp")).creationTimestamp) | del(.. | select(has("namespace")).namespace) | del(.. | select(has("uid")).uid) | del(.metadata.resourceVersion) | del(.spec.template.spec.containers[].env[] | select(.name == "CLUSTER_HASH")) | del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_PATH")) | del(.spec.template.spec.containers[].env[] | select(.name == "BACKUP_PATH")) | del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_URL")) | del(.spec.template.spec.containers[].env[] | select(.name == "AZURE_CONTAINER_NAME")) | del(.spec.template.spec.containers[].env[] | select(.name == "XTRABACKUP_ENABLED")) | del(.metadata.selfLink) | del(.metadata.deletionTimestamp) | del(.metadata.labels."batch.kubernetes.io/job-name") | del(.metadata.labels."job-name") | del(.metadata.labels."topology.kubernetes.io/region") | del(.metadata.labels."topology.kubernetes.io/zone") | del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") | del(.metadata.annotations."kubernetes.io/psp") | del(.metadata.annotations."batch.kubernetes.io/job-tracking") | del(.metadata.annotations."networking.gke.io/target-pool") | del(.metadata.annotations."cloud.google.com/neg") | del(.metadata.annotations."k8s.v1.cni.cncf.io*") | del(.metadata.annotations."k8s.ovn.org/pod-networks") | del(.metadata.annotations."kubernetes.digitalocean.com/load-balancer-id") | del(.metadata.annotations."service.beta.kubernetes.io/do-loadbalancer-type") | del(.spec.template.metadata.annotations."last-applied-secret") | del(.spec.template.metadata.labels."batch.kubernetes.io/job-name") | del(.spec.template.metadata.labels."job-name") | del(.. | select(has("batch.kubernetes.io/controller-uid"))."batch.kubernetes.io/controller-uid") | del(.. | select(has("image")).image) | del(.. | select(has("clusterIP")).clusterIP) | del(.. | select(has("clusterIPs")).clusterIPs) | del(.. | select(has("dataSource")).dataSource) | del(.. | select(has("procMount")).procMount) | del(.. | select(has("storageClassName")).storageClassName) | del(.. | select(has("finalizers")).finalizers) | del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") | del(.. | select(has("volumeName")).volumeName) | del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") | del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") | del(.spec.volumeMode) | del(.spec.nodeName) | del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") | del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") | del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") | del(.. | select(has("percona.com/env-secret-config-hash"))."percona.com/env-secret-config-hash") | del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") | del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") | del(.. | select(has("kubectl.kubernetes.io/default-container"))."kubectl.kubernetes.io/default-container") | del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) | del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) | del(.. | select(has("nodePort")).nodePort) | del(.. | select(has("imagePullSecrets")).imagePullSecrets) | del(.. | select(has("enableServiceLinks")).enableServiceLinks) | del(.status) | del(.spec.volumeClaimTemplates[].apiVersion) | del(.spec.volumeClaimTemplates[].kind) | del(.metadata.ownerReferences[].apiVersion) | del(.. | select(has("controller-uid")).controller-uid) | del(.. | select(has("preemptionPolicy")).preemptionPolicy) | del(.spec.ipFamilies) | del(.spec.ipFamilyPolicy) | del(.spec.updateStrategy.rollingUpdate.maxUnavailable) | (.. | select(. == "policy/v1beta1")) = "policy/v1" | del(.. | select(has("kubernetes.io/hostname"))."kubernetes.io/hostname") | ++ mktemp (.. | select(tag == "!!str")) |= sub("tls-issue-cert-manager-14810", "namespace") | (.. | select(tag == "!!str")) |= sub("kube-api-access-.*", "kube-api-access") | del(.. | select(has("annotations")).annotations | select(length==0)) | del(.spec.crVersion) | del(.. | select(.[] == "percona-xtradb-cluster-operator-workload-token*"))' - + local LAST_OUT=/tmp/tmp.wKsMjoq8nn ++ mktemp + local LAST_ERR=/tmp/tmp.xy9hgeY7I1 + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get -o yaml certificate/some-name-tls-issue-ssl + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.wKsMjoq8nn + cat /tmp/tmp.xy9hgeY7I1 + rm /tmp/tmp.wKsMjoq8nn /tmp/tmp.xy9hgeY7I1 + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/tls-issue-cert-manager/compare/certificate_some-name-tls-issue-ssl.yml /tmp/tmp.vlgOWXkFpe/certificate_some-name-tls-issue-ssl.yml + log 'compare_kubectl: certificate/some-name-tls-issue-ssl OK' ++ date +%Y-%m-%dT%H:%M:%S%z + echo '[2026-10-05T14:32:16+0000]' compare_kubectl: certificate/some-name-tls-issue-ssl OK [2026-10-05T14:32:16+0000] compare_kubectl: certificate/some-name-tls-issue-ssl OK + desc 'check ssl-internal certificate using PXC' + set +o xtrace ----------------------------------------------------------------------------------- check ssl-internal certificate using PXC ----------------------------------------------------------------------------------- + check_verify_identity some-name-tls-issue-pxc + local host=some-name-tls-issue-pxc + local command=exit + local 'args=--ssl-ca=/etc/mysql/ssl-internal/ca.crt --ssl-mode=VERIFY_IDENTITY --protocol=tcp -uroot -proot_password --host=some-name-tls-issue-pxc' + kubectl_bin exec some-name-tls-issue-pxc-0 -- bash -c 'printf '\''%s\n'\'' "exit" | mysql -sN --ssl-ca=/etc/mysql/ssl-internal/ca.crt --ssl-mode=VERIFY_IDENTITY --protocol=tcp -uroot -proot_password --host=some-name-tls-issue-pxc' ++ mktemp + local LAST_OUT=/tmp/tmp.jb5cxBr6MC ++ mktemp + local LAST_ERR=/tmp/tmp.zYnxclnLpY + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec some-name-tls-issue-pxc-0 -- bash -c 'printf '\''%s\n'\'' "exit" | mysql -sN --ssl-ca=/etc/mysql/ssl-internal/ca.crt --ssl-mode=VERIFY_IDENTITY --protocol=tcp -uroot -proot_password --host=some-name-tls-issue-pxc' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.jb5cxBr6MC + cat /tmp/tmp.zYnxclnLpY mysql: [Warning] Using a password on the command line interface can be insecure. + rm /tmp/tmp.jb5cxBr6MC /tmp/tmp.zYnxclnLpY + return 0 + desc 'check ssl-internal certificate using HAProxy' + set +o xtrace ----------------------------------------------------------------------------------- check ssl-internal certificate using HAProxy ----------------------------------------------------------------------------------- + check_verify_identity some-name-tls-issue-haproxy + local host=some-name-tls-issue-haproxy + local command=exit + local 'args=--ssl-ca=/etc/mysql/ssl-internal/ca.crt --ssl-mode=VERIFY_IDENTITY --protocol=tcp -uroot -proot_password --host=some-name-tls-issue-haproxy' + kubectl_bin exec some-name-tls-issue-pxc-0 -- bash -c 'printf '\''%s\n'\'' "exit" | mysql -sN --ssl-ca=/etc/mysql/ssl-internal/ca.crt --ssl-mode=VERIFY_IDENTITY --protocol=tcp -uroot -proot_password --host=some-name-tls-issue-haproxy' ++ mktemp + local LAST_OUT=/tmp/tmp.R1E3RhEn7B ++ mktemp + local LAST_ERR=/tmp/tmp.knNJVmUi2g + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec some-name-tls-issue-pxc-0 -- bash -c 'printf '\''%s\n'\'' "exit" | mysql -sN --ssl-ca=/etc/mysql/ssl-internal/ca.crt --ssl-mode=VERIFY_IDENTITY --protocol=tcp -uroot -proot_password --host=some-name-tls-issue-haproxy' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.R1E3RhEn7B + cat /tmp/tmp.knNJVmUi2g mysql: [Warning] Using a password on the command line interface can be insecure. + rm /tmp/tmp.R1E3RhEn7B /tmp/tmp.knNJVmUi2g + return 0 + desc 'trigger CA rotation and verify leaf cert re-issuance' + set +o xtrace ----------------------------------------------------------------------------------- trigger CA rotation and verify leaf cert re-issuance ----------------------------------------------------------------------------------- + trigger_ca_rotation + desc 'set rotationPolicy=Always on CA certificate' + set +o xtrace ----------------------------------------------------------------------------------- set rotationPolicy=Always on CA certificate ----------------------------------------------------------------------------------- + kubectl_bin patch certificate some-name-tls-issue-ca-cert --type=merge -p '{"spec":{"privateKey":{"rotationPolicy":"Always"}}}' ++ mktemp + local LAST_OUT=/tmp/tmp.a2vvJlY4Ss ++ mktemp + local LAST_ERR=/tmp/tmp.U9ZKJGec3S + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl patch certificate some-name-tls-issue-ca-cert --type=merge -p '{"spec":{"privateKey":{"rotationPolicy":"Always"}}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.a2vvJlY4Ss certificate.cert-manager.io/some-name-tls-issue-ca-cert patched + cat /tmp/tmp.U9ZKJGec3S + rm /tmp/tmp.a2vvJlY4Ss /tmp/tmp.U9ZKJGec3S + return 0 + desc 'capture current CA fingerprint' + set +o xtrace ----------------------------------------------------------------------------------- capture current CA fingerprint ----------------------------------------------------------------------------------- + local old_ca_md5 ++ kubectl_bin get secret some-name-tls-issue-ca-cert -o 'jsonpath={.data.tls\.crt}' ++ base64 -d ++ openssl x509 -noout -fingerprint -sha256 +++ mktemp ++ local LAST_OUT=/tmp/tmp.yI0FpFgRUu +++ mktemp ++ local LAST_ERR=/tmp/tmp.woMxo5Er2M ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get secret some-name-tls-issue-ca-cert -o 'jsonpath={.data.tls\.crt}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.yI0FpFgRUu ++ cat /tmp/tmp.woMxo5Er2M ++ rm /tmp/tmp.yI0FpFgRUu /tmp/tmp.woMxo5Er2M ++ return 0 + old_ca_md5='sha256 Fingerprint=D5:34:82:CE:0B:02:7C:9B:14:9C:D5:A7:EB:30:C8:20:90:28:A7:9D:BA:5F:F7:39:B7:1C:D2:1B:30:0E:65:23' + echo 'old CA: sha256 Fingerprint=D5:34:82:CE:0B:02:7C:9B:14:9C:D5:A7:EB:30:C8:20:90:28:A7:9D:BA:5F:F7:39:B7:1C:D2:1B:30:0E:65:23' old CA: sha256 Fingerprint=D5:34:82:CE:0B:02:7C:9B:14:9C:D5:A7:EB:30:C8:20:90:28:A7:9D:BA:5F:F7:39:B7:1C:D2:1B:30:0E:65:23 + desc 'trigger CA renewal via status condition patch' + set +o xtrace ----------------------------------------------------------------------------------- trigger CA renewal via status condition patch ----------------------------------------------------------------------------------- + local now ++ date -u +%Y-%m-%dT%H:%M:%SZ + now=2026-10-05T14:32:21Z + kubectl_bin get certificate some-name-tls-issue-ca-cert -o json + kubectl_bin replace --subresource=status -f - ++ mktemp + jq --arg now 2026-10-05T14:32:21Z '.status.conditions = [{"type":"Issuing","status":"True","reason":"ManuallyTriggered","message":"Certificate renewal triggered","lastTransitionTime":$now}]' + local LAST_OUT=/tmp/tmp.qsWchDp1zO ++ mktemp ++ mktemp + local LAST_ERR=/tmp/tmp.fyMvI3VcD7 + local exit_status=0 ++ seq 0 2 + local LAST_OUT=/tmp/tmp.vAwmzdqa6B + for i in $(seq 0 2) + set +e + kubectl get certificate some-name-tls-issue-ca-cert -o json ++ mktemp + local LAST_ERR=/tmp/tmp.xsAmPqJmun + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl replace --subresource=status -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.qsWchDp1zO + cat /tmp/tmp.fyMvI3VcD7 + rm /tmp/tmp.qsWchDp1zO /tmp/tmp.fyMvI3VcD7 + return 0 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.vAwmzdqa6B certificate.cert-manager.io/some-name-tls-issue-ca-cert replaced + cat /tmp/tmp.xsAmPqJmun + rm /tmp/tmp.vAwmzdqa6B /tmp/tmp.xsAmPqJmun + return 0 + desc 'wait for cert-manager to issue new CA' + set +o xtrace ----------------------------------------------------------------------------------- wait for cert-manager to issue new CA ----------------------------------------------------------------------------------- + local retries=30 + local new_ca_md5= ++ seq 1 30 + for _ in $(seq 1 $retries) ++ kubectl_bin get secret some-name-tls-issue-ca-cert -o 'jsonpath={.data.tls\.crt}' ++ openssl x509 -noout -fingerprint -sha256 ++ base64 -d +++ mktemp ++ local LAST_OUT=/tmp/tmp.52lv3RB1Aq +++ mktemp ++ local LAST_ERR=/tmp/tmp.dgTqLt38Mm ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get secret some-name-tls-issue-ca-cert -o 'jsonpath={.data.tls\.crt}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.52lv3RB1Aq ++ cat /tmp/tmp.dgTqLt38Mm ++ rm /tmp/tmp.52lv3RB1Aq /tmp/tmp.dgTqLt38Mm ++ return 0 + new_ca_md5='sha256 Fingerprint=53:29:E2:96:3B:35:AF:FB:93:F8:70:DE:6F:85:3E:AE:D5:B6:9C:83:80:E0:73:15:BC:87:AA:34:11:BC:B5:52' + '[' 'sha256 Fingerprint=53:29:E2:96:3B:35:AF:FB:93:F8:70:DE:6F:85:3E:AE:D5:B6:9C:83:80:E0:73:15:BC:87:AA:34:11:BC:B5:52' '!=' 'sha256 Fingerprint=D5:34:82:CE:0B:02:7C:9B:14:9C:D5:A7:EB:30:C8:20:90:28:A7:9D:BA:5F:F7:39:B7:1C:D2:1B:30:0E:65:23' ']' + echo 'new CA: sha256 Fingerprint=53:29:E2:96:3B:35:AF:FB:93:F8:70:DE:6F:85:3E:AE:D5:B6:9C:83:80:E0:73:15:BC:87:AA:34:11:BC:B5:52' new CA: sha256 Fingerprint=53:29:E2:96:3B:35:AF:FB:93:F8:70:DE:6F:85:3E:AE:D5:B6:9C:83:80:E0:73:15:BC:87:AA:34:11:BC:B5:52 + break + '[' 'sha256 Fingerprint=53:29:E2:96:3B:35:AF:FB:93:F8:70:DE:6F:85:3E:AE:D5:B6:9C:83:80:E0:73:15:BC:87:AA:34:11:BC:B5:52' == 'sha256 Fingerprint=D5:34:82:CE:0B:02:7C:9B:14:9C:D5:A7:EB:30:C8:20:90:28:A7:9D:BA:5F:F7:39:B7:1C:D2:1B:30:0E:65:23' ']' + desc 'trigger operator reconcile to detect CA mismatch' + set +o xtrace ----------------------------------------------------------------------------------- trigger operator reconcile to detect CA mismatch ----------------------------------------------------------------------------------- ++ date -u +%Y-%m-%dT%H:%M:%SZ + kubectl_bin annotate pxc some-name-tls-issue percona.com/ca-rotated-at=2026-10-05T14:32:24Z --overwrite ++ mktemp + local LAST_OUT=/tmp/tmp.zjmwSCP2Uu ++ mktemp + local LAST_ERR=/tmp/tmp.yOsp0n4CXT + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl annotate pxc some-name-tls-issue percona.com/ca-rotated-at=2026-10-05T14:32:24Z --overwrite + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.zjmwSCP2Uu perconaxtradbcluster.pxc.percona.com/some-name-tls-issue annotated + cat /tmp/tmp.yOsp0n4CXT + rm /tmp/tmp.zjmwSCP2Uu /tmp/tmp.yOsp0n4CXT + return 0 + desc 'wait for operator to re-issue leaf certs with new CA' + set +o xtrace ----------------------------------------------------------------------------------- wait for operator to re-issue leaf certs with new CA ----------------------------------------------------------------------------------- + local leaf_retries=60 + local leaf_ca_md5= + local leaf_internal_ca_md5= ++ seq 1 60 + for _ in $(seq 1 $leaf_retries) ++ kubectl_bin get secret some-name-tls-issue-ssl -o 'jsonpath={.data.ca\.crt}' ++ base64 -d ++ openssl x509 -noout -fingerprint -sha256 + leaf_ca_md5='sha256 Fingerprint=53:29:E2:96:3B:35:AF:FB:93:F8:70:DE:6F:85:3E:AE:D5:B6:9C:83:80:E0:73:15:BC:87:AA:34:11:BC:B5:52' ++ kubectl_bin get secret some-name-tls-issue-ssl-internal -o 'jsonpath={.data.ca\.crt}' ++ base64 -d ++ openssl x509 -noout -fingerprint -sha256 + leaf_internal_ca_md5='sha256 Fingerprint=53:29:E2:96:3B:35:AF:FB:93:F8:70:DE:6F:85:3E:AE:D5:B6:9C:83:80:E0:73:15:BC:87:AA:34:11:BC:B5:52' + '[' 'sha256 Fingerprint=53:29:E2:96:3B:35:AF:FB:93:F8:70:DE:6F:85:3E:AE:D5:B6:9C:83:80:E0:73:15:BC:87:AA:34:11:BC:B5:52' == 'sha256 Fingerprint=53:29:E2:96:3B:35:AF:FB:93:F8:70:DE:6F:85:3E:AE:D5:B6:9C:83:80:E0:73:15:BC:87:AA:34:11:BC:B5:52' ']' + '[' 'sha256 Fingerprint=53:29:E2:96:3B:35:AF:FB:93:F8:70:DE:6F:85:3E:AE:D5:B6:9C:83:80:E0:73:15:BC:87:AA:34:11:BC:B5:52' == 'sha256 Fingerprint=53:29:E2:96:3B:35:AF:FB:93:F8:70:DE:6F:85:3E:AE:D5:B6:9C:83:80:E0:73:15:BC:87:AA:34:11:BC:B5:52' ']' + break + desc 'verify leaf secrets have new CA' + set +o xtrace ----------------------------------------------------------------------------------- verify leaf secrets have new CA ----------------------------------------------------------------------------------- + '[' 'sha256 Fingerprint=53:29:E2:96:3B:35:AF:FB:93:F8:70:DE:6F:85:3E:AE:D5:B6:9C:83:80:E0:73:15:BC:87:AA:34:11:BC:B5:52' '!=' 'sha256 Fingerprint=53:29:E2:96:3B:35:AF:FB:93:F8:70:DE:6F:85:3E:AE:D5:B6:9C:83:80:E0:73:15:BC:87:AA:34:11:BC:B5:52' ']' + '[' 'sha256 Fingerprint=53:29:E2:96:3B:35:AF:FB:93:F8:70:DE:6F:85:3E:AE:D5:B6:9C:83:80:E0:73:15:BC:87:AA:34:11:BC:B5:52' '!=' 'sha256 Fingerprint=53:29:E2:96:3B:35:AF:FB:93:F8:70:DE:6F:85:3E:AE:D5:B6:9C:83:80:E0:73:15:BC:87:AA:34:11:BC:B5:52' ']' + echo 'leaf certs re-issued with new CA' leaf certs re-issued with new CA + desc 'restart all PXC and HAProxy pods to pick up new certs simultaneously' + set +o xtrace ----------------------------------------------------------------------------------- restart all PXC and HAProxy pods to pick up new certs simultaneously ----------------------------------------------------------------------------------- + sleep 30 + kubectl_bin delete pods -l app.kubernetes.io/instance=some-name-tls-issue,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --force --grace-period=0 ++ mktemp + local LAST_OUT=/tmp/tmp.HbnhertE4W ++ mktemp + local LAST_ERR=/tmp/tmp.2pPGtREr3w + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete pods -l app.kubernetes.io/instance=some-name-tls-issue,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --force --grace-period=0 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.HbnhertE4W pod "some-name-tls-issue-haproxy-0" force deleted from tls-issue-cert-manager-14810 namespace pod "some-name-tls-issue-haproxy-1" force deleted from tls-issue-cert-manager-14810 namespace pod "some-name-tls-issue-pxc-0" force deleted from tls-issue-cert-manager-14810 namespace pod "some-name-tls-issue-pxc-1" force deleted from tls-issue-cert-manager-14810 namespace pod "some-name-tls-issue-pxc-2" force deleted from tls-issue-cert-manager-14810 namespace + cat /tmp/tmp.2pPGtREr3w Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. + rm /tmp/tmp.HbnhertE4W /tmp/tmp.2pPGtREr3w + return 0 + desc 'wait for cluster to recover after full restart' + set +o xtrace ----------------------------------------------------------------------------------- wait for cluster to recover after full restart ----------------------------------------------------------------------------------- + wait_cluster_consistency some-name-tls-issue 3 2 + local cluster_name=some-name-tls-issue + local pxc_size=3 + local proxy_size=2 + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local max_seconds=1500 + local delay=5 + local retries=300 + local pxc_ready + local proxy_ready + sleep 7 + echo -n 'waiting for pxc/some-name-tls-issue to be ready' waiting for pxc/some-name-tls-issue to be ready++ seq 1 300 + for _ in $(seq 1 "${retries}") ++ kubectl_bin get pxc some-name-tls-issue -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.9XbmKlGrfI +++ mktemp ++ local LAST_ERR=/tmp/tmp.QJ8qmBQAZO ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc some-name-tls-issue -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.9XbmKlGrfI ++ cat /tmp/tmp.QJ8qmBQAZO ++ rm /tmp/tmp.9XbmKlGrfI /tmp/tmp.QJ8qmBQAZO ++ return 0 + state=initializing ++ kubectl_bin get pxc some-name-tls-issue -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.mN5sZbu46L +++ mktemp ++ local LAST_ERR=/tmp/tmp.QdmxDkV9KU ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc some-name-tls-issue -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.mN5sZbu46L ++ cat /tmp/tmp.QdmxDkV9KU ++ rm /tmp/tmp.mN5sZbu46L /tmp/tmp.QdmxDkV9KU ++ return 0 + pxc_ready= + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine some-name-tls-issue +++ local cluster_name=some-name-tls-issue ++++ get_proxy some-name-tls-issue ++++ local target_cluster=some-name-tls-issue +++++ kubectl_bin get pxc some-name-tls-issue -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.8vcZX5MBjD ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.RDiTCfOlMH +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get pxc some-name-tls-issue -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.8vcZX5MBjD +++++ cat /tmp/tmp.RDiTCfOlMH +++++ rm /tmp/tmp.8vcZX5MBjD /tmp/tmp.RDiTCfOlMH +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo some-name-tls-issue-haproxy ++++ return +++ local cluster_proxy=some-name-tls-issue-haproxy +++ echo haproxy ++ kubectl_bin get pxc some-name-tls-issue -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.H2YS4kynnd +++ mktemp ++ local LAST_ERR=/tmp/tmp.y24yh9vCrd ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc some-name-tls-issue -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.H2YS4kynnd ++ cat /tmp/tmp.y24yh9vCrd ++ rm /tmp/tmp.H2YS4kynnd /tmp/tmp.y24yh9vCrd ++ return 0 + proxy_ready= + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in $(seq 1 "${retries}") ++ kubectl_bin get pxc some-name-tls-issue -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.uMy25mfxIh +++ mktemp ++ local LAST_ERR=/tmp/tmp.efHzgUiyhw ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc some-name-tls-issue -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.uMy25mfxIh ++ cat /tmp/tmp.efHzgUiyhw ++ rm /tmp/tmp.uMy25mfxIh /tmp/tmp.efHzgUiyhw ++ return 0 + state=initializing ++ kubectl_bin get pxc some-name-tls-issue -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.V0I71eYstv +++ mktemp ++ local LAST_ERR=/tmp/tmp.x8cifZEr5k ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc some-name-tls-issue -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.V0I71eYstv ++ cat /tmp/tmp.x8cifZEr5k ++ rm /tmp/tmp.V0I71eYstv /tmp/tmp.x8cifZEr5k ++ return 0 + pxc_ready= + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine some-name-tls-issue +++ local cluster_name=some-name-tls-issue ++++ get_proxy some-name-tls-issue ++++ local target_cluster=some-name-tls-issue +++++ kubectl_bin get pxc some-name-tls-issue -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.8Fd7MHOxHQ ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.ZCwZCtrB63 +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get pxc some-name-tls-issue -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.8Fd7MHOxHQ +++++ cat /tmp/tmp.ZCwZCtrB63 +++++ rm /tmp/tmp.8Fd7MHOxHQ /tmp/tmp.ZCwZCtrB63 +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo some-name-tls-issue-haproxy ++++ return +++ local cluster_proxy=some-name-tls-issue-haproxy +++ echo haproxy ++ kubectl_bin get pxc some-name-tls-issue -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Xv4pI2rssK +++ mktemp ++ local LAST_ERR=/tmp/tmp.lMTKvid3El ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc some-name-tls-issue -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Xv4pI2rssK ++ cat /tmp/tmp.lMTKvid3El ++ rm /tmp/tmp.Xv4pI2rssK /tmp/tmp.lMTKvid3El ++ return 0 + proxy_ready= + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in $(seq 1 "${retries}") ++ kubectl_bin get pxc some-name-tls-issue -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.a6nocHUj7m +++ mktemp ++ local LAST_ERR=/tmp/tmp.5YBwz7AC4A ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc some-name-tls-issue -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.a6nocHUj7m ++ cat /tmp/tmp.5YBwz7AC4A ++ rm /tmp/tmp.a6nocHUj7m /tmp/tmp.5YBwz7AC4A ++ return 0 + state=initializing ++ kubectl_bin get pxc some-name-tls-issue -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.U319nnov8D +++ mktemp ++ local LAST_ERR=/tmp/tmp.i9pK533Of1 ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc some-name-tls-issue -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.U319nnov8D ++ cat /tmp/tmp.i9pK533Of1 ++ rm /tmp/tmp.U319nnov8D /tmp/tmp.i9pK533Of1 ++ return 0 + pxc_ready= + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine some-name-tls-issue +++ local cluster_name=some-name-tls-issue ++++ get_proxy some-name-tls-issue ++++ local target_cluster=some-name-tls-issue +++++ kubectl_bin get pxc some-name-tls-issue -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.Yum0NDfWYi ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.Kiqzxtk7ua +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get pxc some-name-tls-issue -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.Yum0NDfWYi +++++ cat /tmp/tmp.Kiqzxtk7ua +++++ rm /tmp/tmp.Yum0NDfWYi /tmp/tmp.Kiqzxtk7ua +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo some-name-tls-issue-haproxy ++++ return +++ local cluster_proxy=some-name-tls-issue-haproxy +++ echo haproxy ++ kubectl_bin get pxc some-name-tls-issue -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.0gq3H7OalE +++ mktemp ++ local LAST_ERR=/tmp/tmp.XVtxRUTOSh ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc some-name-tls-issue -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.0gq3H7OalE ++ cat /tmp/tmp.XVtxRUTOSh ++ rm /tmp/tmp.0gq3H7OalE /tmp/tmp.XVtxRUTOSh ++ return 0 + proxy_ready= + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in $(seq 1 "${retries}") ++ kubectl_bin get pxc some-name-tls-issue -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.SgXySjG8Oc +++ mktemp ++ local LAST_ERR=/tmp/tmp.kkmYAw92Y0 ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc some-name-tls-issue -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.SgXySjG8Oc ++ cat /tmp/tmp.kkmYAw92Y0 ++ rm /tmp/tmp.SgXySjG8Oc /tmp/tmp.kkmYAw92Y0 ++ return 0 + state=initializing ++ kubectl_bin get pxc some-name-tls-issue -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.l5zQyLtbSc +++ mktemp ++ local LAST_ERR=/tmp/tmp.vOo5Eiwrgo ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc some-name-tls-issue -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.l5zQyLtbSc ++ cat /tmp/tmp.vOo5Eiwrgo ++ rm /tmp/tmp.l5zQyLtbSc /tmp/tmp.vOo5Eiwrgo ++ return 0 + pxc_ready= + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine some-name-tls-issue +++ local cluster_name=some-name-tls-issue ++++ get_proxy some-name-tls-issue ++++ local target_cluster=some-name-tls-issue +++++ kubectl_bin get pxc some-name-tls-issue -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.PhCRNhcOc8 ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.fzz7KeJGxh +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get pxc some-name-tls-issue -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.PhCRNhcOc8 +++++ cat /tmp/tmp.fzz7KeJGxh +++++ rm /tmp/tmp.PhCRNhcOc8 /tmp/tmp.fzz7KeJGxh +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo some-name-tls-issue-haproxy ++++ return +++ local cluster_proxy=some-name-tls-issue-haproxy +++ echo haproxy ++ kubectl_bin get pxc some-name-tls-issue -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.q4NP3ujurc +++ mktemp ++ local LAST_ERR=/tmp/tmp.R2RZIMvIYo ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc some-name-tls-issue -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.q4NP3ujurc ++ cat /tmp/tmp.R2RZIMvIYo ++ rm /tmp/tmp.q4NP3ujurc /tmp/tmp.R2RZIMvIYo ++ return 0 + proxy_ready= + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in $(seq 1 "${retries}") ++ kubectl_bin get pxc some-name-tls-issue -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.uIz9I8RHbU +++ mktemp ++ local LAST_ERR=/tmp/tmp.bU7mYsl2Gg ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc some-name-tls-issue -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.uIz9I8RHbU ++ cat /tmp/tmp.bU7mYsl2Gg ++ rm /tmp/tmp.uIz9I8RHbU /tmp/tmp.bU7mYsl2Gg ++ return 0 + state=initializing ++ kubectl_bin get pxc some-name-tls-issue -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ZT4qkmIwOk +++ mktemp ++ local LAST_ERR=/tmp/tmp.llE110L9n9 ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc some-name-tls-issue -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ZT4qkmIwOk ++ cat /tmp/tmp.llE110L9n9 ++ rm /tmp/tmp.ZT4qkmIwOk /tmp/tmp.llE110L9n9 ++ return 0 + pxc_ready= + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine some-name-tls-issue +++ local cluster_name=some-name-tls-issue ++++ get_proxy some-name-tls-issue ++++ local target_cluster=some-name-tls-issue +++++ kubectl_bin get pxc some-name-tls-issue -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.MAe2whSimI ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.RSOMjU1TWL +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get pxc some-name-tls-issue -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.MAe2whSimI +++++ cat /tmp/tmp.RSOMjU1TWL +++++ rm /tmp/tmp.MAe2whSimI /tmp/tmp.RSOMjU1TWL +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo some-name-tls-issue-haproxy ++++ return +++ local cluster_proxy=some-name-tls-issue-haproxy +++ echo haproxy ++ kubectl_bin get pxc some-name-tls-issue -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.hiIvfxkZNG +++ mktemp ++ local LAST_ERR=/tmp/tmp.BD9ASY14lj ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc some-name-tls-issue -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.hiIvfxkZNG ++ cat /tmp/tmp.BD9ASY14lj ++ rm /tmp/tmp.hiIvfxkZNG /tmp/tmp.BD9ASY14lj ++ return 0 + proxy_ready= + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in $(seq 1 "${retries}") ++ kubectl_bin get pxc some-name-tls-issue -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.bUbpaQ1jjb +++ mktemp ++ local LAST_ERR=/tmp/tmp.KuKSuHcuE7 ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc some-name-tls-issue -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.bUbpaQ1jjb ++ cat /tmp/tmp.KuKSuHcuE7 ++ rm /tmp/tmp.bUbpaQ1jjb /tmp/tmp.KuKSuHcuE7 ++ return 0 + state=initializing ++ kubectl_bin get pxc some-name-tls-issue -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.pymYro3tnU +++ mktemp ++ local LAST_ERR=/tmp/tmp.9wi7dkIwq7 ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc some-name-tls-issue -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.pymYro3tnU ++ cat /tmp/tmp.9wi7dkIwq7 ++ rm /tmp/tmp.pymYro3tnU /tmp/tmp.9wi7dkIwq7 ++ return 0 + pxc_ready= + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine some-name-tls-issue +++ local cluster_name=some-name-tls-issue ++++ get_proxy some-name-tls-issue ++++ local target_cluster=some-name-tls-issue +++++ kubectl_bin get pxc some-name-tls-issue -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.50SnVKfEB0 ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.pRYwODXszk +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get pxc some-name-tls-issue -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.50SnVKfEB0 +++++ cat /tmp/tmp.pRYwODXszk +++++ rm /tmp/tmp.50SnVKfEB0 /tmp/tmp.pRYwODXszk +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo some-name-tls-issue-haproxy ++++ return +++ local cluster_proxy=some-name-tls-issue-haproxy +++ echo haproxy ++ kubectl_bin get pxc some-name-tls-issue -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.idY12g81IW +++ mktemp ++ local LAST_ERR=/tmp/tmp.hSO4M2YAQ8 ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc some-name-tls-issue -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.idY12g81IW ++ cat /tmp/tmp.hSO4M2YAQ8 ++ rm /tmp/tmp.idY12g81IW /tmp/tmp.hSO4M2YAQ8 ++ return 0 + proxy_ready= + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in $(seq 1 "${retries}") ++ kubectl_bin get pxc some-name-tls-issue -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.KW9r0OBniH +++ mktemp ++ local LAST_ERR=/tmp/tmp.ktDhqICbAA ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc some-name-tls-issue -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.KW9r0OBniH ++ cat /tmp/tmp.ktDhqICbAA ++ rm /tmp/tmp.KW9r0OBniH /tmp/tmp.ktDhqICbAA ++ return 0 + state=initializing ++ kubectl_bin get pxc some-name-tls-issue -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.J7oEz7qLtG +++ mktemp ++ local LAST_ERR=/tmp/tmp.EOVUHrYyZQ ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc some-name-tls-issue -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.J7oEz7qLtG ++ cat /tmp/tmp.EOVUHrYyZQ ++ rm /tmp/tmp.J7oEz7qLtG /tmp/tmp.EOVUHrYyZQ ++ return 0 + pxc_ready= + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine some-name-tls-issue +++ local cluster_name=some-name-tls-issue ++++ get_proxy some-name-tls-issue ++++ local target_cluster=some-name-tls-issue +++++ kubectl_bin get pxc some-name-tls-issue -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.6xe6xU4dmZ ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.3w3uNuK4Mn +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get pxc some-name-tls-issue -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.6xe6xU4dmZ +++++ cat /tmp/tmp.3w3uNuK4Mn +++++ rm /tmp/tmp.6xe6xU4dmZ /tmp/tmp.3w3uNuK4Mn +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo some-name-tls-issue-haproxy ++++ return +++ local cluster_proxy=some-name-tls-issue-haproxy +++ echo haproxy ++ kubectl_bin get pxc some-name-tls-issue -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.0e6sd0lxrE +++ mktemp ++ local LAST_ERR=/tmp/tmp.yHxplJe7vj ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc some-name-tls-issue -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.0e6sd0lxrE ++ cat /tmp/tmp.yHxplJe7vj ++ rm /tmp/tmp.0e6sd0lxrE /tmp/tmp.yHxplJe7vj ++ return 0 + proxy_ready= + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in $(seq 1 "${retries}") ++ kubectl_bin get pxc some-name-tls-issue -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.JZl8ciJeOQ +++ mktemp ++ local LAST_ERR=/tmp/tmp.21PHHPArvq ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc some-name-tls-issue -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.JZl8ciJeOQ ++ cat /tmp/tmp.21PHHPArvq ++ rm /tmp/tmp.JZl8ciJeOQ /tmp/tmp.21PHHPArvq ++ return 0 + state=initializing ++ kubectl_bin get pxc some-name-tls-issue -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.86WIN1VZ5n +++ mktemp ++ local LAST_ERR=/tmp/tmp.PhxEbrWMJm ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc some-name-tls-issue -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.86WIN1VZ5n ++ cat /tmp/tmp.PhxEbrWMJm ++ rm /tmp/tmp.86WIN1VZ5n /tmp/tmp.PhxEbrWMJm ++ return 0 + pxc_ready= + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine some-name-tls-issue +++ local cluster_name=some-name-tls-issue ++++ get_proxy some-name-tls-issue ++++ local target_cluster=some-name-tls-issue +++++ kubectl_bin get pxc some-name-tls-issue -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.h3JYo4fh16 ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.XScDV95Bri +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get pxc some-name-tls-issue -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.h3JYo4fh16 +++++ cat /tmp/tmp.XScDV95Bri +++++ rm /tmp/tmp.h3JYo4fh16 /tmp/tmp.XScDV95Bri +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo some-name-tls-issue-haproxy ++++ return +++ local cluster_proxy=some-name-tls-issue-haproxy +++ echo haproxy ++ kubectl_bin get pxc some-name-tls-issue -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.UQp37UcoxE +++ mktemp ++ local LAST_ERR=/tmp/tmp.P19fpGJ9Bw ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc some-name-tls-issue -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.UQp37UcoxE ++ cat /tmp/tmp.P19fpGJ9Bw ++ rm /tmp/tmp.UQp37UcoxE /tmp/tmp.P19fpGJ9Bw ++ return 0 + proxy_ready= + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in $(seq 1 "${retries}") ++ kubectl_bin get pxc some-name-tls-issue -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.YXzozpM4eL +++ mktemp ++ local LAST_ERR=/tmp/tmp.5cxtO67auv ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc some-name-tls-issue -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.YXzozpM4eL ++ cat /tmp/tmp.5cxtO67auv ++ rm /tmp/tmp.YXzozpM4eL /tmp/tmp.5cxtO67auv ++ return 0 + state=initializing ++ kubectl_bin get pxc some-name-tls-issue -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.QBFd5fIHty +++ mktemp ++ local LAST_ERR=/tmp/tmp.TGOit6RJhP ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc some-name-tls-issue -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.QBFd5fIHty ++ cat /tmp/tmp.TGOit6RJhP ++ rm /tmp/tmp.QBFd5fIHty /tmp/tmp.TGOit6RJhP ++ return 0 + pxc_ready= + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine some-name-tls-issue +++ local cluster_name=some-name-tls-issue ++++ get_proxy some-name-tls-issue ++++ local target_cluster=some-name-tls-issue +++++ kubectl_bin get pxc some-name-tls-issue -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.MJQoWsZ3YF ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.u9R2QfvJax +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get pxc some-name-tls-issue -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.MJQoWsZ3YF +++++ cat /tmp/tmp.u9R2QfvJax +++++ rm /tmp/tmp.MJQoWsZ3YF /tmp/tmp.u9R2QfvJax +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo some-name-tls-issue-haproxy ++++ return +++ local cluster_proxy=some-name-tls-issue-haproxy +++ echo haproxy ++ kubectl_bin get pxc some-name-tls-issue -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.UMp8YjPw8C +++ mktemp ++ local LAST_ERR=/tmp/tmp.2leZxxO7EI ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc some-name-tls-issue -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.UMp8YjPw8C ++ cat /tmp/tmp.2leZxxO7EI ++ rm /tmp/tmp.UMp8YjPw8C /tmp/tmp.2leZxxO7EI ++ return 0 + proxy_ready= + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in $(seq 1 "${retries}") ++ kubectl_bin get pxc some-name-tls-issue -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.euifCMsXWY +++ mktemp ++ local LAST_ERR=/tmp/tmp.xduiXtEdZk ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc some-name-tls-issue -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.euifCMsXWY ++ cat /tmp/tmp.xduiXtEdZk ++ rm /tmp/tmp.euifCMsXWY /tmp/tmp.xduiXtEdZk ++ return 0 + state=initializing ++ kubectl_bin get pxc some-name-tls-issue -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.pR9h6GNFAF +++ mktemp ++ local LAST_ERR=/tmp/tmp.V2xpzvvakK ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc some-name-tls-issue -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.pR9h6GNFAF ++ cat /tmp/tmp.V2xpzvvakK ++ rm /tmp/tmp.pR9h6GNFAF /tmp/tmp.V2xpzvvakK ++ return 0 + pxc_ready= + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine some-name-tls-issue +++ local cluster_name=some-name-tls-issue ++++ get_proxy some-name-tls-issue ++++ local target_cluster=some-name-tls-issue +++++ kubectl_bin get pxc some-name-tls-issue -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.o7s8yFogzI ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.PABbVPwx3M +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get pxc some-name-tls-issue -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.o7s8yFogzI +++++ cat /tmp/tmp.PABbVPwx3M +++++ rm /tmp/tmp.o7s8yFogzI /tmp/tmp.PABbVPwx3M +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo some-name-tls-issue-haproxy ++++ return +++ local cluster_proxy=some-name-tls-issue-haproxy +++ echo haproxy ++ kubectl_bin get pxc some-name-tls-issue -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Udv8bqUoXv +++ mktemp ++ local LAST_ERR=/tmp/tmp.z6UpY1JNp2 ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc some-name-tls-issue -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Udv8bqUoXv ++ cat /tmp/tmp.z6UpY1JNp2 ++ rm /tmp/tmp.Udv8bqUoXv /tmp/tmp.z6UpY1JNp2 ++ return 0 + proxy_ready= + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in $(seq 1 "${retries}") ++ kubectl_bin get pxc some-name-tls-issue -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.8gbAl0yhYB +++ mktemp ++ local LAST_ERR=/tmp/tmp.Osxcri0uHM ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc some-name-tls-issue -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.8gbAl0yhYB ++ cat /tmp/tmp.Osxcri0uHM ++ rm /tmp/tmp.8gbAl0yhYB /tmp/tmp.Osxcri0uHM ++ return 0 + state=initializing ++ kubectl_bin get pxc some-name-tls-issue -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.KykbA18z7z +++ mktemp ++ local LAST_ERR=/tmp/tmp.2NxHlVC4tA ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc some-name-tls-issue -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.KykbA18z7z ++ cat /tmp/tmp.2NxHlVC4tA ++ rm /tmp/tmp.KykbA18z7z /tmp/tmp.2NxHlVC4tA ++ return 0 + pxc_ready= + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine some-name-tls-issue +++ local cluster_name=some-name-tls-issue ++++ get_proxy some-name-tls-issue ++++ local target_cluster=some-name-tls-issue +++++ kubectl_bin get pxc some-name-tls-issue -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.gEBNgQ9vK5 ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.SvAQpK1HEU +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get pxc some-name-tls-issue -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.gEBNgQ9vK5 +++++ cat /tmp/tmp.SvAQpK1HEU +++++ rm /tmp/tmp.gEBNgQ9vK5 /tmp/tmp.SvAQpK1HEU +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo some-name-tls-issue-haproxy ++++ return +++ local cluster_proxy=some-name-tls-issue-haproxy +++ echo haproxy ++ kubectl_bin get pxc some-name-tls-issue -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.nt6xHUjgv4 +++ mktemp ++ local LAST_ERR=/tmp/tmp.Iq4QFZD44O ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc some-name-tls-issue -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.nt6xHUjgv4 ++ cat /tmp/tmp.Iq4QFZD44O ++ rm /tmp/tmp.nt6xHUjgv4 /tmp/tmp.Iq4QFZD44O ++ return 0 + proxy_ready= + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in $(seq 1 "${retries}") ++ kubectl_bin get pxc some-name-tls-issue -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.f6yzAbTgts +++ mktemp ++ local LAST_ERR=/tmp/tmp.PEkBIGIBr6 ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc some-name-tls-issue -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.f6yzAbTgts ++ cat /tmp/tmp.PEkBIGIBr6 ++ rm /tmp/tmp.f6yzAbTgts /tmp/tmp.PEkBIGIBr6 ++ return 0 + state=initializing ++ kubectl_bin get pxc some-name-tls-issue -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.lyrddQG9WC +++ mktemp ++ local LAST_ERR=/tmp/tmp.ObC7UL5lKw ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc some-name-tls-issue -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.lyrddQG9WC ++ cat /tmp/tmp.ObC7UL5lKw ++ rm /tmp/tmp.lyrddQG9WC /tmp/tmp.ObC7UL5lKw ++ return 0 + pxc_ready= + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine some-name-tls-issue +++ local cluster_name=some-name-tls-issue ++++ get_proxy some-name-tls-issue ++++ local target_cluster=some-name-tls-issue +++++ kubectl_bin get pxc some-name-tls-issue -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.DLr1pwVme9 ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.LMMDiyMGzj +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get pxc some-name-tls-issue -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.DLr1pwVme9 +++++ cat /tmp/tmp.LMMDiyMGzj +++++ rm /tmp/tmp.DLr1pwVme9 /tmp/tmp.LMMDiyMGzj +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo some-name-tls-issue-haproxy ++++ return +++ local cluster_proxy=some-name-tls-issue-haproxy +++ echo haproxy ++ kubectl_bin get pxc some-name-tls-issue -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.vUdgg7KSPR +++ mktemp ++ local LAST_ERR=/tmp/tmp.nxNELERyWN ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc some-name-tls-issue -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.vUdgg7KSPR ++ cat /tmp/tmp.nxNELERyWN ++ rm /tmp/tmp.vUdgg7KSPR /tmp/tmp.nxNELERyWN ++ return 0 + proxy_ready= + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in $(seq 1 "${retries}") ++ kubectl_bin get pxc some-name-tls-issue -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.WrJtgw68DO +++ mktemp ++ local LAST_ERR=/tmp/tmp.UGBUapL1T6 ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc some-name-tls-issue -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.WrJtgw68DO ++ cat /tmp/tmp.UGBUapL1T6 ++ rm /tmp/tmp.WrJtgw68DO /tmp/tmp.UGBUapL1T6 ++ return 0 + state=initializing ++ kubectl_bin get pxc some-name-tls-issue -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.smckYAVTAJ +++ mktemp ++ local LAST_ERR=/tmp/tmp.2PxLhZprH1 ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc some-name-tls-issue -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.smckYAVTAJ ++ cat /tmp/tmp.2PxLhZprH1 ++ rm /tmp/tmp.smckYAVTAJ /tmp/tmp.2PxLhZprH1 ++ return 0 + pxc_ready= + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine some-name-tls-issue +++ local cluster_name=some-name-tls-issue ++++ get_proxy some-name-tls-issue ++++ local target_cluster=some-name-tls-issue +++++ kubectl_bin get pxc some-name-tls-issue -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.ru85OEUnxk ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.2FdaorYXEI +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get pxc some-name-tls-issue -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.ru85OEUnxk +++++ cat /tmp/tmp.2FdaorYXEI +++++ rm /tmp/tmp.ru85OEUnxk /tmp/tmp.2FdaorYXEI +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo some-name-tls-issue-haproxy ++++ return +++ local cluster_proxy=some-name-tls-issue-haproxy +++ echo haproxy ++ kubectl_bin get pxc some-name-tls-issue -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.RJrRXMyNjD +++ mktemp ++ local LAST_ERR=/tmp/tmp.0rv5AxFi1V ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc some-name-tls-issue -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.RJrRXMyNjD ++ cat /tmp/tmp.0rv5AxFi1V ++ rm /tmp/tmp.RJrRXMyNjD /tmp/tmp.0rv5AxFi1V ++ return 0 + proxy_ready= + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in $(seq 1 "${retries}") ++ kubectl_bin get pxc some-name-tls-issue -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.BaTgn5SsCB +++ mktemp ++ local LAST_ERR=/tmp/tmp.4CU6w9zfTK ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc some-name-tls-issue -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.BaTgn5SsCB ++ cat /tmp/tmp.4CU6w9zfTK ++ rm /tmp/tmp.BaTgn5SsCB /tmp/tmp.4CU6w9zfTK ++ return 0 + state=initializing ++ kubectl_bin get pxc some-name-tls-issue -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.DqToT2bF39 +++ mktemp ++ local LAST_ERR=/tmp/tmp.GrLI1ZYRo2 ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc some-name-tls-issue -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.DqToT2bF39 ++ cat /tmp/tmp.GrLI1ZYRo2 ++ rm /tmp/tmp.DqToT2bF39 /tmp/tmp.GrLI1ZYRo2 ++ return 0 + pxc_ready=1 + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine some-name-tls-issue +++ local cluster_name=some-name-tls-issue ++++ get_proxy some-name-tls-issue ++++ local target_cluster=some-name-tls-issue +++++ kubectl_bin get pxc some-name-tls-issue -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.KbmJANN7Cm ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.FfunjCbgcF +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get pxc some-name-tls-issue -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.KbmJANN7Cm +++++ cat /tmp/tmp.FfunjCbgcF +++++ rm /tmp/tmp.KbmJANN7Cm /tmp/tmp.FfunjCbgcF +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo some-name-tls-issue-haproxy ++++ return +++ local cluster_proxy=some-name-tls-issue-haproxy +++ echo haproxy ++ kubectl_bin get pxc some-name-tls-issue -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ZSwiJPrJIk +++ mktemp ++ local LAST_ERR=/tmp/tmp.gt5xpmIzxL ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc some-name-tls-issue -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ZSwiJPrJIk ++ cat /tmp/tmp.gt5xpmIzxL ++ rm /tmp/tmp.ZSwiJPrJIk /tmp/tmp.gt5xpmIzxL ++ return 0 + proxy_ready=1 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in $(seq 1 "${retries}") ++ kubectl_bin get pxc some-name-tls-issue -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.6WCpZX8bai +++ mktemp ++ local LAST_ERR=/tmp/tmp.UOH4JxkLyC ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc some-name-tls-issue -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.6WCpZX8bai ++ cat /tmp/tmp.UOH4JxkLyC ++ rm /tmp/tmp.6WCpZX8bai /tmp/tmp.UOH4JxkLyC ++ return 0 + state=initializing ++ kubectl_bin get pxc some-name-tls-issue -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.eO4p6aAiln +++ mktemp ++ local LAST_ERR=/tmp/tmp.ReEsfhyH7A ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc some-name-tls-issue -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.eO4p6aAiln ++ cat /tmp/tmp.ReEsfhyH7A ++ rm /tmp/tmp.eO4p6aAiln /tmp/tmp.ReEsfhyH7A ++ return 0 + pxc_ready=1 + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine some-name-tls-issue +++ local cluster_name=some-name-tls-issue ++++ get_proxy some-name-tls-issue ++++ local target_cluster=some-name-tls-issue +++++ kubectl_bin get pxc some-name-tls-issue -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.omIP8rBEaQ ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.d9hTVRNnu5 +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get pxc some-name-tls-issue -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.omIP8rBEaQ +++++ cat /tmp/tmp.d9hTVRNnu5 +++++ rm /tmp/tmp.omIP8rBEaQ /tmp/tmp.d9hTVRNnu5 +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo some-name-tls-issue-haproxy ++++ return +++ local cluster_proxy=some-name-tls-issue-haproxy +++ echo haproxy ++ kubectl_bin get pxc some-name-tls-issue -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.0aRuZY5whY +++ mktemp ++ local LAST_ERR=/tmp/tmp.qDXAvdsyaK ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc some-name-tls-issue -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.0aRuZY5whY ++ cat /tmp/tmp.qDXAvdsyaK ++ rm /tmp/tmp.0aRuZY5whY /tmp/tmp.qDXAvdsyaK ++ return 0 + proxy_ready=2 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in $(seq 1 "${retries}") ++ kubectl_bin get pxc some-name-tls-issue -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.uSw0ig79x4 +++ mktemp ++ local LAST_ERR=/tmp/tmp.5FMdjohz3j ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc some-name-tls-issue -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.uSw0ig79x4 ++ cat /tmp/tmp.5FMdjohz3j ++ rm /tmp/tmp.uSw0ig79x4 /tmp/tmp.5FMdjohz3j ++ return 0 + state=initializing ++ kubectl_bin get pxc some-name-tls-issue -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.4poKc0H0hH +++ mktemp ++ local LAST_ERR=/tmp/tmp.3mYaYhi2u7 ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc some-name-tls-issue -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.4poKc0H0hH ++ cat /tmp/tmp.3mYaYhi2u7 ++ rm /tmp/tmp.4poKc0H0hH /tmp/tmp.3mYaYhi2u7 ++ return 0 + pxc_ready=2 + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine some-name-tls-issue +++ local cluster_name=some-name-tls-issue ++++ get_proxy some-name-tls-issue ++++ local target_cluster=some-name-tls-issue +++++ kubectl_bin get pxc some-name-tls-issue -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.EA00KanCJi ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.GcWzBJKgVM +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get pxc some-name-tls-issue -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.EA00KanCJi +++++ cat /tmp/tmp.GcWzBJKgVM +++++ rm /tmp/tmp.EA00KanCJi /tmp/tmp.GcWzBJKgVM +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo some-name-tls-issue-haproxy ++++ return +++ local cluster_proxy=some-name-tls-issue-haproxy +++ echo haproxy ++ kubectl_bin get pxc some-name-tls-issue -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.gx4Hr5fKE7 +++ mktemp ++ local LAST_ERR=/tmp/tmp.RmVHrKw5jl ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc some-name-tls-issue -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.gx4Hr5fKE7 ++ cat /tmp/tmp.RmVHrKw5jl ++ rm /tmp/tmp.gx4Hr5fKE7 /tmp/tmp.RmVHrKw5jl ++ return 0 + proxy_ready=2 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in $(seq 1 "${retries}") ++ kubectl_bin get pxc some-name-tls-issue -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.6AHUeftmAs +++ mktemp ++ local LAST_ERR=/tmp/tmp.8unh66fWMP ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc some-name-tls-issue -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.6AHUeftmAs ++ cat /tmp/tmp.8unh66fWMP ++ rm /tmp/tmp.6AHUeftmAs /tmp/tmp.8unh66fWMP ++ return 0 + state=ready ++ kubectl_bin get pxc some-name-tls-issue -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.OuJPyEBhp0 +++ mktemp ++ local LAST_ERR=/tmp/tmp.KteBfjpF4h ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc some-name-tls-issue -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.OuJPyEBhp0 ++ cat /tmp/tmp.KteBfjpF4h ++ rm /tmp/tmp.OuJPyEBhp0 /tmp/tmp.KteBfjpF4h ++ return 0 + pxc_ready=3 + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine some-name-tls-issue +++ local cluster_name=some-name-tls-issue ++++ get_proxy some-name-tls-issue ++++ local target_cluster=some-name-tls-issue +++++ kubectl_bin get pxc some-name-tls-issue -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.7NaY7NlHHZ ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.7OE99rcDSa +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get pxc some-name-tls-issue -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.7NaY7NlHHZ +++++ cat /tmp/tmp.7OE99rcDSa +++++ rm /tmp/tmp.7NaY7NlHHZ /tmp/tmp.7OE99rcDSa +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo some-name-tls-issue-haproxy ++++ return +++ local cluster_proxy=some-name-tls-issue-haproxy +++ echo haproxy ++ kubectl_bin get pxc some-name-tls-issue -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.vKPLYp1Un7 +++ mktemp ++ local LAST_ERR=/tmp/tmp.lrvNvkRiED ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc some-name-tls-issue -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.vKPLYp1Un7 ++ cat /tmp/tmp.lrvNvkRiED ++ rm /tmp/tmp.vKPLYp1Un7 /tmp/tmp.lrvNvkRiED ++ return 0 + proxy_ready=2 + [[ ready == \r\e\a\d\y ]] + [[ 3 == \3 ]] + [[ 2 == \2 ]] + echo + return + desc 'check ssl-internal certificate using PXC after CA rotation' + set +o xtrace ----------------------------------------------------------------------------------- check ssl-internal certificate using PXC after CA rotation ----------------------------------------------------------------------------------- + check_verify_identity some-name-tls-issue-pxc + local host=some-name-tls-issue-pxc + local command=exit + local 'args=--ssl-ca=/etc/mysql/ssl-internal/ca.crt --ssl-mode=VERIFY_IDENTITY --protocol=tcp -uroot -proot_password --host=some-name-tls-issue-pxc' + kubectl_bin exec some-name-tls-issue-pxc-0 -- bash -c 'printf '\''%s\n'\'' "exit" | mysql -sN --ssl-ca=/etc/mysql/ssl-internal/ca.crt --ssl-mode=VERIFY_IDENTITY --protocol=tcp -uroot -proot_password --host=some-name-tls-issue-pxc' ++ mktemp + local LAST_OUT=/tmp/tmp.CpPzqpnwof ++ mktemp + local LAST_ERR=/tmp/tmp.GiFgxV6MIy + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec some-name-tls-issue-pxc-0 -- bash -c 'printf '\''%s\n'\'' "exit" | mysql -sN --ssl-ca=/etc/mysql/ssl-internal/ca.crt --ssl-mode=VERIFY_IDENTITY --protocol=tcp -uroot -proot_password --host=some-name-tls-issue-pxc' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.CpPzqpnwof + cat /tmp/tmp.GiFgxV6MIy mysql: [Warning] Using a password on the command line interface can be insecure. + rm /tmp/tmp.CpPzqpnwof /tmp/tmp.GiFgxV6MIy + return 0 + desc 'check ssl-internal certificate using HAProxy after CA rotation' + set +o xtrace ----------------------------------------------------------------------------------- check ssl-internal certificate using HAProxy after CA rotation ----------------------------------------------------------------------------------- + check_verify_identity some-name-tls-issue-haproxy + local host=some-name-tls-issue-haproxy + local command=exit + local 'args=--ssl-ca=/etc/mysql/ssl-internal/ca.crt --ssl-mode=VERIFY_IDENTITY --protocol=tcp -uroot -proot_password --host=some-name-tls-issue-haproxy' + kubectl_bin exec some-name-tls-issue-pxc-0 -- bash -c 'printf '\''%s\n'\'' "exit" | mysql -sN --ssl-ca=/etc/mysql/ssl-internal/ca.crt --ssl-mode=VERIFY_IDENTITY --protocol=tcp -uroot -proot_password --host=some-name-tls-issue-haproxy' ++ mktemp + local LAST_OUT=/tmp/tmp.VtBhnseYMK ++ mktemp + local LAST_ERR=/tmp/tmp.UMfAazip6j + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec some-name-tls-issue-pxc-0 -- bash -c 'printf '\''%s\n'\'' "exit" | mysql -sN --ssl-ca=/etc/mysql/ssl-internal/ca.crt --ssl-mode=VERIFY_IDENTITY --protocol=tcp -uroot -proot_password --host=some-name-tls-issue-haproxy' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.VtBhnseYMK + cat /tmp/tmp.UMfAazip6j mysql: [Warning] Using a password on the command line interface can be insecure. + rm /tmp/tmp.VtBhnseYMK /tmp/tmp.UMfAazip6j + return 0 + destroy tls-issue-cert-manager-14810 + local namespace=tls-issue-cert-manager-14810 + local ignore_logs=true + [[ 0 == 1 ]] + desc 'destroy cluster/operator and all other resources' + set +o xtrace ----------------------------------------------------------------------------------- destroy cluster/operator and all other resources ----------------------------------------------------------------------------------- + '[' true == false -o 1 == 1 ']' + grep -v level=info + grep -v 'the object has been modified' + grep -v 'get backup status: Job.batch' + sort -u + tee /tmp/tmp.vlgOWXkFpe/operator.log + /usr/bin/sed -r 's/"ts":[0-9.]+//; s^limits-[0-9.]+/^^g' ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ grep -c percona-xtradb-cluster-operator +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name' ++ head -1 ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.IXSlaD5Pp1 +++ mktemp ++ local LAST_ERR=/tmp/tmp.67851D9Mb7 ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.IXSlaD5Pp1 ++ cat /tmp/tmp.67851D9Mb7 ++ rm /tmp/tmp.IXSlaD5Pp1 /tmp/tmp.67851D9Mb7 ++ return 0 + kubectl_bin logs -n pxc-operator percona-xtradb-cluster-operator-658fcf4b98-4pqtj ++ mktemp + local LAST_OUT=/tmp/tmp.wYFeOR1VEZ ++ mktemp + local LAST_ERR=/tmp/tmp.fDFFcjdsaU + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs -n pxc-operator percona-xtradb-cluster-operator-658fcf4b98-4pqtj + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.wYFeOR1VEZ + cat /tmp/tmp.fDFFcjdsaU + rm /tmp/tmp.wYFeOR1VEZ /tmp/tmp.fDFFcjdsaU + return 0 /go/pkg/mod/k8s.io/apimachinery@v0.37.0/pkg/util/runtime/runtime.go:144 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:201 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494 /go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:382 /go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/tls.go:29 /go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/tls.go:506 /go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/naming/condition.go:17 /usr/local/go/src/runtime/panic.go:336 /usr/local/go/src/runtime/panic.go:860 /usr/local/go/src/runtime/signal_unix.go:931 ContainerPort: 3306, ContainerPort: 33060, ContainerPort: 33062, ContainerPort: 4444, ContainerPort: 4567, ContainerPort: 4568, DefaultUser: nil, DefaultUser: nil, HostIP: "", HostPort: 0, Items: nil, Items: nil, LocalObjectReference: {Name: "auto-some-name-tls-issue-pxc"}, LocalObjectReference: {Name: "some-name-tls-issue-pxc"}, Name: "ist", Name: "mysql", Name: "mysql-admin", Name: "mysqlx", Name: "sst", Name: "write-set", Optional: &true, Optional: &false, Optional: &true, SecretName: "internal-some-name-tls-issue", SecretName: "some-name-tls-issue-mysql-init", SecretName: "some-name-tls-issue-ssl", SecretName: "some-name-tls-issue-ssl-internal", SecretName: "some-name-vault", ... // 16 identical fields ... // 22 identical fields ... // 3 identical fields ... // 9 identical fields AWSElasticBlockStore: nil, AzureFile: nil, ConfigMap: &v1.ConfigMapVolumeSource{ FC: nil, FailureThreshold: 3, GitRepo: nil, ISCSI: nil, InitialDelaySeconds: 300, NFS: nil, ProbeHandler: {Exec: &{Command: {"/var/lib/mysql/liveness-check.sh"}}}, Quobyte: nil, Secret: &v1.SecretVolumeSource{ SuccessThreshold: 1, TerminationGracePeriodSeconds: nil, TimeoutSeconds: 5, VsphereVolume: nil, { }, ... // 16 identical fields ... // 3 identical fields ... // 4 identical fields Args: {"mysqld"}, Env: {{Name: "PXC_SERVICE", Value: "some-name-tls-issue-pxc-unready"}, {Name: "MONITOR_HOST", Value: "%"}, {Name: "MYSQL_ROOT_PASSWORD", ValueFrom: &{SecretKeyRef: &{LocalObjectReference: {Name: "internal-some-name-tls-issue"}, Key: "root"}}}, {Name: "XTRABACKUP_PASSWORD", ValueFrom: &{SecretKeyRef: &{LocalObjectReference: {Name: "internal-some-name-tls-issue"}, Key: "xtrabackup"}}}, ...}, EnvFrom: {{SecretRef: &{LocalObjectReference: {Name: "some-name-tls-issue-env-vars-pxc"}, Optional: &true}}}, ImagePullPolicy: "Always", Lifecycle: nil, LivenessProbe: &v1.Probe{ Name: "auto-config", Name: "config", Name: "mysql-init-file", Name: "mysql-users-secret-file", Name: "ssl", Name: "ssl-internal", Name: "vault-keyring-secret", Ports: []v1.ContainerPort{ ReadinessProbe: &{ProbeHandler: {Exec: &{Command: {"/var/lib/mysql/readiness-check.sh"}}}, InitialDelaySeconds: 15, TimeoutSeconds: 15, PeriodSeconds: 30, ...}, SecurityContext: nil, StartupProbe: nil, VolumeDevices: nil, VolumeMounts: {{Name: "datadir", MountPath: "/var/lib/mysql"}, {Name: "config", MountPath: "/etc/percona-xtradb-cluster.conf.d"}, {Name: "tmp", MountPath: "/tmp"}, {Name: "ssl", MountPath: "/etc/mysql/ssl"}, ...}, VolumeSource: v1.VolumeSource{ WorkingDir: "", }, "kubectl.kubernetes.io/default-container": "pxc", "percona.com/configuration-hash": "d41d8cd98f00b204e9800998ecf8427e", "percona.com/ssl-hash": strings.Join({ "percona.com/ssl-internal-hash": strings.Join({ ... // 3 identical fields ... // 6 identical fields AccessModes: nil, Capacity: nil, DataSource: nil, DataSourceRef: nil, StorageClassName: nil, VolumeAttributesClassName: nil, VolumeName: "", { {Name: "bin", VolumeSource: {EmptyDir: &{}}}, {Name: "tmp", VolumeSource: {EmptyDir: &{}}}, }, }, ""), ... // 6 identical fields ... // 7 identical fields ... // 9 identical fields ActiveDeadlineSeconds: nil, Affinity: &{PodAntiAffinity: &{RequiredDuringSchedulingIgnoredDuringExecution: {{LabelSelector: &{MatchLabels: {"app.kubernetes.io/component": "pxc", "app.kubernetes.io/instance": "some-name-tls-issue", "app.kubernetes.io/managed-by": "percona-xtradb-cluster-operator", "app.kubernetes.io/name": "percona-xtradb-cluster", ...}}, TopologyKey: "kubernetes.io/hostname"}}}}, Annotations: map[string]string{ AutomountServiceAccountToken: nil, Containers: []v1.Container{ DeletionGracePeriodSeconds: nil, EphemeralContainers: nil, Finalizers: nil, HostAliases: nil, InitContainers: []v1.Container{ Labels: {"app.kubernetes.io/component": "pxc", "app.kubernetes.io/instance": "some-name-tls-issue", "app.kubernetes.io/managed-by": "percona-xtradb-cluster-operator", "app.kubernetes.io/name": "percona-xtradb-cluster", ...}, ManagedFields: nil, NodeName: "", NodeSelector: nil, OS: nil, ObjectMeta: {Name: "datadir", Labels: {"app.kubernetes.io/component": "pxc", "app.kubernetes.io/instance": "some-name-tls-issue", "app.kubernetes.io/managed-by": "percona-xtradb-cluster-operator", "app.kubernetes.io/name": "percona-xtradb-cluster", ...}}, Overhead: nil, OwnerReferences: nil, PreemptionPolicy: nil, ServiceAccountName: "default", SetHostnameAsFQDN: nil, Spec: v1.PersistentVolumeClaimSpec{ Status: v1.PersistentVolumeClaimStatus{ Subdomain: "", TerminationGracePeriodSeconds: &600, Tolerations: nil, TypeMeta: {}, Volumes: []v1.Volume{ }, ObjectMeta: v1.ObjectMeta{ Spec: v1.PodSpec{ { }, ... // 2 identical fields Conditions: nil, DeletionGracePeriodSeconds: nil, DeletionTimestamp: nil, Finalizers: nil, Labels: nil, MinReadySeconds: 0, Namespace: "tls-issue-cert-manager-14810", Ordinals: nil, OwnerReferences: {{APIVersion: "pxc.percona.com/v1", Kind: "PerconaXtraDBCluster", Name: "some-name-tls-issue", UID: "3d35ed7c-8f7e-4d77-b423-0043a8d5fcb4", ...}}, Replicas: &3, Selector: &{MatchLabels: {"app.kubernetes.io/component": "pxc", "app.kubernetes.io/instance": "some-name-tls-issue", "app.kubernetes.io/managed-by": "percona-xtradb-cluster-operator", "app.kubernetes.io/name": "percona-xtradb-cluster", ...}}, SelfLink: "", ServiceName: "some-name-tls-issue-pxc", Template: v1.PodTemplateSpec{ UpdateStrategy: {Type: "RollingUpdate", RollingUpdate: &{Partition: &0}}, VolumeClaimTemplates: []v1.PersistentVolumeClaim{ }, ObjectMeta: v1.ObjectMeta{ Spec: v1.StatefulSetSpec{ Status: v1.StatefulSetStatus{ TypeMeta: {Kind: "StatefulSet", APIVersion: "apps/v1"}, }, &v1.StatefulSet{ } + DefaultMode: nil, + DefaultMode: nil, + Protocol: "", + PeriodSeconds: 0, + "8f476761d03cfe15c3a753b9039284ed", + "d0480fd7b82faf6066455d4abb43ae9f", + TerminationMessagePath: "", + TerminationMessagePolicy: "", + Phase: "", + VolumeMode: nil, + DNSPolicy: "", + DeprecatedServiceAccount: "", + RestartPolicy: "", + SchedulerName: "", + TopologySpreadConstraints: []v1.TopologySpreadConstraint{}, + "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Mywic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJzb21lLW5hbWUtdGxzLWlzc3VlIiwiYXBwLmt1YmVybmV0ZXMuaW8vbWFuYWdlZC1ieSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3IiLCJhcHAua3ViZXJuZXRlcy5pby9uYW1lIjoicGVyY29uYS14dHJhZGItY2x1c3RlciIsImFwcC5rdWJlcm5ldGVzLmlvL3BhcnQtb2YiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIn19LCJ0ZW1wbGF0ZSI6eyJtZXRhZGF0YSI6eyJsYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJzb21lLW5hbWUtdGxzLWlzc3VlIiwiYXBwLmt1YmVybmV0ZXMuaW8vbWFuYWdlZC1ieSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3IiLCJhcHAua3ViZXJuZXRlcy5pby9uYW1lIjoicGVyY29uYS14dHJhZGItY2x1c3RlciIsImFwcC5rdWJlcm5ldGVzLmlvL3BhcnQtb2YiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIn0sImFubm90YXRpb25zIjp7Imt1YmVjdGwua3ViZXJuZXRlcy5pby9kZWZhdWx0LWNvbnRhaW5lciI6InB4YyIsInBlcmNvbmEuY29tL2NvbmZpZ3VyYXRpb24taGFzaCI6ImQ0MWQ4Y2Q5OGYwMGIyMDRlOTgwMDk5OGVjZjg0MjdlIiwicGVyY29uYS5jb20vc3NsLWhhc2giOiI4ZjQ3Njc2MWQwM2NmZTE1YzNhNzUzYjkw"..., + Annotations: map[string]string{ + AvailableReplicas: 0, + CollisionCount: nil, + CreationTimestamp: v1.Time{}, + CurrentReplicas: 0, + CurrentRevision: "", + Generation: 0, + ManagedFields: nil, + ObservedGeneration: 0, + PersistentVolumeClaimRetentionPolicy: nil, + PodManagementPolicy: "", + ReadyReplicas: 0, + Replicas: 0, + ResourceVersion: "", + RevisionHistoryLimit: nil, + UID: "", + UpdateRevision: "", + UpdatedReplicas: 0, + }, - DefaultMode: &420, - DefaultMode: &420, - Protocol: "TCP", - PeriodSeconds: 10, - "62026f05886912f447e3a39c9ce37ca3", - "64b3c3a46b29866fd7723b84956c6179", - TerminationMessagePath: "/dev/termination-log", - TerminationMessagePolicy: "File", - Phase: "Pending", - VolumeMode: &"Filesystem", - APIVersion: "apps/v1", - APIVersion: "apps/v1", - DNSPolicy: "ClusterFirst", - DeprecatedServiceAccount: "default", - FieldsType: "FieldsV1", - FieldsType: "FieldsV1", - FieldsV1: s`{"f:status":{"f:availableReplicas":{},"f:collisionCount":{},"f:c`..., - FieldsV1: s`{"f:metadata":{"f:annotations":{".":{},"f:percona.com/last-confi`..., - Manager: "kube-controller-manager", - Manager: "percona-xtradb-cluster-operator", - Operation: "Update", - Operation: "Update", - RestartPolicy: "Always", - SchedulerName: "default-scheduler", - Subresource: "status", - Time: s"2026-10-05 14:31:52 +0000 UTC", - Time: s"2026-10-05 14:28:21 +0000 UTC", - TopologySpreadConstraints: nil, - "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Mywic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJzb21lLW5hbWUtdGxzLWlzc3VlIiwiYXBwLmt1YmVybmV0ZXMuaW8vbWFuYWdlZC1ieSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3IiLCJhcHAua3ViZXJuZXRlcy5pby9uYW1lIjoicGVyY29uYS14dHJhZGItY2x1c3RlciIsImFwcC5rdWJlcm5ldGVzLmlvL3BhcnQtb2YiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIn19LCJ0ZW1wbGF0ZSI6eyJtZXRhZGF0YSI6eyJsYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJzb21lLW5hbWUtdGxzLWlzc3VlIiwiYXBwLmt1YmVybmV0ZXMuaW8vbWFuYWdlZC1ieSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3IiLCJhcHAua3ViZXJuZXRlcy5pby9uYW1lIjoicGVyY29uYS14dHJhZGItY2x1c3RlciIsImFwcC5rdWJlcm5ldGVzLmlvL3BhcnQtb2YiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIn0sImFubm90YXRpb25zIjp7Imt1YmVjdGwua3ViZXJuZXRlcy5pby9kZWZhdWx0LWNvbnRhaW5lciI6InB4YyIsInBlcmNvbmEuY29tL2NvbmZpZ3VyYXRpb24taGFzaCI6ImQ0MWQ4Y2Q5OGYwMGIyMDRlOTgwMDk5OGVjZjg0MjdlIiwicGVyY29uYS5jb20vc3NsLWhhc2giOiI2MjAyNmYwNTg4NjkxMmY0NDdlM2EzOWM5"..., - { - }, - Annotations: map[string]string{ - AvailableReplicas: 3, - CollisionCount: &0, - CreationTimestamp: v1.Time{Time: s"2026-10-05 14:28:21 +0000 UTC"}, - CurrentReplicas: 3, - CurrentRevision: "some-name-tls-issue-pxc-5f44b9ff64", - Generation: 1, - ManagedFields: []v1.ManagedFieldsEntry{ - ObservedGeneration: 1, - PersistentVolumeClaimRetentionPolicy: s"&StatefulSetPersistentVolumeClaimRetentionPolicy{WhenDeleted:Retain,WhenScaled:Retain,}", - PodManagementPolicy: "OrderedReady", - ReadyReplicas: 3, - Replicas: 3, - ResourceVersion: "1791210712108511001", - RevisionHistoryLimit: &10, - UID: "16190017-a00a-498b-b1c9-5ebdac9a6261", - UpdateRevision: "some-name-tls-issue-pxc-5f44b9ff64", - UpdatedReplicas: 3, - }, 2026-10-05T14:26:47.691Z INFO setup Manager starting up {"gitCommit": "caf25420ee3d0d26f55c22c9a7661405b27337d1", "gitBranch": "PR-2546-caf25420", "buildTime": "2026-10-05T11:36:48Z", "goVersion": "go1.26.8", "os": "linux", "arch": "amd64"} 2026-10-05T14:26:47.691Z INFO setup Runs on {"platform": "kubernetes", "version": "v1.34.9-gke.1655001"} 2026-10-05T14:26:47.692Z INFO setup Feature gates {"PXCO_FEATURE_GATES": "", "enabled": ""} 2026-10-05T14:26:47.695Z INFO setup Registering Components. 2026-10-05T14:26:48.901Z INFO controller-runtime.webhook Registering webhook {"path": "/validate-percona-xtradbcluster"} 2026-10-05T14:26:48.902Z INFO controller-runtime.certwatcher Starting certificate poll+watcher {"cert": "/tmp/k8s-webhook-server/serving-certs/tls.crt", "key": "/tmp/k8s-webhook-server/serving-certs/tls.key", "interval": "10s"} 2026-10-05T14:26:48.902Z INFO controller-runtime.certwatcher Updated current TLS certificate {"cert": "/tmp/k8s-webhook-server/serving-certs/tls.crt", "key": "/tmp/k8s-webhook-server/serving-certs/tls.key"} 2026-10-05T14:26:48.902Z INFO controller-runtime.metrics Serving metrics server {"bindAddress": ":8080", "secure": false} 2026-10-05T14:26:48.902Z INFO controller-runtime.metrics Starting metrics server 2026-10-05T14:26:48.902Z INFO controller-runtime.webhook Serving webhook server {"host": "", "port": 9443} 2026-10-05T14:26:48.902Z INFO controller-runtime.webhook Starting webhook server 2026-10-05T14:26:48.902Z INFO setup Starting the Cmd. 2026-10-05T14:26:48.902Z INFO starting server {"name": "health probe", "addr": "[::]:8081"} 2026-10-05T14:26:49.003Z INFO Attempting to acquire leader lease... {"lock": "pxc-operator/08db1feb.percona.com"} 2026-10-05T14:26:49.049Z INFO Successfully acquired lease {"lock": "pxc-operator/08db1feb.percona.com"} 2026-10-05T14:26:49.050Z DEBUG events percona-xtradb-cluster-operator-658fcf4b98-4pqtj_d077aa24-8673-4c33-b3bd-bc6e1a4df71f became leader {"type": "Normal", "object": {"kind":"Lease","namespace":"pxc-operator","name":"08db1feb.percona.com","uid":"e545fe62-5232-417e-b888-cc8a84f0a472","apiVersion":"coordination.k8s.io/v1","resourceVersion":"1791210409042207009"}, "reason": "LeaderElection"} 2026-10-05T14:26:49.050Z INFO Starting EventSource {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "source": "kind source: *v1.PerconaXtraDBCluster"} 2026-10-05T14:26:49.050Z INFO Starting EventSource {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "source": "kind source: *v1.Secret"} 2026-10-05T14:26:49.050Z INFO Starting EventSource {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "source": "kind source: *v1.PerconaXtraDBClusterBackup"} 2026-10-05T14:26:49.050Z INFO Starting EventSource {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "source": "kind source: *v1.PerconaXtraDBClusterRestore"} 2026-10-05T14:26:49.151Z INFO Starting Controller {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup"} 2026-10-05T14:26:49.151Z INFO Starting workers {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "worker count": 1} 2026-10-05T14:26:49.251Z INFO Starting Controller {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster"} 2026-10-05T14:26:49.251Z INFO Starting Controller {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore"} 2026-10-05T14:26:49.251Z INFO Starting workers {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "worker count": 1} 2026-10-05T14:26:49.251Z INFO Starting workers {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "worker count": 1} 2026-10-05T14:28:14.847Z INFO Set CR version {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name-tls-issue","namespace":"tls-issue-cert-manager-14810"}, "namespace": "tls-issue-cert-manager-14810", "name": "some-name-tls-issue", "reconcileID": "8fd0ac1c-80f3-4cbc-a067-89c4a81e1a96", "version": "1.21.0"} 2026-10-05T14:28:21.361Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name-tls-issue","namespace":"tls-issue-cert-manager-14810"}, "namespace": "tls-issue-cert-manager-14810", "name": "some-name-tls-issue", "reconcileID": "8fd0ac1c-80f3-4cbc-a067-89c4a81e1a96", "object": "auto-some-name-tls-issue-pxc", "kind": "&TypeMeta{Kind:ConfigMap,APIVersion:v1,}"} 2026-10-05T14:28:21.479Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name-tls-issue","namespace":"tls-issue-cert-manager-14810"}, "namespace": "tls-issue-cert-manager-14810", "name": "some-name-tls-issue", "reconcileID": "8fd0ac1c-80f3-4cbc-a067-89c4a81e1a96", "object": "some-name-tls-issue-haproxy", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"} 2026-10-05T14:28:21.533Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name-tls-issue","namespace":"tls-issue-cert-manager-14810"}, "namespace": "tls-issue-cert-manager-14810", "name": "some-name-tls-issue", "reconcileID": "8fd0ac1c-80f3-4cbc-a067-89c4a81e1a96", "object": "some-name-tls-issue-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"} 2026-10-05T14:28:21.633Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name-tls-issue","namespace":"tls-issue-cert-manager-14810"}, "namespace": "tls-issue-cert-manager-14810", "name": "some-name-tls-issue", "reconcileID": "8fd0ac1c-80f3-4cbc-a067-89c4a81e1a96", "object": "some-name-tls-issue-pxc", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-10-05T14:28:21.684Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name-tls-issue","namespace":"tls-issue-cert-manager-14810"}, "namespace": "tls-issue-cert-manager-14810", "name": "some-name-tls-issue", "reconcileID": "8fd0ac1c-80f3-4cbc-a067-89c4a81e1a96", "object": "some-name-tls-issue-pxc-unready", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-10-05T14:28:21.737Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name-tls-issue","namespace":"tls-issue-cert-manager-14810"}, "namespace": "tls-issue-cert-manager-14810", "name": "some-name-tls-issue", "reconcileID": "8fd0ac1c-80f3-4cbc-a067-89c4a81e1a96", "object": "some-name-tls-issue-haproxy", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-10-05T14:28:21.951Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name-tls-issue","namespace":"tls-issue-cert-manager-14810"}, "namespace": "tls-issue-cert-manager-14810", "name": "some-name-tls-issue", "reconcileID": "8fd0ac1c-80f3-4cbc-a067-89c4a81e1a96", "object": "some-name-tls-issue-haproxy-replicas", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-10-05T14:28:23.098Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name-tls-issue","namespace":"tls-issue-cert-manager-14810"}, "namespace": "tls-issue-cert-manager-14810", "name": "some-name-tls-issue", "reconcileID": "a2a88ead-fd86-410c-832c-0fc0be68585f", "object": "some-name-tls-issue-pxc", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"} 2026-10-05T14:28:23.133Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name-tls-issue","namespace":"tls-issue-cert-manager-14810"}, "namespace": "tls-issue-cert-manager-14810", "name": "some-name-tls-issue", "reconcileID": "a2a88ead-fd86-410c-832c-0fc0be68585f", "object": "some-name-tls-issue-haproxy", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"} 2026-10-05T14:29:14.765Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name-tls-issue","namespace":"tls-issue-cert-manager-14810"}, "namespace": "tls-issue-cert-manager-14810", "name": "some-name-tls-issue", "reconcileID": "cc45b80c-2add-481c-ae1f-f4edf65b4b12", "user": "operator"} 2026-10-05T14:29:14.807Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name-tls-issue","namespace":"tls-issue-cert-manager-14810"}, "namespace": "tls-issue-cert-manager-14810", "name": "some-name-tls-issue", "reconcileID": "cc45b80c-2add-481c-ae1f-f4edf65b4b12", "user": "monitor"} 2026-10-05T14:29:14.849Z INFO User monitor: granted privileges {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name-tls-issue","namespace":"tls-issue-cert-manager-14810"}, "namespace": "tls-issue-cert-manager-14810", "name": "some-name-tls-issue", "reconcileID": "cc45b80c-2add-481c-ae1f-f4edf65b4b12"} 2026-10-05T14:29:14.882Z INFO monitor user privileges granted {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name-tls-issue","namespace":"tls-issue-cert-manager-14810"}, "namespace": "tls-issue-cert-manager-14810", "name": "some-name-tls-issue", "reconcileID": "cc45b80c-2add-481c-ae1f-f4edf65b4b12"} 2026-10-05T14:29:14.917Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name-tls-issue","namespace":"tls-issue-cert-manager-14810"}, "namespace": "tls-issue-cert-manager-14810", "name": "some-name-tls-issue", "reconcileID": "cc45b80c-2add-481c-ae1f-f4edf65b4b12", "user": "xtrabackup"} 2026-10-05T14:29:14.956Z INFO User xtrabackup: granted privileges {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name-tls-issue","namespace":"tls-issue-cert-manager-14810"}, "namespace": "tls-issue-cert-manager-14810", "name": "some-name-tls-issue", "reconcileID": "cc45b80c-2add-481c-ae1f-f4edf65b4b12"} 2026-10-05T14:29:14.985Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name-tls-issue","namespace":"tls-issue-cert-manager-14810"}, "namespace": "tls-issue-cert-manager-14810", "name": "some-name-tls-issue", "reconcileID": "cc45b80c-2add-481c-ae1f-f4edf65b4b12", "user": "replication"} 2026-10-05T14:31:54.009Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name-tls-issue","namespace":"tls-issue-cert-manager-14810"}, "namespace": "tls-issue-cert-manager-14810", "name": "some-name-tls-issue", "reconcileID": "652f7067-c17f-4412-aeda-96b0ce452a20", "user": "root"} 2026-10-05T14:31:54.128Z INFO update PXC version (fetched from db) {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name-tls-issue","namespace":"tls-issue-cert-manager-14810"}, "namespace": "tls-issue-cert-manager-14810", "name": "some-name-tls-issue", "reconcileID": "652f7067-c17f-4412-aeda-96b0ce452a20", "new version": "8.0.43-34.1"} 2026-10-05T14:32:25.621Z DEBUG events CA certificate rotated, re-issuing leaf TLS certificates {"type": "Normal", "object": {"kind":"PerconaXtraDBCluster","namespace":"tls-issue-cert-manager-14810","name":"some-name-tls-issue","uid":"3d35ed7c-8f7e-4d77-b423-0043a8d5fcb4","apiVersion":"pxc.percona.com/v1","resourceVersion":"1791210745509583007"}, "reason": "CARotation"} 2026-10-05T14:32:25.621Z INFO CA certificate rotation detected, re-issuing leaf TLS certificates {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name-tls-issue","namespace":"tls-issue-cert-manager-14810"}, "namespace": "tls-issue-cert-manager-14810", "name": "some-name-tls-issue", "reconcileID": "3b8415e5-7a8a-4055-bca1-f65b1c681ea3", "cluster": "some-name-tls-issue", "sslMismatch": true, "sslInternalMismatch": true} 2026-10-05T14:32:25.721Z INFO Triggering leaf certificate re-issuance {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name-tls-issue","namespace":"tls-issue-cert-manager-14810"}, "namespace": "tls-issue-cert-manager-14810", "name": "some-name-tls-issue", "reconcileID": "3b8415e5-7a8a-4055-bca1-f65b1c681ea3", "certificate": "some-name-tls-issue-ssl"} 2026-10-05T14:32:25.758Z INFO Triggering leaf certificate re-issuance {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name-tls-issue","namespace":"tls-issue-cert-manager-14810"}, "namespace": "tls-issue-cert-manager-14810", "name": "some-name-tls-issue", "reconcileID": "3b8415e5-7a8a-4055-bca1-f65b1c681ea3", "certificate": "some-name-tls-issue-ssl-internal"} 2026-10-05T14:32:31.596Z DEBUG Updating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name-tls-issue","namespace":"tls-issue-cert-manager-14810"}, "namespace": "tls-issue-cert-manager-14810", "name": "some-name-tls-issue", "reconcileID": "b18118d0-2672-4556-9aac-84bbe3739ece", "object": "some-name-tls-issue-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true} 2026-10-05T14:32:31.740Z DEBUG Updating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name-tls-issue","namespace":"tls-issue-cert-manager-14810"}, "namespace": "tls-issue-cert-manager-14810", "name": "some-name-tls-issue", "reconcileID": "b18118d0-2672-4556-9aac-84bbe3739ece", "object": "some-name-tls-issue-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true} 2026-10-05T14:35:49.765Z INFO CrashRecovery Results of scanning sequences {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name-tls-issue","namespace":"tls-issue-cert-manager-14810"}, "namespace": "tls-issue-cert-manager-14810", "name": "some-name-tls-issue", "reconcileID": "8a68dd7d-6b7d-412b-bceb-536fce4e5060", "pod": "some-name-tls-issue-pxc-0", "clusterUUID": "1155439f-c0c9-11f1-a4f9-72dee4771af6", "maxSeq": 39} 2026-10-05T14:35:49.765Z INFO CrashRecovery We are in full cluster crash, starting recovery {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name-tls-issue","namespace":"tls-issue-cert-manager-14810"}, "namespace": "tls-issue-cert-manager-14810", "name": "some-name-tls-issue", "reconcileID": "8a68dd7d-6b7d-412b-bceb-536fce4e5060"} 2026-10-05T14:35:49.766Z DEBUG events We are in full cluster crash {"type": "Warning", "object": {"kind":"PerconaXtraDBCluster","namespace":"tls-issue-cert-manager-14810","name":"some-name-tls-issue","uid":"3d35ed7c-8f7e-4d77-b423-0043a8d5fcb4","apiVersion":"pxc.percona.com/v1","resourceVersion":"1791210784347599007"}, "reason": "FullClusterCrashDetected"} 2026-10-05T14:35:50.040Z DEBUG events Recovery started in some-name-tls-issue-pxc-0, position 1155439f-c0c9-11f1-a4f9-72dee4771af6:39 {"type": "Normal", "object": {"kind":"PerconaXtraDBCluster","namespace":"tls-issue-cert-manager-14810","name":"some-name-tls-issue","uid":"3d35ed7c-8f7e-4d77-b423-0043a8d5fcb4","apiVersion":"pxc.percona.com/v1","resourceVersion":"1791210784347599007"}, "reason": "RecoveryStarted"} 2026-10-05T14:35:50.040Z INFO CrashRecovery Recovery started {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name-tls-issue","namespace":"tls-issue-cert-manager-14810"}, "namespace": "tls-issue-cert-manager-14810", "name": "some-name-tls-issue", "reconcileID": "8a68dd7d-6b7d-412b-bceb-536fce4e5060", "pod": "some-name-tls-issue-pxc-0", "position": "1155439f-c0c9-11f1-a4f9-72dee4771af6:39"} 2026-10-05T14:36:20.248Z ERROR Observed a panic {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name-tls-issue","namespace":"tls-issue-cert-manager-14810"}, "namespace": "tls-issue-cert-manager-14810", "name": "some-name-tls-issue", "reconcileID": "8a68dd7d-6b7d-412b-bceb-536fce4e5060", "panic": "runtime error: invalid memory address or nil pointer dereference", "panicGoValue": "\"invalid memory address or nil pointer dereference\"", "stacktrace": "goroutine 236 [running]:\nk8s.io/apimachinery/pkg/util/runtime.logPanic({0x243aa78, 0x2fa956e570e0}, {0x1f53d00, 0x386de70})\n\t/go/pkg/mod/k8s.io/apimachinery@v0.37.0/pkg/util/runtime/runtime.go:134 +0xbc\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile.func1()\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:201 +0x103\npanic({0x1f53d00?, 0x386de70?})\n\t/usr/local/go/src/runtime/panic.go:860 +0x13a\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/naming.GetConditionTLSState(...)\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/naming/condition.go:17\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileTLSToggle(0x2fa9562231f0, {0x243aa78, 0x2fa956e570e0}, 0x2fa9570bfb88)\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/tls.go:506 +0xe4\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileSSL(0x2fa9562231f0, {0x243aa78, 0x2fa956e570e0}, 0x2fa9570bfb88)\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/tls.go:29 +0x3f\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile(0x2fa9562231f0, {0x243aa78, 0x2fa956e570e0}, {{{0x2fa9573441c0?, 0x17b76c5?}, {0x2fa9563af788?, 0x0?}}})\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:382 +0x8f9\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile(0x2fa956e56fc0, {0x243aa78?, 0x2fa956e570e0}, {{{0x2fa9573441c0, 0x0?}, {0x2fa9563af788?, 0x0?}}})\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221 +0x1ab\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler(0x2460140, {0x243aab0, 0x2fa9563c1db0}, {{{0x2fa9573441c0, 0x1c}, {0x2fa9563af788, 0x13}}}, 0x0)\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478 +0x39b\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem(0x2460140, {0x243aab0, 0x2fa9563c1db0})\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437 +0x1f8\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1()\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312 +0x85\ncreated by sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1 in goroutine 149\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:308 +0x26b\n"} 2026-10-05T14:36:20.248Z ERROR Reconciler error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name-tls-issue","namespace":"tls-issue-cert-manager-14810"}, "namespace": "tls-issue-cert-manager-14810", "name": "some-name-tls-issue", "reconcileID": "8a68dd7d-6b7d-412b-bceb-536fce4e5060", "error": "panic: runtime error: invalid memory address or nil pointer dereference [recovered]"} 2026-10-05T14:36:22.919Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name-tls-issue","namespace":"tls-issue-cert-manager-14810"}, "namespace": "tls-issue-cert-manager-14810", "name": "some-name-tls-issue", "reconcileID": "05c9486d-e54b-4e78-9176-358fc4533dcc", "err": "get primary pxc pod: failed to get proxy connection: dial tcp 34.118.228.130:3306: connect: connection refused"} 2026-10-05T14:36:30.171Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name-tls-issue","namespace":"tls-issue-cert-manager-14810"}, "namespace": "tls-issue-cert-manager-14810", "name": "some-name-tls-issue", "reconcileID": "71c51fac-4523-4beb-894f-cde55ddd9182", "err": "get primary pxc pod: failed to get proxy connection: dial tcp 34.118.228.130:3306: connect: connection refused"} 2026-10-05T14:36:32.435Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name-tls-issue","namespace":"tls-issue-cert-manager-14810"}, "namespace": "tls-issue-cert-manager-14810", "name": "some-name-tls-issue", "reconcileID": "ca67753e-c19a-4c3c-a241-e1883de0d4d1", "err": "get primary pxc pod: failed to get proxy connection: dial tcp 34.118.228.130:3306: connect: connection refused"} 2026-10-05T14:36:37.699Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name-tls-issue","namespace":"tls-issue-cert-manager-14810"}, "namespace": "tls-issue-cert-manager-14810", "name": "some-name-tls-issue", "reconcileID": "734f279a-0264-4fd4-8153-1ad822f90a7d", "err": "failed to ensure cluster readonly status: connect to pod some-name-tls-issue-pxc-1: dial tcp 10.144.40.50:33062: connect: connection refused"} 2026-10-05T14:36:43.061Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name-tls-issue","namespace":"tls-issue-cert-manager-14810"}, "namespace": "tls-issue-cert-manager-14810", "name": "some-name-tls-issue", "reconcileID": "451af584-4b89-4d9d-b9df-82593ff1d2a0", "err": "failed to ensure cluster readonly status: connect to pod some-name-tls-issue-pxc-2: dial tcp 10.144.41.27:33062: connect: connection refused"} github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileSSL github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileTLSToggle github.com/percona/percona-xtradb-cluster-operator/pkg/naming.GetConditionTLSState k8s.io/apimachinery/pkg/util/runtime.logPanic runtime.gopanic runtime.panicmem runtime.sigpanic sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile.func1 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler + local backup_names ++ kubectl_bin get pxc-backup --all-namespaces -o 'jsonpath={range .items[*]}{.metadata.namespace}/{.metadata.name}{"\n"}{end}' + backup_names= + [[ -n '' ]] + local pxc_names ++ kubectl_bin get pxc --all-namespaces -o 'jsonpath={range .items[*]}{.metadata.namespace}/{.metadata.name}{"\n"}{end}' + pxc_names=tls-issue-cert-manager-14810/some-name-tls-issue + [[ -n tls-issue-cert-manager-14810/some-name-tls-issue ]] + read -r pns name + kubectl_bin get pxc --all-namespaces -o 'jsonpath={range .items[*]}{.metadata.namespace} {.metadata.name}{"\n"}{end}' + kubectl_bin patch pxc -n tls-issue-cert-manager-14810 some-name-tls-issue --type=merge -p '{"metadata":{"finalizers":["percona.com/delete-pxc-pvc"]}}' perconaxtradbcluster.pxc.percona.com/some-name-tls-issue patched + read -r pns name + timeout 300 kubectl delete pxc --all --all-namespaces --ignore-not-found perconaxtradbcluster.pxc.percona.com "some-name-tls-issue" deleted from tls-issue-cert-manager-14810 namespace ++ date +%s + local deadline=1791211396 ++ kubectl_bin get pxc --all-namespaces -o name --ignore-not-found + [[ -z '' ]] + timeout 300 kubectl delete pxc-restore --all --all-namespaces --wait=false --ignore-not-found No resources found + kubectl_bin delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.1xGGzXGxnN ++ mktemp + local LAST_ERR=/tmp/tmp.CnRMq3JfaY + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.1xGGzXGxnN validatingwebhookconfiguration.admissionregistration.k8s.io "percona-xtradbcluster-webhook" deleted + cat /tmp/tmp.CnRMq3JfaY + rm /tmp/tmp.1xGGzXGxnN /tmp/tmp.CnRMq3JfaY + return 0 + kubectl_bin delete -f https://github.com/jetstack/cert-manager/releases/download/v1.20.2/cert-manager.yaml namespace "cert-manager" deleted customresourcedefinition.apiextensions.k8s.io "challenges.acme.cert-manager.io" deleted customresourcedefinition.apiextensions.k8s.io "orders.acme.cert-manager.io" deleted customresourcedefinition.apiextensions.k8s.io "certificaterequests.cert-manager.io" deleted customresourcedefinition.apiextensions.k8s.io "certificates.cert-manager.io" deleted customresourcedefinition.apiextensions.k8s.io "clusterissuers.cert-manager.io" deleted customresourcedefinition.apiextensions.k8s.io "issuers.cert-manager.io" deleted serviceaccount "cert-manager-cainjector" deleted from cert-manager namespace serviceaccount "cert-manager" deleted from cert-manager namespace serviceaccount "cert-manager-webhook" deleted from cert-manager namespace clusterrole.rbac.authorization.k8s.io "cert-manager-cainjector" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-controller-issuers" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-controller-certificates" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-controller-orders" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-controller-challenges" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-cluster-view" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-view" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-edit" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-cainjector" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-issuers" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-certificates" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-orders" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-challenges" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" deleted role.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" deleted from kube-system namespace role.rbac.authorization.k8s.io "cert-manager:leaderelection" deleted from kube-system namespace role.rbac.authorization.k8s.io "cert-manager-tokenrequest" deleted from cert-manager namespace role.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" deleted from cert-manager namespace rolebinding.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" deleted from kube-system namespace rolebinding.rbac.authorization.k8s.io "cert-manager:leaderelection" deleted from kube-system namespace rolebinding.rbac.authorization.k8s.io "cert-manager-tokenrequest" deleted from cert-manager namespace rolebinding.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" deleted from cert-manager namespace service "cert-manager-cainjector" deleted from cert-manager namespace service "cert-manager" deleted from cert-manager namespace service "cert-manager-webhook" deleted from cert-manager namespace deployment.apps "cert-manager-cainjector" deleted from cert-manager namespace deployment.apps "cert-manager" deleted from cert-manager namespace deployment.apps "cert-manager-webhook" deleted from cert-manager namespace mutatingwebhookconfiguration.admissionregistration.k8s.io "cert-manager-webhook" deleted validatingwebhookconfiguration.admissionregistration.k8s.io "cert-manager-webhook" deleted + '[' '!' -z '' ']' + '[' -n pxc-operator ']' + kubectl_bin delete --grace-period=0 --force=true namespace tls-issue-cert-manager-14810 + kubectl_bin delete --grace-period=0 --force=true namespace pxc-operator ++ date +%s + local deadline=1791211251 ++ mktemp + local LAST_OUT=/tmp/tmp.sfjcMmKHFb ++ mktemp + local LAST_OUT=/tmp/tmp.qB0hhsZ6No ++ mktemp ++ mktemp ++ kubectl_bin get namespace tls-issue-cert-manager-14810 --ignore-not-found -o name + local LAST_ERR=/tmp/tmp.fwvtIdxoNo + local exit_status=0 + local LAST_ERR=/tmp/tmp.QYGpzZj1Gf + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete --grace-period=0 --force=true namespace tls-issue-cert-manager-14810 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete --grace-period=0 --force=true namespace pxc-operator + [[ -z namespace/tls-issue-cert-manager-14810 ]] ++ date +%s + [[ 1791211134 -ge 1791211251 ]] + sleep 5 ++ kubectl_bin get namespace tls-issue-cert-manager-14810 --ignore-not-found -o name + [[ -z namespace/tls-issue-cert-manager-14810 ]] ++ date +%s + [[ 1791211142 -ge 1791211251 ]] + sleep 5 ++ kubectl_bin get namespace tls-issue-cert-manager-14810 --ignore-not-found -o name + [[ -z namespace/tls-issue-cert-manager-14810 ]] ++ date +%s + [[ 1791211149 -ge 1791211251 ]] + sleep 5 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.sfjcMmKHFb namespace "pxc-operator" force deleted + cat /tmp/tmp.QYGpzZj1Gf Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. + rm /tmp/tmp.sfjcMmKHFb /tmp/tmp.QYGpzZj1Gf + return 0 ++ kubectl_bin get namespace tls-issue-cert-manager-14810 --ignore-not-found -o name + [[ -z namespace/tls-issue-cert-manager-14810 ]] ++ date +%s + [[ 1791211158 -ge 1791211251 ]] + sleep 5 ++ kubectl_bin get namespace tls-issue-cert-manager-14810 --ignore-not-found -o name + [[ -z namespace/tls-issue-cert-manager-14810 ]] ++ date +%s + [[ 1791211168 -ge 1791211251 ]] + sleep 5 ++ kubectl_bin get namespace tls-issue-cert-manager-14810 --ignore-not-found -o name + [[ -z namespace/tls-issue-cert-manager-14810 ]] ++ date +%s + [[ 1791211177 -ge 1791211251 ]] + sleep 5 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.qB0hhsZ6No namespace "tls-issue-cert-manager-14810" force deleted + cat /tmp/tmp.fwvtIdxoNo Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. + rm /tmp/tmp.qB0hhsZ6No /tmp/tmp.fwvtIdxoNo + return 0 ++ kubectl_bin get namespace tls-issue-cert-manager-14810 --ignore-not-found -o name + [[ -z '' ]] ++ date +%s + deadline=1791211306 ++ awk '$2 != ""' ++ kubectl_bin get pv -o 'jsonpath={range .items[*]}{.metadata.name}{"\t"}{.metadata.deletionTimestamp}{"\n"}{end}' + [[ -z '' ]] + rm -rf /tmp/tmp.vlgOWXkFpe + desc 'test passed' + set +o xtrace ----------------------------------------------------------------------------------- test passed -----------------------------------------------------------------------------------