From 36b5fc64fa99e274195aba16892b5a40f3a71e4e Mon Sep 17 00:00:00 2001 From: dnskr Date: Thu, 12 Jan 2023 00:57:04 +0100 Subject: [PATCH] [K8S][HELM] Add configuration support for multiple frontends to helm chart --- charts/kyuubi/templates/NOTES.txt | 30 +++++++-- charts/kyuubi/templates/_helpers.tpl | 37 +++------- charts/kyuubi/templates/kyuubi-configmap.yaml | 15 +++-- .../kyuubi/templates/kyuubi-deployment.yaml | 19 +++--- charts/kyuubi/templates/kyuubi-service.yaml | 35 ++++++---- charts/kyuubi/values.yaml | 67 +++++++++++++++---- 6 files changed, 132 insertions(+), 71 deletions(-) diff --git a/charts/kyuubi/templates/NOTES.txt b/charts/kyuubi/templates/NOTES.txt index 44a35b6b736..baa9ccffffd 100644 --- a/charts/kyuubi/templates/NOTES.txt +++ b/charts/kyuubi/templates/NOTES.txt @@ -15,7 +15,29 @@ # limitations under the License. # -Get kyuubi expose URL by running these commands: - export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "kyuubi.fullname" . }}-nodeport) - export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") - echo $NODE_IP:$NODE_PORT \ No newline at end of file +The chart has been installed! + +In order to check the release status, use: + helm status {{ .Release.Name }} -n {{ .Release.Namespace }} + or for more detailed info + helm get all {{ .Release.Name }} -n {{ .Release.Namespace }} + +************************ +*** Kyuubi frontends *** +************************ +{{- range $name, $frontend := .Values.frontend }} +{{- if $frontend.enabled }} +{{ $name | snakecase | upper }}: +- To access {{ $.Release.Name }}-{{ $name | kebabcase }} service within the cluster, use the following URL: + {{ $.Release.Name }}-{{ $name | kebabcase }}.{{ $.Release.Namespace }}.svc.cluster.local +- To access {{ $.Release.Name }}-{{ $name | kebabcase }} service from outside the cluster for debugging, run the following command: + kubectl port-forward svc/{{ $.Release.Name }}-{{ $name | kebabcase }} {{ tpl $frontend.service.port $ }}:{{ tpl $frontend.service.port $ }} -n {{ $.Release.Namespace }} + and use 127.0.0.1:{{ tpl $frontend.service.port $ }} +{{- if eq $frontend.service.type "NodePort" }} +- To access {{ $.Release.Name }}-{{ $name | kebabcase }} service from outside the cluster through configured NodePort, run the following commands: + export NODE_PORT=$(kubectl get service {{ $.Release.Name }}-{{ $name | kebabcase }} -n {{ $.Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}") + export NODE_IP=$(kubectl get nodes -n {{ $.Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/kyuubi/templates/_helpers.tpl b/charts/kyuubi/templates/_helpers.tpl index 684c1f354b1..0479f9efe4d 100644 --- a/charts/kyuubi/templates/_helpers.tpl +++ b/charts/kyuubi/templates/_helpers.tpl @@ -16,33 +16,18 @@ */}} {{/* -Expand the name of the chart. +A comma separated string of enabled frontend protocols, e.g. "MYSQL,REST,THRIFT_BINARY". +For details, see 'kyuubi.frontend.protocols': https://kyuubi.readthedocs.io/en/master/deployment/settings.html#frontend */}} -{{- define "kyuubi.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "kyuubi.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- define "kyuubi.frontend.protocols" -}} +{{- $protocols := list }} +{{- range $name, $frontend := .Values.frontend }} + {{- if $frontend.enabled }} + {{- $protocols = $name | snakecase | upper | append $protocols }} + {{- end }} {{- end }} +{{- if not $protocols }} + {{ fail "At least one frontend protocol must be enabled!" }} {{- end }} +{{- $protocols | join "," }} {{- end }} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "kyuubi.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{- end }} \ No newline at end of file diff --git a/charts/kyuubi/templates/kyuubi-configmap.yaml b/charts/kyuubi/templates/kyuubi-configmap.yaml index ada9e3dc876..efd1514a462 100644 --- a/charts/kyuubi/templates/kyuubi-configmap.yaml +++ b/charts/kyuubi/templates/kyuubi-configmap.yaml @@ -26,22 +26,27 @@ metadata: app.kubernetes.io/version: {{ .Values.image.tag | default .Chart.AppVersion | quote }} app.kubernetes.io/managed-by: {{ .Release.Service }} data: - {{- with .Values.server.conf.kyuubiEnv }} + {{- with .Values.kyuubiConf.kyuubiEnv }} kyuubi-env.sh: | #!/usr/bin/env bash {{- tpl . $ | nindent 4 }} {{- end }} kyuubi-defaults.conf: | ## Helm chart provided Kyuubi configurations - kyuubi.frontend.bind.host={{ .Values.server.bind.host }} - kyuubi.frontend.bind.port={{ .Values.server.bind.port }} kyuubi.kubernetes.namespace={{ .Release.Namespace }} + kyuubi.frontend.bind.host=localhost + kyuubi.frontend.thrift.binary.bind.port={{ .Values.frontend.thriftBinary.port }} + kyuubi.frontend.thrift.http.bind.port={{ .Values.frontend.thriftHttp.port }} + kyuubi.frontend.rest.bind.port={{ .Values.frontend.rest.port }} + kyuubi.frontend.mysql.bind.port={{ .Values.frontend.mysql.port }} + kyuubi.frontend.trino.bind.port={{ .Values.frontend.trino.port }} + kyuubi.frontend.protocols={{ include "kyuubi.frontend.protocols" . }} ## User provided Kyuubi configurations - {{- with .Values.server.conf.kyuubiDefaults }} + {{- with .Values.kyuubiConf.kyuubiDefaults }} {{- tpl . $ | nindent 4 }} {{- end }} - {{- with .Values.server.conf.log4j2 }} + {{- with .Values.kyuubiConf.log4j2 }} log4j2.xml: | {{- tpl . $ | nindent 4 }} {{- end }} diff --git a/charts/kyuubi/templates/kyuubi-deployment.yaml b/charts/kyuubi/templates/kyuubi-deployment.yaml index 941fdf164c6..d409eb1aca6 100644 --- a/charts/kyuubi/templates/kyuubi-deployment.yaml +++ b/charts/kyuubi/templates/kyuubi-deployment.yaml @@ -57,13 +57,16 @@ spec: envFrom: {{- tpl (toYaml .) $ | nindent 12 }} {{- end }} ports: - - name: frontend-port - containerPort: {{ .Values.server.bind.port }} - protocol: TCP + {{- range $name, $frontend := .Values.frontend }} + {{- if $frontend.enabled }} + - name: {{ $name | kebabcase }} + containerPort: {{ $frontend.port }} + {{- end }} + {{- end }} {{- if .Values.probe.liveness.enabled }} livenessProbe: - tcpSocket: - port: {{ .Values.server.bind.port }} + exec: + command: ["/bin/sh", "-c", "bin/kyuubi status"] initialDelaySeconds: {{ .Values.probe.liveness.initialDelaySeconds }} periodSeconds: {{ .Values.probe.liveness.periodSeconds }} timeoutSeconds: {{ .Values.probe.liveness.timeoutSeconds }} @@ -72,8 +75,8 @@ spec: {{- end }} {{- if .Values.probe.readiness.enabled }} readinessProbe: - tcpSocket: - port: {{ .Values.server.bind.port }} + exec: + command: [ "/bin/sh", "-c", "$KYUUBI_HOME/bin/kyuubi status" ] initialDelaySeconds: {{ .Values.probe.readiness.initialDelaySeconds }} periodSeconds: {{ .Values.probe.readiness.periodSeconds }} timeoutSeconds: {{ .Values.probe.readiness.timeoutSeconds }} @@ -85,7 +88,7 @@ spec: {{- end }} volumeMounts: - name: conf - mountPath: {{ .Values.server.confDir }} + mountPath: {{ .Values.kyuubiConfDir }} {{- with .Values.volumeMounts }} {{- tpl (toYaml .) $ | nindent 12 }} {{- end }} diff --git a/charts/kyuubi/templates/kyuubi-service.yaml b/charts/kyuubi/templates/kyuubi-service.yaml index 0152bd23d1f..25c83041281 100644 --- a/charts/kyuubi/templates/kyuubi-service.yaml +++ b/charts/kyuubi/templates/kyuubi-service.yaml @@ -15,27 +15,34 @@ # limitations under the License. # +{{- range $name, $frontend := .Values.frontend }} +{{- if $frontend.enabled }} apiVersion: v1 kind: Service metadata: - name: {{ .Release.Name }} + name: {{ $.Release.Name }}-{{ $name | kebabcase }} labels: - helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }} - app.kubernetes.io/name: {{ .Chart.Name }} - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/version: {{ .Values.image.tag | default .Chart.AppVersion | quote }} - app.kubernetes.io/managed-by: {{ .Release.Service }} - {{- with .Values.service.annotations }} + helm.sh/chart: {{ $.Chart.Name }}-{{ $.Chart.Version }} + app.kubernetes.io/name: {{ $.Chart.Name }} + app.kubernetes.io/instance: {{ $.Release.Name }} + app.kubernetes.io/version: {{ $.Values.image.tag | default $.Chart.AppVersion | quote }} + app.kubernetes.io/managed-by: {{ $.Release.Service }} + {{- with $frontend.service.annotations }} annotations: {{- toYaml . | nindent 4 }} {{- end }} spec: + type: {{ $frontend.service.type }} ports: - - name: http - nodePort: {{ .Values.service.port }} - port: {{ .Values.server.bind.port }} - protocol: TCP - type: {{ .Values.service.type }} + - name: {{ $name | kebabcase }} + port: {{ tpl $frontend.service.port $ }} + targetPort: {{ $frontend.port }} + {{- if and (eq $frontend.service.type "NodePort") ($frontend.service.nodePort) }} + nodePort: {{ $frontend.service.nodePort }} + {{- end }} selector: - app.kubernetes.io/name: {{ .Chart.Name }} - app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/name: {{ $.Chart.Name }} + app.kubernetes.io/instance: {{ $.Release.Name }} +--- +{{- end }} +{{- end }} diff --git a/charts/kyuubi/values.yaml b/charts/kyuubi/values.yaml index 22ae9d5a914..7be0ce75896 100644 --- a/charts/kyuubi/values.yaml +++ b/charts/kyuubi/values.yaml @@ -57,12 +57,60 @@ probe: failureThreshold: 10 successThreshold: 1 -server: - bind: - host: 0.0.0.0 +# Kyuubi frontends +frontend: + # Thrift Binary protocol (HiveServer2 compatible) + thriftBinary: + enabled: true port: 10009 - confDir: /opt/kyuubi/conf - conf: + service: + type: ClusterIP + port: "{{ .Values.frontend.thriftBinary.port }}" + nodePort: ~ + annotations: {} + + # Thrift HTTP protocol (HiveServer2 compatible) + thriftHttp: + enabled: false + port: 10010 + service: + type: ClusterIP + port: "{{ .Values.frontend.thriftHttp.port }}" + nodePort: ~ + annotations: {} + + # REST API protocol (experimental) + rest: + enabled: false + port: 10099 + service: + type: ClusterIP + port: "{{ .Values.frontend.rest.port }}" + nodePort: ~ + annotations: {} + + # MySQL compatible text protocol (experimental) + mysql: + enabled: false + port: 3309 + service: + type: ClusterIP + port: "{{ .Values.frontend.mysql.port }}" + nodePort: ~ + annotations: {} + + # Trino compatible http protocol (experimental) + trino: + enabled: false + port: 10999 + service: + type: ClusterIP + port: "{{ .Values.frontend.trino.port }}" + nodePort: ~ + annotations: {} + +kyuubiConfDir: /opt/kyuubi/conf +kyuubiConf: # The value (templated string) is used for kyuubi-env.sh file # See https://kyuubi.apache.org/docs/latest/deployment/settings.html#environments for more details kyuubiEnv: ~ @@ -89,15 +137,6 @@ initContainers: [] # Additional containers for Kyuubi pod (templated) containers: [] -service: - type: NodePort - # The default port limit of kubernetes is 30000-32767 - # to change: - # vim kube-apiserver.yaml (usually under path: /etc/kubernetes/manifests/) - # add or change line 'service-node-port-range=1-32767' under kube-apiserver - port: 30009 - annotations: {} - resources: {} # Used to specify resource, default unlimited. # If you do want to specify resources: