Helm chart with dependencies

.gitignore

@@ -15,3 +15,4 @@ docs/flytekit/flytekit.interfaces.html
 docs/searchindex.js
 docs/
 __pycache__/
+/helm/charts/

helm/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

helm/Chart.yaml

@@ -0,0 +1,20 @@
+apiVersion: v2
+name: flyte
+description: A Helm chart for Flyte
+type: application
+version: 0.2.0
+appVersion: 0.8.0
+dependencies:
+  - name: contour
+    version: 4.1.2
+    repository: https://charts.bitnami.com/bitnami
+    condition: contour.enabled
+  - name: spark-operator
+    alias: sparkoperator
+    version: 1.0.6
+    repository: https://googlecloudplatform.github.io/spark-on-k8s-operator
+    condition: spark.enabled
+  - name: kubernetes-dashboard
+    version: 4.0.2
+    repository: https://kubernetes.github.io/dashboard/
+    condition: kubernetes-dashboard.enabled

helm/README.md.gotmpl

@@ -0,0 +1,28 @@
+{{ template "chart.header" . }}
+{{ template "chart.deprecationWarning" . }}
+
+{{ template "chart.versionBadge" . }}{{ template "chart.typeBadge" . }}{{ template "chart.appVersionBadge" . }}
+
+{{ template "chart.description" . }}
+
+{{ template "chart.homepageLine" . }}
+
+{{ template "chart.maintainersSection" . }}
+
+{{ template "chart.sourcesSection" . }}
+
+{{ template "chart.requirementsSection" . }}
+
+### SANDBOX INSTALLATION:
+- Install `helm 3` realese version
+- Generate `helm template --name-template=flyte-sandbox . -n flyte -f values-sandbox.yaml > flyte_generated_sandbox.yaml`
+- Deploy the manifest `kubectl apply -f flyte_generated_sandbox.yaml`
+- When all pods are running - run end2end tests: `kubectl apply -f ../end2end/tests/endtoend.yaml`
+- Get flyte host `minikube service contour -n heptio-contour --url`. And then visit `http://<HOST>/console`
+
+### CONFIGURATION NOTES:
+- The docker images, their tags and other default parameters are configured in `values.yaml` file.
+- Each Flyte installation type should have separate `values-*.yaml` file: for sandbox, EKS and etc. The configuration in `values.yaml` and the choosen config `values-*.yaml` are merged when generating the deployment manifest.
+- The configuration in `values-sandbox.yaml` is ready for installation in minikube. But `values-eks.yaml` should be edited before installation: s3 bucket, RDS hosts, iam roles, secrets and etc need to be modified.
+
+{{ template "chart.valuesSection" . }}

helm/templates/_helpers.tpl

@@ -0,0 +1,161 @@
+{{/* vim: set filetype=mustache: */}}
+
+{{- define "flyte.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{- define "flyte.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{- define "flyte.namespace" -}}
+{{- default .Release.Namespace .Values.forceNamespace | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+
+{{- define "flyteadmin.name" -}}
+flyteadmin
+{{- end -}}
+
+{{- define "flyteadmin.selectorLabels" -}}
+app.kubernetes.io/name: {{ template "flyteadmin.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end -}}
+
+{{- define "flyteadmin.labels" -}}
+{{ include "flyteadmin.selectorLabels" . }}
+helm.sh/chart: {{ include "flyte.chart" . }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end -}}
+
+
+{{- define "datacatalog.name" -}}
+datacatalog
+{{- end -}}
+
+{{- define "datacatalog.selectorLabels" -}}
+app.kubernetes.io/name: {{ template "datacatalog.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end -}}
+
+{{- define "datacatalog.labels" -}}
+{{ include "datacatalog.selectorLabels" . }}
+helm.sh/chart: {{ include "flyte.chart" . }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end -}}
+
+
+{{- define "flytepropeller.name" -}}
+flytepropeller
+{{- end -}}
+
+{{- define "flytepropeller.selectorLabels" -}}
+app.kubernetes.io/name: {{ template "flytepropeller.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end -}}
+
+{{- define "flytepropeller.labels" -}}
+{{ include "flytepropeller.selectorLabels" . }}
+helm.sh/chart: {{ include "flyte.chart" . }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end -}}
+
+
+{{- define "flyteconsole.name" -}}
+flyteconsole
+{{- end -}}
+
+{{- define "flyteconsole.selectorLabels" -}}
+app.kubernetes.io/name: {{ template "flyteconsole.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end -}}
+
+{{- define "flyteconsole.labels" -}}
+{{ include "flyteconsole.selectorLabels" . }}
+helm.sh/chart: {{ include "flyte.chart" . }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end -}}
+
+
+{{- define "redis.name" -}}
+redis
+{{- end -}}
+
+{{- define "redis.selectorLabels" -}}
+app.kubernetes.io/name: {{ template "redis.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end -}}
+
+{{- define "redis.labels" -}}
+{{ include "redis.selectorLabels" . }}
+helm.sh/chart: {{ include "flyte.chart" . }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end -}}
+
+
+{{- define "postgres.name" -}}
+postgres
+{{- end -}}
+
+{{- define "postgres.selectorLabels" -}}
+app.kubernetes.io/name: {{ template "postgres.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end -}}
+
+{{- define "postgres.labels" -}}
+{{ include "postgres.selectorLabels" . }}
+helm.sh/chart: {{ include "flyte.chart" . }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end -}}
+
+
+{{- define "minio.name" -}}
+minio
+{{- end -}}
+
+{{- define "minio.selectorLabels" -}}
+app.kubernetes.io/name: {{ template "minio.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end -}}
+
+{{- define "minio.labels" -}}
+{{ include "minio.selectorLabels" . }}
+helm.sh/chart: {{ include "flyte.chart" . }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end -}}
+
+{{- define "pytorch-operator.name" -}}
+pytorch-operator
+{{- end -}}
+
+{{- define "pytorch-operator.namespace" -}}
+pytorch-operator
+{{- end -}}
+
+{{- define "pytorch-operator.selectorLabels" -}}
+app.kubernetes.io/name: {{ template "pytorch-operator.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end -}}
+
+{{- define "pytorch-operator.labels" -}}
+{{ include "pytorch-operator.selectorLabels" . }}
+helm.sh/chart: {{ include "flyte.chart" . }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end -}}
+
+# Optional blocks for secret mount 
+
+{{- define "databaseSecret.volume" -}}
+{{- with .Values.common.databaseSecret.name -}}
+- name: {{ . }}
+  secret:
+    secretName: {{ . }}
+{{- end }}
+{{- end }}
+
+{{- define "databaseSecret.volumeMount" -}}
+{{- with .Values.common.databaseSecret.name -}}
+- mountPath: /etc/db
+  name: {{ . }}
+{{- end }}
+{{- end }}

helm/templates/admin/configmap.yaml

@@ -0,0 +1,31 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: flyte-admin-config
+  namespace: {{ template "flyte.namespace" . }}
+  labels: {{ include "flyteadmin.labels" . | nindent 4 }}
+data:
+{{- with .Values.configmap.cluster_resources }}
+  cluster_resources.yaml: | {{ toYaml . | nindent 4 }}
+{{- end }}
+{{- with .Values.configmap.db }}
+  db.yaml: | {{ toYaml . | nindent 4 }}
+{{- end }}
+{{- with .Values.configmap.domain }}
+  domain.yaml: | {{ toYaml . | nindent 4 }}
+{{- end }}
+{{- with .Values.configmap.logger }}
+  logger.yaml: | {{ toYaml . | nindent 4 }}
+{{- end }}
+{{- with .Values.configmap.remote_data }}
+  remote_data.yaml: | {{ toYaml . | nindent 4 }}
+{{- end }}
+{{- with .Values.configmap.server }}
+  server.yaml: | {{ toYaml . | nindent 4 }}
+{{- end }}
+{{- with .Values.configmap.storage }}
+  storage.yaml: | {{ toYaml . | nindent 4 }}
+{{- end }}
+{{- with .Values.configmap.task_resource_defaults }}
+  task_resource_defaults.yaml: | {{ toYaml . | nindent 4 }}
+{{- end }}

helm/templates/admin/cronjob.yaml

@@ -0,0 +1,40 @@
+apiVersion: batch/v1beta1
+kind: CronJob
+metadata:
+  name: syncresources
+  namespace: {{ template "flyte.namespace" . }}
+  labels: {{ include "flyteadmin.labels" . | nindent 4 }}
+spec:
+  schedule: '*/1 * * * *'
+  jobTemplate:
+    spec:
+      template:
+        {{- with .Values.flyteadmin.podAnnotations }}
+        metadata:
+          annotations: {{ toYaml . | nindent 12 }}
+        {{- end }}
+        spec:
+          containers:
+          - command:
+            - flyteadmin
+            - --config
+            - {{ .Values.flyteadmin.configPath }}
+            - clusterresource
+            - sync
+            image: "{{ .Values.flyteadmin.image.repository }}:{{ .Values.flyteadmin.image.tag }}"
+            imagePullPolicy: "{{ .Values.flyteadmin.image.pullPolicy }}"
+            name: sync-cluster-resources
+            volumeMounts: {{- include "databaseSecret.volumeMount" . | nindent 12 }}
+            - mountPath: /etc/flyte/clusterresource/templates
+              name: resource-templates
+            - mountPath: /etc/flyte/config
+              name: config-volume
+          restartPolicy: OnFailure
+          serviceAccountName: flyteadmin
+          volumes: {{- include "databaseSecret.volume" . | nindent 10 }}
+          - configMap:
+              name: clusterresource-template
+            name: resource-templates
+          - configMap:
+              name: flyte-admin-config
+            name: config-volume