Simplify Helm storage configuration

Signed-off-by: Sören Brunk <[email protected]>
flyteorg · Apr 14, 2021 · 151446d · 151446d
1 parent 1556a0d
commit 151446d
Show file tree

Hide file tree

Showing 7 changed files with 64 additions and 39 deletions.
diff --git a/helm/templates/_helpers.tpl b/helm/templates/_helpers.tpl
@@ -159,3 +159,39 @@ app.kubernetes.io/managed-by: {{ .Release.Service }}
   name: {{ . }}
 {{- end }}
 {{- end }}
+
+{{- define "storage" -}}
+storage:
+{{- if eq .Values.storage.type "s3" }}
+  type: s3
+  container: {{ .Values.storage.bucketName | quote }}
+  connection:
+    auth-type: iam
+    region: {{ .Values.storage.s3.region }}
+{{- else if eq .Values.storage.type "gcs" }}
+  type: stow
+  stow:
+    kind: google
+    config:
+      json: ""
+      project_id: {{ .Values.storage.gcs.projectId }}
+      scopes: https://www.googleapis.com/auth/devstorage.read_write
+  container: {{ .Values.storage.bucketName | quote }}
+{{- else if eq .Values.storage.type "sandbox" }}
+  type: minio
+  container: {{ .Values.storage.bucketName | quote }}
+  connection:
+    access-key: minio
+    auth-type: accesskey
+    secret-key: miniostorage
+    disable-ssl: true
+    endpoint: http://minio.flyte.svc.cluster.local:9000
+    region: us-east-1
+{{- else if eq .Values.storage.type "custom" }}
+{{- with .Values.storage.custom -}}
+  {{ toYaml . | nindent 2 }}
+{{- end }}
+{{- end }}
+  limits:
+    maxDownloadMBs: 10
+{{- end }}
diff --git a/helm/templates/admin/configmap.yaml b/helm/templates/admin/configmap.yaml
@@ -23,9 +23,7 @@ data:
 {{- with .Values.configmap.server }}
   server.yaml: | {{ toYaml . | nindent 4 }}
 {{- end }}
-{{- with .Values.configmap.storage }}
-  storage.yaml: | {{ toYaml . | nindent 4 }}
-{{- end }}
+  storage.yaml: | {{ include "storage" . | nindent 4 }}
 {{- with .Values.configmap.task_resource_defaults }}
   task_resource_defaults.yaml: | {{ toYaml . | nindent 4 }}
 {{- end }}
diff --git a/helm/templates/datacatalog/configmap.yaml b/helm/templates/datacatalog/configmap.yaml
@@ -14,6 +14,5 @@ data:
 {{- with .Values.configmap.datacatalogServer }}
   server.yaml: | {{ toYaml . | nindent 4 }}
 {{- end }}
-{{- with .Values.configmap.storage }}
-  storage.yaml: | {{ toYaml . | nindent 4 }}
-{{- end }}
+  storage.yaml: | {{ include "storage" . | nindent 4 }}
+
diff --git a/helm/templates/propeller/configmap.yaml b/helm/templates/propeller/configmap.yaml
@@ -40,9 +40,11 @@ data:
   spark.yaml: | {{ toYaml . | nindent 4 }}
 {{- end }}
 {{- end }}
-{{- with .Values.configmap.storage }}
-  storage.yaml: | {{ toYaml . | nindent 4 }}
-{{- end }}
+  storage.yaml: | {{ include "storage" . | nindent 4 }}
+  cache.yaml: |
+    cache:
+      max_size_mbs: {{ .Values.flytepropeller.cacheSizeMbs }}
+      target_gc_percent: 70
 {{- with .Values.configmap.task_logs }}
   task_logs.yaml: | {{ toYaml . | nindent 4 }}
 {{- end }}
diff --git a/helm/values-eks.yaml b/helm/values-eks.yaml
@@ -272,15 +272,6 @@ configmap:
         - project: flytetester
         - project: spark-workflows
 
-
-  storage:
-    storage:
-      type: s3
-      container: s3-bucket-for-flyte
-      connection:
-        auth-type: iam
-        region: us-east-1
-
   task_resource_defaults:
     task_resources:
       defaults:

diff --git a/helm/values-sandbox.yaml b/helm/values-sandbox.yaml
@@ -113,18 +113,6 @@ configmap:
       host: postgres
       dbname: flyte_development
 
-  storage:
-    storage:
-      type: minio
-      container: my-s3-bucket
-      connection:
-        access-key: minio
-        auth-type: accesskey
-        secret-key: miniostorage
-        disable-ssl: true
-        endpoint: http://minio.flyte.svc.cluster.local:9000
-        region: us-east-1
-
   # Task default resources configuration
   task_resource_defaults:
     task_resources:

diff --git a/helm/values.yaml b/helm/values.yaml
@@ -104,6 +104,7 @@ flytepropeller:
       cpu: 10m
       ephemeral-storage: 50Mi
       memory: 50Mi
+  cacheSizeMbs: 0
   # -- Default regex string for searching configuration files
   configPath: /etc/flyte/config/*.yaml
   # -- Annotations for ServiceAccount attached to Flytepropeller pods
@@ -390,6 +391,25 @@ common:
     # --- Enable or disable creating Flyte namespace in template. Enable when using helm as template-engine only. Disable when using `helm install ...`.
     enabled: false
 
+#
+# STORAGE SETTINGS
+#
+
+storage:
+  # -- Sets the storage type. Supported values are sandbox, s3, gcs and custom.
+  type: sandbox
+  # -- bucketName defines the storage bucket flyte will use. Required for all types except for sandbox.
+  # bucketName: my-s3-bucket
+  # -- settings for storage type s3
+  s3:
+    region: us-east-1
+  # -- settings for storage type gcs
+  gcs:
+    # -- GCP project ID. Required for storage type gcs.
+    # projectId:
+  # -- Settings for storage type custom. See https://github.com/graymeta/stow for supported storage providers/settings.
+  custom: {}
+
 #
 # CONFIGMAPS SETTINGS
 #
@@ -482,15 +502,6 @@ configmap:
       httpPort: 8080
       grpcServerReflection: true
 
-  # Storage configuration
-  storage:
-    storage:
-      type: minio
-      container: my-s3-bucket
-      connection:
-        auth-type: accesskey
-        region: us-east-1
-
   # Task default resources configuration
   task_resource_defaults:
     # -- Task default resources parameters