diff --git a/helm/templates/_helpers.tpl b/helm/templates/_helpers.tpl index 244619bbad..3ae15cccf4 100644 --- a/helm/templates/_helpers.tpl +++ b/helm/templates/_helpers.tpl @@ -159,3 +159,39 @@ app.kubernetes.io/managed-by: {{ .Release.Service }} name: {{ . }} {{- end }} {{- end }} + +{{- define "storage" -}} +storage: +{{- if eq .Values.storage.type "s3" }} + type: s3 + container: {{ .Values.storage.bucketName | quote }} + connection: + auth-type: iam + region: {{ .Values.storage.s3.region }} +{{- else if eq .Values.storage.type "gcs" }} + type: stow + stow: + kind: google + config: + json: "" + project_id: {{ .Values.storage.gcs.projectId }} + scopes: https://www.googleapis.com/auth/devstorage.read_write + container: {{ .Values.storage.bucketName | quote }} +{{- else if eq .Values.storage.type "sandbox" }} + type: minio + container: {{ .Values.storage.bucketName | quote }} + connection: + access-key: minio + auth-type: accesskey + secret-key: miniostorage + disable-ssl: true + endpoint: http://minio.flyte.svc.cluster.local:9000 + region: us-east-1 +{{- else if eq .Values.storage.type "custom" }} +{{- with .Values.storage.custom -}} + {{ toYaml . | nindent 2 }} +{{- end }} +{{- end }} + limits: + maxDownloadMBs: 10 +{{- end }} \ No newline at end of file diff --git a/helm/templates/admin/configmap.yaml b/helm/templates/admin/configmap.yaml index a061058caa..d344fe9172 100644 --- a/helm/templates/admin/configmap.yaml +++ b/helm/templates/admin/configmap.yaml @@ -23,9 +23,7 @@ data: {{- with .Values.configmap.server }} server.yaml: | {{ toYaml . | nindent 4 }} {{- end }} -{{- with .Values.configmap.storage }} - storage.yaml: | {{ toYaml . | nindent 4 }} -{{- end }} + storage.yaml: | {{ include "storage" . | nindent 4 }} {{- with .Values.configmap.task_resource_defaults }} task_resource_defaults.yaml: | {{ toYaml . | nindent 4 }} {{- end }} diff --git a/helm/templates/datacatalog/configmap.yaml b/helm/templates/datacatalog/configmap.yaml index dce73e47e4..0e35f32cef 100644 --- a/helm/templates/datacatalog/configmap.yaml +++ b/helm/templates/datacatalog/configmap.yaml @@ -14,6 +14,5 @@ data: {{- with .Values.configmap.datacatalogServer }} server.yaml: | {{ toYaml . | nindent 4 }} {{- end }} -{{- with .Values.configmap.storage }} - storage.yaml: | {{ toYaml . | nindent 4 }} -{{- end }} + storage.yaml: | {{ include "storage" . | nindent 4 }} + diff --git a/helm/templates/propeller/configmap.yaml b/helm/templates/propeller/configmap.yaml index 70b4d58a0e..593d968248 100644 --- a/helm/templates/propeller/configmap.yaml +++ b/helm/templates/propeller/configmap.yaml @@ -40,9 +40,11 @@ data: spark.yaml: | {{ toYaml . | nindent 4 }} {{- end }} {{- end }} -{{- with .Values.configmap.storage }} - storage.yaml: | {{ toYaml . | nindent 4 }} -{{- end }} + storage.yaml: | {{ include "storage" . | nindent 4 }} + cache.yaml: | + cache: + max_size_mbs: {{ .Values.flytepropeller.cacheSizeMbs }} + target_gc_percent: 70 {{- with .Values.configmap.task_logs }} task_logs.yaml: | {{ toYaml . | nindent 4 }} {{- end }} diff --git a/helm/values-eks.yaml b/helm/values-eks.yaml index 3089b72968..77e49d0f22 100644 --- a/helm/values-eks.yaml +++ b/helm/values-eks.yaml @@ -272,15 +272,6 @@ configmap: - project: flytetester - project: spark-workflows - - storage: - storage: - type: s3 - container: s3-bucket-for-flyte - connection: - auth-type: iam - region: us-east-1 - task_resource_defaults: task_resources: defaults: diff --git a/helm/values-sandbox.yaml b/helm/values-sandbox.yaml index dff617bb7b..1965aea392 100644 --- a/helm/values-sandbox.yaml +++ b/helm/values-sandbox.yaml @@ -113,18 +113,6 @@ configmap: host: postgres dbname: flyte_development - storage: - storage: - type: minio - container: my-s3-bucket - connection: - access-key: minio - auth-type: accesskey - secret-key: miniostorage - disable-ssl: true - endpoint: http://minio.flyte.svc.cluster.local:9000 - region: us-east-1 - # Task default resources configuration task_resource_defaults: task_resources: diff --git a/helm/values.yaml b/helm/values.yaml index 5f45bde061..b18ef46515 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -104,6 +104,7 @@ flytepropeller: cpu: 10m ephemeral-storage: 50Mi memory: 50Mi + cacheSizeMbs: 0 # -- Default regex string for searching configuration files configPath: /etc/flyte/config/*.yaml # -- Annotations for ServiceAccount attached to Flytepropeller pods @@ -390,6 +391,25 @@ common: # --- Enable or disable creating Flyte namespace in template. Enable when using helm as template-engine only. Disable when using `helm install ...`. enabled: false +# +# STORAGE SETTINGS +# + +storage: + # -- Sets the storage type. Supported values are sandbox, s3, gcs and custom. + type: sandbox + # -- bucketName defines the storage bucket flyte will use. Required for all types except for sandbox. + # bucketName: my-s3-bucket + # -- settings for storage type s3 + s3: + region: us-east-1 + # -- settings for storage type gcs + gcs: + # -- GCP project ID. Required for storage type gcs. + # projectId: + # -- Settings for storage type custom. See https://github:com/graymeta/stow for supported storage providers/settings. + custom: {} + # # CONFIGMAPS SETTINGS # @@ -482,15 +502,6 @@ configmap: httpPort: 8080 grpcServerReflection: true - # Storage configuration - storage: - storage: - type: minio - container: my-s3-bucket - connection: - auth-type: accesskey - region: us-east-1 - # Task default resources configuration task_resource_defaults: # -- Task default resources parameters