Switch to using stow for all supported backends in flyte-binary chart (…

…#3195) Signed-off-by: Jeev B <[email protected]>
flyteorg · Dec 29, 2022 · 9c4ec0a · 9c4ec0a
1 parent 61e5fa4
commit 9c4ec0a
Show file tree

Hide file tree

Showing 10 changed files with 56 additions and 43 deletions.
diff --git a/charts/flyte-binary/README.md b/charts/flyte-binary/README.md
@@ -48,6 +48,7 @@ Chart for basic single Flyte executable deployment
 | configuration.storage.providerConfig.s3.endpoint | string | `""` |  |
 | configuration.storage.providerConfig.s3.region | string | `"us-east-1"` |  |
 | configuration.storage.providerConfig.s3.secretKey | string | `""` |  |
+| configuration.storage.providerConfig.s3.v2Signing | bool | `false` |  |
 | configuration.storage.userDataContainer | string | `"my-organization-flyte-container"` |  |
 | deployment.annotations | object | `{}` |  |
 | deployment.args | list | `[]` |  |

diff --git a/charts/flyte-binary/templates/configmap.yaml b/charts/flyte-binary/templates/configmap.yaml
@@ -76,36 +76,37 @@ data:
       rawoutput-prefix: {{ include "flyte-binary.configuration.storage.userDataPrefix" . }}
     {{- with .Values.configuration.storage }}
     storage:
-    {{- if eq "s3" .provider }}
-      {{- with .providerConfig.s3 }}
-      type: s3
-      connection:
-        region: {{ required "Region required for S3 storage provider" .region }}
-        disable-ssl: {{ .disableSSL }}
-        {{- if .endpoint }}
-        endpoint: {{ tpl .endpoint $ }}
-        {{- end }}
-        {{- if eq "iam" .authType }}
-        auth-type: "iam"
-        {{- else if eq "accesskey" .authType }}
-        auth-type: "accesskey"
-        access-key: {{ required "Access key required for S3 storage provider" .accessKey }}
-        secret-key: {{ required "Secret key required for S3 storage provider" .secretKey }}
-        {{- else }}
-        {{- printf "Invalid value for S3 storage provider authentication type. Expected one of (iam, accesskey), but got: %s" .authType | fail }}
-        {{- end }}
-      {{- end }}
-    {{- else if eq "gcs" .provider }}
       type: stow
       stow:
+        {{- if eq "s3" .provider }}
+        {{- with .providerConfig.s3 }}
+        kind: s3
+        config:
+          region: {{ required "Region required for S3 storage provider" .region }}
+          disable_ssl: {{ .disableSSL }}
+          v2_signing: {{ .v2Signing }}
+          {{- if .endpoint }}
+          endpoint: {{ tpl .endpoint $ }}
+          {{- end }}
+          {{- if eq "iam" .authType }}
+          auth_type: iam
+          {{- else if eq "accesskey" .authType }}
+          auth_type: accesskey
+          access_key_id: {{ required "Access key required for S3 storage provider" .accessKey }}
+          secret_key: {{ required "Secret key required for S3 storage provider" .secretKey }}
+          {{- else }}
+          {{- printf "Invalid value for S3 storage provider authentication type. Expected one of (iam, accesskey), but got: %s" .authType | fail }}
+          {{- end }}
+        {{- end }}
+        {{- else if eq "gcs" .provider }}
         kind: google
         config:
           json: ""
           project_id: {{ required "GCP project required for GCS storage provider" .providerConfig.gcs.project }}
           scopes: https://www.googleapis.com/auth/devstorage.read_write
-    {{- else }}
-    {{- printf "Invalid value for storage provider. Expected one of (s3, gcs), but got: %s" .provider | fail }}
-    {{- end }}
+        {{- else }}
+        {{- printf "Invalid value for storage provider. Expected one of (s3, gcs), but got: %s" .provider | fail }}
+        {{- end }}
       container: {{ required "Metadata container required" .metadataContainer }}
     {{- end }}
   {{- if.Values.configuration.auth.enabled }}

diff --git a/charts/flyte-binary/values.yaml b/charts/flyte-binary/values.yaml
@@ -42,6 +42,9 @@ configuration:
         region: "us-east-1"
         # disableSSL Switch to disable SSL for communicating with S3-compatible service
         disableSSL: false
+        # v2Signing Flag to sign requests with v2 signature
+        # Useful for s3-compatible blob stores (e.g. minio)
+        v2Signing: false
         # endpoint URL of S3-compatible service
         endpoint: ""
         # authType Type of authentication to use for connecting to S3-compatible service (Supported values: iam, accesskey)

diff --git a/charts/flyte-sandbox/Chart.lock b/charts/flyte-sandbox/Chart.lock
@@ -15,4 +15,4 @@ dependencies:
   repository: https://charts.bitnami.com/bitnami
   version: 12.1.0
 digest: sha256:398b480f861351e28080997e6fc1058519e33171494e52beae6a7acf9daa54d6
-generated: "2022-12-28T10:53:15.75989-08:00"
+generated: "2022-12-28T21:05:49.8877-08:00"
diff --git a/charts/flyte-sandbox/README.md b/charts/flyte-sandbox/README.md
@@ -40,6 +40,7 @@ A Helm chart for the Flyte local sandbox
 | flyte-binary.configuration.storage.providerConfig.s3.disableSSL | bool | `true` |  |
 | flyte-binary.configuration.storage.providerConfig.s3.endpoint | string | `"http://localhost:30002"` |  |
 | flyte-binary.configuration.storage.providerConfig.s3.secretKey | string | `"miniostorage"` |  |
+| flyte-binary.configuration.storage.providerConfig.s3.v2Signing | bool | `true` |  |
 | flyte-binary.configuration.storage.userDataContainer | string | `"my-s3-bucket"` |  |
 | flyte-binary.deployment.extraPodSpec.hostNetwork | bool | `true` |  |
 | flyte-binary.deployment.image.pullPolicy | string | `"Never"` |  |

diff --git a/charts/flyte-sandbox/charts/flyte-binary-v0.1.10.tgz b/charts/flyte-sandbox/charts/flyte-binary-v0.1.10.tgz
diff --git a/charts/flyte-sandbox/values.yaml b/charts/flyte-sandbox/values.yaml
@@ -23,6 +23,7 @@ flyte-binary:
       providerConfig:
         s3:
           disableSSL: true
+          v2Signing: true
           endpoint: http://localhost:30002
           authType: accesskey
           accessKey: minio

diff --git a/deployment/sandbox-binary/flyte_sandbox_binary_helm_generated.yaml b/deployment/sandbox-binary/flyte_sandbox_binary_helm_generated.yaml
@@ -103,11 +103,14 @@ data:
     propeller:
       rawoutput-prefix: s3://my-organization-flyte-container/data
     storage:
-      type: s3
-      connection:
-        region: us-east-1
-        disable-ssl: false
-        auth-type: "iam"
+      type: stow
+      stow:
+        kind: s3
+        config:
+          region: us-east-1
+          disable_ssl: false
+          v2_signing: false
+          auth_type: iam
       container: my-organization-flyte-container
 ---
 # Source: flyte-binary/templates/clusterrole.yaml
@@ -290,7 +293,7 @@ spec:
         app.kubernetes.io/name: flyte-binary
         app.kubernetes.io/instance: flyte
       annotations:
-        checksum/configuration: db5acb3814626b8953a925e83ed5b99dc9379794cc57eee9db0f5d36e50532b8
+        checksum/configuration: d80542e06800d62b504d192eb056bf05a4ab5506815a3ec6bd34def2ea92f2ff
         checksum/cluster-resource-templates: 7dfa59f3d447e9c099b8f8ffad3af466fecbc9cf9f8c97295d9634254a55d4ae
     spec:
       serviceAccountName: flyte-flyte-binary

diff --git a/docker/sandbox-bundled/manifests/complete.yaml b/docker/sandbox-bundled/manifests/complete.yaml
@@ -495,14 +495,17 @@ data:
     propeller:
       rawoutput-prefix: s3://my-s3-bucket/data
     storage:
-      type: s3
-      connection:
-        region: us-east-1
-        disable-ssl: true
-        endpoint: http://localhost:30002
-        auth-type: "accesskey"
-        access-key: minio
-        secret-key: miniostorage
+      type: stow
+      stow:
+        kind: s3
+        config:
+          region: us-east-1
+          disable_ssl: true
+          v2_signing: true
+          endpoint: http://localhost:30002
+          auth_type: accesskey
+          access_key_id: minio
+          secret_key: miniostorage
       container: my-s3-bucket
   010-inline-config.yaml: |
     plugins:
@@ -738,7 +741,7 @@ type: Opaque
 ---
 apiVersion: v1
 data:
-  haSharedSecret: TWw0WjRnTjJHekJNT3hDSQ==
+  haSharedSecret: Y2FvUHdKb1U3bENqU2o4RQ==
   proxyPassword: ""
   proxyUsername: ""
 kind: Secret
@@ -1040,7 +1043,7 @@ spec:
     metadata:
       annotations:
         checksum/config: 044987b193c168f87ad6b75510b710dae15de36461cb822559e13e6f3bf1789a
-        checksum/secret: 54cd610d8dd5a08c5375ae8d750da1de89177c74de718b1d4d6086b763cede19
+        checksum/secret: e852f9e87464782ba96aab43776db1b88cd2e0c14cfd9c4acdec46c6c5652ce1
       labels:
         app: docker-registry
         release: sandbox
@@ -1110,7 +1113,7 @@ spec:
     metadata:
       annotations:
         checksum/cluster-resource-templates: e2b58d2eaf0d0a1a949f4aec30ac4e4e46cbae8fed5b431e150116feec9d2c84
-        checksum/configuration: 9e2776c8a2f487b6dbe2f40ebef0a9462802c0c556e40ca0d3ad59fe6fb80d26
+        checksum/configuration: f43bf6c89d35eedd481af4d6552922a024b264337df298df9583000ffa78f5f2
         checksum/db-password-secret: 8ae0efad50be9c783ea14baad4edb9dcf623bc824516f5251ae2cda93396327b
       labels:
         app.kubernetes.io/instance: sandbox

diff --git a/docker/sandbox-bundled/manifests/dev.yaml b/docker/sandbox-bundled/manifests/dev.yaml
@@ -523,7 +523,7 @@ type: Opaque
 ---
 apiVersion: v1
 data:
-  haSharedSecret: NVpXeUNsTGQ4NWZzQ0tLZg==
+  haSharedSecret: R1o3S2xMUUhjTklKdUFhOA==
   proxyPassword: ""
   proxyUsername: ""
 kind: Secret
@@ -763,7 +763,7 @@ spec:
     metadata:
       annotations:
         checksum/config: 044987b193c168f87ad6b75510b710dae15de36461cb822559e13e6f3bf1789a
-        checksum/secret: 13520ef4524c22bab9c6fb02f2ad04cce98b90a1eaa2805ba0900f36b3aab34d
+        checksum/secret: 259b65b8d801f1fb4a4cf936da375c9268594733a3bc0baf9c4300aa5b6feafc
       labels:
         app: docker-registry
         release: sandbox