Updating Spark config to force default credentials chain for AWS (#674)

flyteorg · Jan 21, 2021 · 0e2dd72 · 0e2dd72
1 parent 17352c7
commit 0e2dd72
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 2 deletions.
diff --git a/deployment/eks/flyte_generated.yaml b/deployment/eks/flyte_generated.yaml
@@ -8285,6 +8285,10 @@ data:
     plugins:
       spark:
         spark-config-default:
+          # We override the default credentials chain provider for Hadoop so that
+          # it can use the serviceAccount based IAM role or ec2 metadata based.
+          # This is more in line with how AWS works
+          - spark.hadoop.fs.s3a.aws.credentials.provider: "com.amazonaws.auth.DefaultAWSCredentialsProviderChain"
           - spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version: "2"
           - spark.kubernetes.allocation.batch.size: "50"
           - spark.hadoop.fs.s3a.acl.default: "BucketOwnerFullControl"
@@ -8333,7 +8337,7 @@ kind: ConfigMap
 metadata:
   annotations: {}
   labels: {}
-  name: flyte-propeller-config-88bdc8dc9f
+  name: flyte-propeller-config-ct29cf4cch
   namespace: flyte
 ---
 apiVersion: v1
@@ -8755,7 +8759,7 @@ spec:
       serviceAccountName: flytepropeller
       volumes:
       - configMap:
-          name: flyte-propeller-config-88bdc8dc9f
+          name: flyte-propeller-config-ct29cf4cch
         name: config-volume
 ---
 apiVersion: apps/v1

diff --git a/kustomize/overlays/eks/config/propeller/plugins/spark.yaml b/kustomize/overlays/eks/config/propeller/plugins/spark.yaml
@@ -1,6 +1,10 @@
 plugins:
   spark:
     spark-config-default:
+      # We override the default credentials chain provider for Hadoop so that
+      # it can use the serviceAccount based IAM role or ec2 metadata based.
+      # This is more in line with how AWS works
+      - spark.hadoop.fs.s3a.aws.credentials.provider: "com.amazonaws.auth.DefaultAWSCredentialsProviderChain"
       - spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version: "2"
       - spark.kubernetes.allocation.batch.size: "50"
       - spark.hadoop.fs.s3a.acl.default: "BucketOwnerFullControl"