From 9701cf8eadb68bf44a9b2d163fb1c4c6d1dc4cd3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6ren=20Brunk?= Date: Fri, 19 Mar 2021 23:51:29 +0100 Subject: [PATCH] Helm - Updating Spark config to force default credentials chain for AWS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Based on the following kustomize change: - 0e2dd723 Updating Spark config to force default credentials chain for AWS (#674) Signed-off-by: Sören Brunk --- helm/values-eks.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/helm/values-eks.yaml b/helm/values-eks.yaml index 175dd2fcc4..178a1e1504 100644 --- a/helm/values-eks.yaml +++ b/helm/values-eks.yaml @@ -504,6 +504,10 @@ configmap: plugins: spark: spark-config-default: + - # We override the default credentials chain provider for Hadoop so that + # it can use the serviceAccount based IAM role or ec2 metadata based. + # This is more in line with how AWS works + - spark.hadoop.fs.s3a.aws.credentials.provider: "com.amazonaws.auth.DefaultAWSCredentialsProviderChain" - spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version: "2" - spark.kubernetes.allocation.batch.size: "50" - spark.hadoop.fs.s3a.acl.default: "BucketOwnerFullControl"