remove hosts variable and unify Vault URI (apache#65)

jlopezmalla · Sep 19, 2017 · 118985a · 118985a
1 parent 6afaa14
commit 118985a
Show file tree

Hide file tree

Showing 6 changed files with 11 additions and 13 deletions.
diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
@@ -710,7 +710,7 @@ object SparkSubmit extends CommandLineUtils {
     }
 
     val vaultProtocol = args.sparkProperties.get("spark.secret.vault.protocol")
-    val vaultHost = args.sparkProperties.get("spark.secret.vault.hosts")
+    val vaultHost = args.sparkProperties.get("spark.secret.vault.host")
     val vaultPort = args.sparkProperties.get("spark.secret.vault.port")
 
     val vaultUrlParams = (vaultProtocol, vaultHost, vaultPort)

diff --git a/core/src/main/scala/org/apache/spark/executor/CoarseGrainedExecutorBackend.scala b/core/src/main/scala/org/apache/spark/executor/CoarseGrainedExecutorBackend.scala
@@ -287,8 +287,7 @@ private[spark] object CoarseGrainedExecutorBackend extends Logging {
       printUsageAndExit()
     }
     ConfigSecurity.prepareEnvironment(scala.util.Try{
-      VaultHelper.getRealToken(ConfigSecurity.vaultUri.get,
-        sys.env("VAULT_TEMP_TOKEN"))}.toOption)
+      VaultHelper.getRealToken(sys.env("VAULT_URI"), sys.env("VAULT_TEMP_TOKEN"))}.toOption)
 
     run(driverUrl, executorId, hostname, cores, appId, workerUrl, userClassPath)
     System.exit(0)

diff --git a/core/src/main/scala/org/apache/spark/security/ConfigSecurity.scala b/core/src/main/scala/org/apache/spark/security/ConfigSecurity.scala
@@ -23,9 +23,8 @@ import org.apache.spark.internal.Logging
 object ConfigSecurity extends Logging{
 
   var vaultToken: Option[String] = None
-  val vaultHost: Option[String] = sys.env.get("VAULT_HOST")
   val vaultUri: Option[String] = getVaultUri(sys.env.get("VAULT_PROTOCOL"),
-    vaultHost, sys.env.get("VAULT_PORT"))
+    sys.env.get("VAULT_HOST"), sys.env.get("VAULT_PORT"))
 
     def getVaultUri(vaultProtocol: Option[String],
                     vaultHost: Option[String],

diff --git a/...ce-managers/mesos/src/main/scala/org/apache/spark/deploy/rest/mesos/MesosRestServer.scala b/...ce-managers/mesos/src/main/scala/org/apache/spark/deploy/rest/mesos/MesosRestServer.scala
@@ -47,14 +47,14 @@ private[spark] class MesosRestServer(
 
   protected lazy val token = {
     require((masterConf.getOption("spark.secret.vault.protocol").isDefined
-      && masterConf.getOption("spark.secret.vault.hosts").isDefined
+      && masterConf.getOption("spark.secret.vault.host").isDefined
       && masterConf.getOption("spark.secret.vault.port").isDefined),
       "You are attempting to login in Vault but no Vault obtained," +
         " please configure spark.secret.vault.protocol," +
         " spark.vault.host and spark.secret.vault.port" +
         " in your Stratio Spark Dispatcher instance")
     val vaultUrl = s"${masterConf.get("spark.secret.vault.protocol")}://" +
-      s"${masterConf.get("spark.secret.vault.hosts").split(",")
+      s"${masterConf.get("spark.secret.vault.host").split(",")
         .map(host => s"$host:${masterConf.get("spark.secret.vault.port")}").mkString(",")}"
     VaultHelper.getTokenFromAppRole(vaultUrl,
       sys.env("VAULT_ROLE_ID"),
@@ -120,11 +120,11 @@ private[mesos] class MesosSubmitRequestServlet(
     val sparkJavaOpts = Utils.sparkJavaOpts(conf)
     val javaOpts = sparkJavaOpts ++ extraJavaOpts
     val securitySparkOpts: Map[String, String] = {
-        if (sparkProperties.get("spark.secret.vault.hosts").isDefined
+        if (sparkProperties.get("spark.secret.vault.host").isDefined
           && sparkProperties.get("spark.secret.vault.protocol").isDefined
           && sparkProperties.get("spark.secret.vault.port").isDefined) {
             val vaultUrl = s"${sparkProperties("spark.secret.vault.protocol")}://" +
-                s"${sparkProperties("spark.secret.vault.hosts").split(",")
+                s"${sparkProperties("spark.secret.vault.host").split(",")
                     .map(host => s"$host:${sparkProperties("spark.secret.vault.port")}").mkString(",")}"
             (sparkProperties.get("spark.secret.vault.role"),
                 sys.env.get("VAULT_ROLE"),

diff --git a/...mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosClusterScheduler.scala b/...mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosClusterScheduler.scala
@@ -701,11 +701,11 @@ private[spark] class MesosClusterScheduler(
 
           var sparkProperties = state.driverDescription.conf.getAll.toMap
           if (sparkProperties.get("spark.secret.vault.protocol").isDefined
-            && sparkProperties.get("spark.secret.vault.hosts").isDefined
+            && sparkProperties.get("spark.secret.vault.host").isDefined
             && sparkProperties.get("spark.secret.vault.port").isDefined)
           {
             val vaultUrl = s"${sparkProperties("spark.secret.vault.protocol")}://" +
-                s"${sparkProperties("spark.secret.vault.hosts").split(",")
+                s"${sparkProperties("spark.secret.vault.host").split(",")
                   .map(host => s"$host:${sparkProperties("spark.secret.vault.port")}")
                   .mkString(",")}"
             val role = sparkProperties("spark.secret.vault.role")

diff --git a/...n/scala/org/apache/spark/scheduler/cluster/mesos/MesosCoarseGrainedSchedulerBackend.scala b/...n/scala/org/apache/spark/scheduler/cluster/mesos/MesosCoarseGrainedSchedulerBackend.scala
@@ -219,8 +219,8 @@ private[spark] class MesosCoarseGrainedSchedulerBackend(
           ConfigSecurity.vaultToken.get))
         .build())
       environment.addVariables(Environment.Variable.newBuilder()
-        .setName("VAULT_HOST")
-        .setValue(ConfigSecurity.vaultHost.get)
+        .setName("VAULT_URI")
+        .setValue(ConfigSecurity.vaultUri.get)
         .build())
     }