Merge remote-tracking branch 'upstream/master' into avro-hadoop2

README.md

@@ -26,7 +26,7 @@ To build Spark and its example programs, run:
 
 (You do not need to do this if you downloaded a pre-built package.)
 More detailed documentation is available from the project site, at
-["Building Spark with Maven"](http://spark.apache.org/docs/latest/building-spark.html).
+["Building Spark"](http://spark.apache.org/docs/latest/building-spark.html).
 
 ## Interactive Scala Shell
 

core/pom.xml

@@ -122,6 +122,16 @@
       <artifactId>jetty-http</artifactId>
       <scope>compile</scope>
     </dependency>
+    <dependency>
+      <groupId>org.eclipse.jetty</groupId>
+      <artifactId>jetty-continuation</artifactId>
+      <scope>compile</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.eclipse.jetty</groupId>
+      <artifactId>jetty-servlet</artifactId>
+      <scope>compile</scope>
+    </dependency>
 
     <dependency>
       <groupId>org.apache.commons</groupId>
@@ -377,7 +387,7 @@
               <overWriteIfNewer>true</overWriteIfNewer>
               <useSubDirectoryPerType>true</useSubDirectoryPerType>
               <includeArtifactIds>
-                guava,jetty-io,jetty-http,jetty-plus,jetty-util,jetty-server
+                guava,jetty-io,jetty-servlet,jetty-continuation,jetty-http,jetty-plus,jetty-util,jetty-server
               </includeArtifactIds>
               <silent>true</silent>
             </configuration>

core/src/main/java/org/apache/spark/SparkFirehoseListener.java

@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark;
+
+import org.apache.spark.scheduler.*;
+
+/**
+ * Class that allows users to receive all SparkListener events.
+ * Users should override the onEvent method.
+ *
+ * This is a concrete Java class in order to ensure that we don't forget to update it when adding
+ * new methods to SparkListener: forgetting to add a method will result in a compilation error (if
+ * this was a concrete Scala class, default implementations of new event handlers would be inherited
+ * from the SparkListener trait).
+ */
+public class SparkFirehoseListener implements SparkListener {
+
+    public void onEvent(SparkListenerEvent event) { }
+
+    @Override
+    public final void onStageCompleted(SparkListenerStageCompleted stageCompleted) {
+        onEvent(stageCompleted);
+    }
+
+    @Override
+    public final void onStageSubmitted(SparkListenerStageSubmitted stageSubmitted) {
+        onEvent(stageSubmitted);
+    }
+
+    @Override
+    public final void onTaskStart(SparkListenerTaskStart taskStart) {
+        onEvent(taskStart);
+    }
+
+    @Override
+    public final void onTaskGettingResult(SparkListenerTaskGettingResult taskGettingResult) {
+        onEvent(taskGettingResult);
+    }
+
+    @Override
+    public final void onTaskEnd(SparkListenerTaskEnd taskEnd) {
+        onEvent(taskEnd);
+    }
+
+    @Override
+    public final void onJobStart(SparkListenerJobStart jobStart) {
+        onEvent(jobStart);
+    }
+
+    @Override
+    public final void onJobEnd(SparkListenerJobEnd jobEnd) {
+        onEvent(jobEnd);
+    }
+
+    @Override
+    public final void onEnvironmentUpdate(SparkListenerEnvironmentUpdate environmentUpdate) {
+        onEvent(environmentUpdate);
+    }
+
+    @Override
+    public final void onBlockManagerAdded(SparkListenerBlockManagerAdded blockManagerAdded) {
+        onEvent(blockManagerAdded);
+    }
+
+    @Override
+    public final void onBlockManagerRemoved(SparkListenerBlockManagerRemoved blockManagerRemoved) {
+        onEvent(blockManagerRemoved);
+    }
+
+    @Override
+    public final void onUnpersistRDD(SparkListenerUnpersistRDD unpersistRDD) {
+        onEvent(unpersistRDD);
+    }
+
+    @Override
+    public final void onApplicationStart(SparkListenerApplicationStart applicationStart) {
+        onEvent(applicationStart);
+    }
+
+    @Override
+    public final void onApplicationEnd(SparkListenerApplicationEnd applicationEnd) {
+        onEvent(applicationEnd);
+    }
+
+    @Override
+    public final void onExecutorMetricsUpdate(
+            SparkListenerExecutorMetricsUpdate executorMetricsUpdate) {
+        onEvent(executorMetricsUpdate);
+    }
+
+    @Override
+    public final void onExecutorAdded(SparkListenerExecutorAdded executorAdded) {
+        onEvent(executorAdded);
+    }
+
+    @Override
+    public final void onExecutorRemoved(SparkListenerExecutorRemoved executorRemoved) {
+        onEvent(executorRemoved);
+    }
+}

core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala

@@ -49,6 +49,7 @@ import org.apache.spark.scheduler._
  *   spark.dynamicAllocation.enabled - Whether this feature is enabled
  *   spark.dynamicAllocation.minExecutors - Lower bound on the number of executors
  *   spark.dynamicAllocation.maxExecutors - Upper bound on the number of executors
+ *   spark.dynamicAllocation.initialExecutors - Number of executors to start with
  *
  *   spark.dynamicAllocation.schedulerBacklogTimeout (M) -
  *     If there are backlogged tasks for this duration, add new executors
@@ -70,9 +71,10 @@ private[spark] class ExecutorAllocationManager(
 
   import ExecutorAllocationManager._
 
-  // Lower and upper bounds on the number of executors. These are required.
-  private val minNumExecutors = conf.getInt("spark.dynamicAllocation.minExecutors", -1)
-  private val maxNumExecutors = conf.getInt("spark.dynamicAllocation.maxExecutors", -1)
+  // Lower and upper bounds on the number of executors.
+  private val minNumExecutors = conf.getInt("spark.dynamicAllocation.minExecutors", 0)
+  private val maxNumExecutors = conf.getInt("spark.dynamicAllocation.maxExecutors",
+    Integer.MAX_VALUE)
 
   // How long there must be backlogged tasks for before an addition is triggered
   private val schedulerBacklogTimeout = conf.getLong(
@@ -132,10 +134,10 @@ private[spark] class ExecutorAllocationManager(
    */
   private def validateSettings(): Unit = {
     if (minNumExecutors < 0 || maxNumExecutors < 0) {
-      throw new SparkException("spark.dynamicAllocation.{min/max}Executors must be set!")
+      throw new SparkException("spark.dynamicAllocation.{min/max}Executors must be positive!")
     }
-    if (minNumExecutors == 0 || maxNumExecutors == 0) {
-      throw new SparkException("spark.dynamicAllocation.{min/max}Executors cannot be 0!")
+    if (maxNumExecutors == 0) {
+      throw new SparkException("spark.dynamicAllocation.maxExecutors cannot be 0!")
     }
     if (minNumExecutors > maxNumExecutors) {
       throw new SparkException(s"spark.dynamicAllocation.minExecutors ($minNumExecutors) must " +

core/src/main/scala/org/apache/spark/HttpFileServer.scala

@@ -36,7 +36,7 @@ private[spark] class HttpFileServer(
   var serverUri : String = null
 
   def initialize() {
-    baseDir = Utils.createTempDir()
+    baseDir = Utils.createTempDir(Utils.getLocalDir(conf), "httpd")
     fileDir = new File(baseDir, "files")
     jarDir = new File(baseDir, "jars")
     fileDir.mkdir()

core/src/main/scala/org/apache/spark/HttpServer.scala

@@ -19,6 +19,7 @@ package org.apache.spark
 
 import java.io.File
 
+import org.eclipse.jetty.server.ssl.SslSocketConnector
 import org.eclipse.jetty.util.security.{Constraint, Password}
 import org.eclipse.jetty.security.authentication.DigestAuthenticator
 import org.eclipse.jetty.security.{ConstraintMapping, ConstraintSecurityHandler, HashLoginService}
@@ -72,7 +73,10 @@ private[spark] class HttpServer(
    */
   private def doStart(startPort: Int): (Server, Int) = {
     val server = new Server()
-    val connector = new SocketConnector
+
+    val connector = securityManager.fileServerSSLOptions.createJettySslContextFactory()
+      .map(new SslSocketConnector(_)).getOrElse(new SocketConnector)
+
     connector.setMaxIdleTime(60 * 1000)
     connector.setSoLingerTime(-1)
     connector.setPort(startPort)
@@ -149,13 +153,14 @@ private[spark] class HttpServer(
   }
 
   /**
-   * Get the URI of this HTTP server (http://host:port)
+   * Get the URI of this HTTP server (http://host:port or https://host:port)
    */
   def uri: String = {
     if (server == null) {
       throw new ServerStateException("Server is not started")
     } else {
-      "http://" + Utils.localIpAddress + ":" + port
+      val scheme = if (securityManager.fileServerSSLOptions.enabled) "https" else "http"
+      s"$scheme://${Utils.localIpAddress}:$port"
     }
   }
 }