[SPARK-21099][Spark Core] INFO Log Message Using Incorrect Executor I…

…dle Timeout Value
apache · Jun 14, 2017 · 00a42e7 · 00a42e7
1 parent 77a2fc5
commit 00a42e7
Showing 1 changed file with 3 additions and 1 deletion.
diff --git a/core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala b/core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala
@@ -432,8 +432,10 @@ private[spark] class ExecutorAllocationManager(
     if (testing || executorsRemoved.nonEmpty) {
       executorsRemoved.foreach { removedExecutorId =>
         newExecutorTotal -= 1
+        val hasCachedBlocks = SparkEnv.get.blockManager.master.hasCachedBlocks(executorId);
+        val timeout = if (hasCachedBlocks) cachedExecutorIdleTimeoutS else executorIdleTimeoutS;
         logInfo(s"Removing executor $removedExecutorId because it has been idle for " +
-          s"$executorIdleTimeoutS seconds (new desired total will be $newExecutorTotal)")
+          s"$timeout seconds (new desired total will be $newExecutorTotal)")
         executorsPendingToRemove.add(removedExecutorId)
       }
       executorsRemoved