From 00a42e7c3d08ba01265db8c7329f0ba2148ce41a Mon Sep 17 00:00:00 2001 From: root Date: Wed, 14 Jun 2017 15:56:24 -0700 Subject: [PATCH 1/3] [SPARK-21099][Spark Core] INFO Log Message Using Incorrect Executor Idle Timeout Value --- .../scala/org/apache/spark/ExecutorAllocationManager.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala b/core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala index fcc72ff49276d..8a82d91980b40 100644 --- a/core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala +++ b/core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala @@ -432,8 +432,10 @@ private[spark] class ExecutorAllocationManager( if (testing || executorsRemoved.nonEmpty) { executorsRemoved.foreach { removedExecutorId => newExecutorTotal -= 1 + val hasCachedBlocks = SparkEnv.get.blockManager.master.hasCachedBlocks(executorId); + val timeout = if (hasCachedBlocks) cachedExecutorIdleTimeoutS else executorIdleTimeoutS; logInfo(s"Removing executor $removedExecutorId because it has been idle for " + - s"$executorIdleTimeoutS seconds (new desired total will be $newExecutorTotal)") + s"$timeout seconds (new desired total will be $newExecutorTotal)") executorsPendingToRemove.add(removedExecutorId) } executorsRemoved From 0f1c467ffaa6240c137b98baf76d6af6fb5f3e57 Mon Sep 17 00:00:00 2001 From: Hazem Mahmoud Date: Thu, 15 Jun 2017 10:42:23 -0700 Subject: [PATCH 2/3] [SPARK-21099][Core] INFO Log Message Using Incorrect Executor Idle Timeout --- .../scala/org/apache/spark/ExecutorAllocationManager.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala b/core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala index 8a82d91980b40..caff6297a7e22 100644 --- a/core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala +++ b/core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala @@ -432,8 +432,8 @@ private[spark] class ExecutorAllocationManager( if (testing || executorsRemoved.nonEmpty) { executorsRemoved.foreach { removedExecutorId => newExecutorTotal -= 1 - val hasCachedBlocks = SparkEnv.get.blockManager.master.hasCachedBlocks(executorId); - val timeout = if (hasCachedBlocks) cachedExecutorIdleTimeoutS else executorIdleTimeoutS; + val hasCachedBlocks = SparkEnv.get.blockManager.master.hasCachedBlocks(removedExecutorId) + val timeout = if (hasCachedBlocks) cachedExecutorIdleTimeoutS else executorIdleTimeoutS logInfo(s"Removing executor $removedExecutorId because it has been idle for " + s"$timeout seconds (new desired total will be $newExecutorTotal)") executorsPendingToRemove.add(removedExecutorId) From 871ed1df8995cce8ddfa9a5dc12d40b8e530b7f0 Mon Sep 17 00:00:00 2001 From: Hazem Mahmoud Date: Sun, 2 Jul 2017 11:25:54 -0700 Subject: [PATCH 3/3] [SPARK-21099][Core] INFO Log Message Using Incorrect Executor Idle Timeout --- .../main/scala/org/apache/spark/ExecutorAllocationManager.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala b/core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala index caff6297a7e22..1e9d992d09db7 100644 --- a/core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala +++ b/core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala @@ -435,7 +435,7 @@ private[spark] class ExecutorAllocationManager( val hasCachedBlocks = SparkEnv.get.blockManager.master.hasCachedBlocks(removedExecutorId) val timeout = if (hasCachedBlocks) cachedExecutorIdleTimeoutS else executorIdleTimeoutS logInfo(s"Removing executor $removedExecutorId because it has been idle for " + - s"$timeout seconds (new desired total will be $newExecutorTotal)") + s"${if (SparkEnv.get.blockManager.master.hasCachedBlocks(executorId)) cachedExecutorIdleTimeoutS else executorIdleTimeoutS} seconds (new desired total will be $newExecutorTotal)") executorsPendingToRemove.add(removedExecutorId) } executorsRemoved