Skip to content

Commit

Permalink
update config name and docs
Browse files Browse the repository at this point in the history
  • Loading branch information
jiangxb1987 committed May 24, 2018
1 parent 4a4ab59 commit 2011eed
Show file tree
Hide file tree
Showing 4 changed files with 9 additions and 10 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@

import com.google.common.util.concurrent.MoreExecutors;
import org.junit.Test;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;

import org.apache.spark.network.util.MapConfigProvider;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -99,9 +99,9 @@ private[deploy] class Worker(
private val APP_DATA_RETENTION_SECONDS =
conf.getLong("spark.worker.cleanup.appDataTtl", 7 * 24 * 3600)

// Whether or not cleanup the non-shuffle files on executor death.
// Whether or not cleanup the non-shuffle files on executor exits.
private val CLEANUP_NON_SHUFFLE_FILES_ENABLED =
conf.getBoolean("spark.storage.cleanupFilesAfterExecutorDeath", true)
conf.getBoolean("spark.storage.cleanupFilesAfterExecutorExit", true)

private val testing: Boolean = sys.props.contains("spark.testing")
private var master: Option[RpcEndpointRef] = None
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -212,18 +212,18 @@ class WorkerSuite extends SparkFunSuite with Matchers with BeforeAndAfter {
}
}

test("cleanup non-shuffle files after executor death when config " +
"spark.storage.cleanupFilesAfterExecutorDeath=true") {
test("cleanup non-shuffle files after executor exits when config " +
"spark.storage.cleanupFilesAfterExecutorExit=true") {
testCleanupFilesWithConfig(true)
}

test("don't cleanup non-shuffle files after executor death when config " +
"spark.storage.cleanupFilesAfterExecutorDeath=false") {
test("don't cleanup non-shuffle files after executor exits when config " +
"spark.storage.cleanupFilesAfterExecutorExit=false") {
testCleanupFilesWithConfig(false)
}

private def testCleanupFilesWithConfig(value: Boolean) = {
val conf = new SparkConf().set("spark.storage.cleanupFilesAfterExecutorDeath", value.toString)
val conf = new SparkConf().set("spark.storage.cleanupFilesAfterExecutorExit", value.toString)

val cleanupCalled = new AtomicBoolean(false)
when(shuffleService.executorRemoved(any[String], any[String])).thenAnswer(new Answer[Unit] {
Expand Down
4 changes: 2 additions & 2 deletions docs/spark-standalone.md
Original file line number Diff line number Diff line change
Expand Up @@ -255,11 +255,11 @@ SPARK_WORKER_OPTS supports the following system properties:
</td>
</tr>
<tr>
<td><code>spark.storage.cleanupFilesAfterExecutorDeath</code></td>
<td><code>spark.storage.cleanupFilesAfterExecutorExit</code></td>
<td>true</td>
<td>
Enable cleanup non-shuffle files(such as temp. shuffle blocks, cached RDD/broadcast blocks,
spill files, etc) of worker directories following executor death. Note that this doesn't
spill files, etc) of worker directories following executor exits. Note that this doesn't
overlap with `spark.worker.cleanup.enabled`, as this enables cleanup of non-shuffle files in
local directories of a dead executor, while `spark.worker.cleanup.enabled` enables cleanup of
all files/subdirectories of a stopped and timeout application.
Expand Down

0 comments on commit 2011eed

Please sign in to comment.