Skip to content

Commit

Permalink
Make automatic cleanup configurable (not documented)
Browse files Browse the repository at this point in the history
  • Loading branch information
andrewor14 committed Apr 4, 2014
1 parent ada45f0 commit cd72d19
Show file tree
Hide file tree
Showing 2 changed files with 9 additions and 5 deletions.
12 changes: 8 additions & 4 deletions core/src/main/scala/org/apache/spark/SparkContext.scala
Original file line number Diff line number Diff line change
Expand Up @@ -228,8 +228,12 @@ class SparkContext(
@volatile private[spark] var dagScheduler = new DAGScheduler(this)
dagScheduler.start()

private[spark] val cleaner = new ContextCleaner(this)
cleaner.start()
private[spark] val cleaner: Option[ContextCleaner] =
if (conf.getBoolean("spark.cleaner.automatic", true)) {
Some(new ContextCleaner(this))
} else None

cleaner.foreach(_.start())

postEnvironmentUpdate()

Expand Down Expand Up @@ -646,7 +650,7 @@ class SparkContext(
*/
def broadcast[T](value: T): Broadcast[T] = {
val bc = env.broadcastManager.newBroadcast[T](value, isLocal)
cleaner.registerBroadcastForCleanup(bc)
cleaner.foreach(_.registerBroadcastForCleanup(bc))
bc
}

Expand Down Expand Up @@ -841,7 +845,7 @@ class SparkContext(
dagScheduler = null
if (dagSchedulerCopy != null) {
metadataCleaner.cancel()
cleaner.stop()
cleaner.foreach(_.stop())
dagSchedulerCopy.stop()
listenerBus.stop()
taskScheduler = null
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ import org.apache.spark.scheduler.SchedulingMode.SchedulingMode
*
* THREADING: SchedulerBackends and task-submitting clients can call this class from multiple
* threads, so it needs locks in public API methods to maintain its state. In addition, some
* SchedulerBackends sycnchronize on themselves when they want to send events here, and then
* SchedulerBackends synchronize on themselves when they want to send events here, and then
* acquire a lock on us, so we need to make sure that we don't try to lock the backend while
* we are holding a lock on ourselves.
*/
Expand Down

0 comments on commit cd72d19

Please sign in to comment.