From 4ecf84100e22224aed204c3c4251c6ab20ff8bf6 Mon Sep 17 00:00:00 2001 From: William Benton Date: Fri, 14 Mar 2014 12:33:33 -0500 Subject: [PATCH] Make proactive serializability checking optional. SparkContext.clean uses ClosureCleaner's proactive serializability checking by default. This commit adds an overloaded clean method to SparkContext that allows clients to specify that serializability checking should not occur as part of closure cleaning. --- .../src/main/scala/org/apache/spark/SparkContext.scala | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala b/core/src/main/scala/org/apache/spark/SparkContext.scala index a1003b7925715..c6f3b7a8494f8 100644 --- a/core/src/main/scala/org/apache/spark/SparkContext.scala +++ b/core/src/main/scala/org/apache/spark/SparkContext.scala @@ -1032,7 +1032,15 @@ class SparkContext( * (removes unreferenced variables in $outer's, updates REPL variables) */ private[spark] def clean[F <: AnyRef](f: F): F = { - ClosureCleaner.clean(f) + clean(f, true) + } + + /** + * Clean a closure to make it ready to serialized and send to tasks + * (removes unreferenced variables in $outer's, updates REPL variables) + */ + private[spark] def clean[F <: AnyRef](f: F, checkSerializable: Boolean): F = { + ClosureCleaner.clean(f, checkSerializable) f }