Fix issues with the repl class loader

witgo · Apr 8, 2014 · bb8d179 · bb8d179
1 parent 241b03d
commit bb8d179
Show file tree

Hide file tree

Showing 3 changed files with 16 additions and 16 deletions.
diff --git a/core/src/main/scala/org/apache/spark/executor/Executor.scala b/core/src/main/scala/org/apache/spark/executor/Executor.scala
@@ -314,11 +314,12 @@ private[spark] class Executor(
     val classUri = conf.get("spark.repl.class.uri", null)
     if (classUri != null) {
       logInfo("Using REPL class URI: " + classUri)
+      val userClassPathFirst: java.lang.Boolean = conf.getBoolean("spark.classpath.userClassPathFirst", false)
       try {
-        val klass = Class.forName("org.apache.spark.repl.ExecutorClassLoader")
+        val klass = Class.forName("org.apache.spark.repl.FlexibleExecutorClassLoader")
           .asInstanceOf[Class[_ <: ClassLoader]]
-        val constructor = klass.getConstructor(classOf[String], classOf[ClassLoader])
-        constructor.newInstance(classUri, parent)
+        val constructor = klass.getConstructor(classOf[String], classOf[ClassLoader], classOf[Boolean])
+        constructor.newInstance(classUri, parent, userClassPathFirst)
       } catch {
         case _: ClassNotFoundException =>
           logError("Could not find org.apache.spark.repl.ExecutorClassLoader on classpath!")

diff --git a/repl/src/main/scala/org/apache/spark/repl/ExecutorClassLoader.scala b/repl/src/main/scala/org/apache/spark/repl/ExecutorClassLoader.scala
@@ -26,7 +26,7 @@ import org.apache.hadoop.fs.{FileSystem, Path}
 
 import org.apache.spark.SparkEnv
 import org.apache.spark.util.Utils
-
+import org.apache.spark.util.ParentClassLoader
 
 import com.esotericsoftware.reflectasm.shaded.org.objectweb.asm._
 import com.esotericsoftware.reflectasm.shaded.org.objectweb.asm.Opcodes._
@@ -35,18 +35,17 @@ import com.esotericsoftware.reflectasm.shaded.org.objectweb.asm.Opcodes._
  * A ClassLoader that reads classes from a Hadoop FileSystem or HTTP URI,
  * used to load classes defined by the interpreter when the REPL is used
  */
-class ExecutorClassLoader(classUri: String, parent: ClassLoader, userClassPathFirst: Boolean)
+class ExecutorClassLoader(classUri: String, parent: ClassLoader) extends FlexibleExecutorClassLoader(classUri, parent, false) {
+}
+/**
+ * Allows the user to specify if user class path should be first
+ */ 
+class FlexibleExecutorClassLoader(classUri: String, parent: ClassLoader, userClassPathFirst: Boolean)
 extends ClassLoader {
   val uri = new URI(classUri)
   val directory = uri.getPath
 
-  class ParentLoader(parent: ClassLoader) extends ClassLoader(parent) {
-    override def findClass(name: String): Class[_] = {
-      super.findClass(name)
-    }
-  }
-
-  val parentLoader = new ParentLoader(parent)
+  val parentLoader = new ParentClassLoader(parent)
 
   // Hadoop FileSystem object for our URI, if it isn't using HTTP
   var fileSystem: FileSystem = {

diff --git a/repl/src/test/scala/org/apache/spark/repl/ExecutorClassLoaderSuite.scala b/repl/src/test/scala/org/apache/spark/repl/ExecutorClassLoaderSuite.scala
@@ -30,31 +30,31 @@ class ExecutorClassLoaderSuite extends FunSuite {
 
   test("child first") {
     val parentLoader = new URLClassLoader(urls2, null)
-    val classLoader = new ExecutorClassLoader(url1, parentLoader, true)
+    val classLoader = new FlexibleExecutorClassLoader(url1, parentLoader, true)
     val fakeClass = classLoader.loadClass("org.apache.spark.test.FakeClass2").newInstance()
     val fakeClassVersion = fakeClass.toString
     assert(fakeClassVersion === "1")
   }
 
   test("parent first") {
     val parentLoader = new URLClassLoader(urls2, null)
-    val classLoader = new ExecutorClassLoader(url1, parentLoader, false)
+    val classLoader = new ExecutorClassLoader(url1, parentLoader)
     val fakeClass = classLoader.loadClass("org.apache.spark.test.FakeClass1").newInstance()
     val fakeClassVersion = fakeClass.toString
     assert(fakeClassVersion === "2")
   }
 
   test("child first can fall back") {
     val parentLoader = new URLClassLoader(urls2, null)
-    val classLoader = new ExecutorClassLoader(url1, parentLoader, true)
+    val classLoader = new FlexibleExecutorClassLoader(url1, parentLoader, true)
     val fakeClass = classLoader.loadClass("org.apache.spark.test.FakeClass3").newInstance()
     val fakeClassVersion = fakeClass.toString
     assert(fakeClassVersion === "2")
   }
 
   test("child first can fail") {
     val parentLoader = new URLClassLoader(urls2, null)
-    val classLoader = new ExecutorClassLoader(url1, parentLoader, true)
+    val classLoader = new FlexibleExecutorClassLoader(url1, parentLoader, true)
     intercept[java.lang.ClassNotFoundException] {
       classLoader.loadClass("org.apache.spark.test.FakeClassDoesNotExist").newInstance()
     }