[LIVY-511][LIVY-512] Remove support for old Spark, Scala versions.

This change restricts Livy support to Spark 2.2+ and Scala 2.11. Both changes are made together because by supporting Spark 2.2+ only, it becomes impossible to test Scala 2.10. As part of the change, a lot of code that used reflection to support different versions of Spark could be cleaned up and directly call Spark APIs. The Scala 2.10 parts of the builds also have been removed, but the actual support for building and running with different Scala versions (and related tests) have been left as is. This will allow us to support 2.12 in the future. This change intentionally does not touch the public API (the "api/" module). There are things that could be cleaned up now that Spark 1.x is not supported, but that would mean an API breakage so I chose to leave those alone for now. The test matrix and build profiles have also been simplified a lot. There are now two profiles to choose from (for Spark 2.2 and 2.3); integration tests can be run against a different version of Spark by running just the integration test module with the desired profile. Tested with Spark 2.2 and 2.3, and also by building against 2.2 and running integration tests against 2.3. Author: Marcelo Vanzin <[email protected]> Closes apache#112 from vanzin/LIVY-511.
taklwu · Sep 27, 2018 · 56c76bc · 56c76bc
1 parent 134713d
commit 56c76bc
Show file tree

Hide file tree

Showing 29 changed files with 113 additions and 873 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -19,24 +19,16 @@ sudo: required
 dist: trusty
 language: scala
 
-env:
-  matrix:
-    - MVN_FLAG='-DskipTests'
-    - MVN_FLAG='-Pspark-2.0-it -DskipTests'
-    - MVN_FLAG='-Pspark-2.1-it -DskipTests'
-    - MVN_FLAG='-Pspark-1.6 -DskipITs'
-    - MVN_FLAG='-Pspark-2.0 -DskipITs'
-    - MVN_FLAG='-Pspark-2.1 -DskipITs'
-
 matrix:
   include:
-      # Spark 2.2+ will only be verified using JDK8
-      # Thriftserver requires JDK8
-    - env: MVN_FLAG='-Pthriftserver -Pspark-2.2-it -DskipTests'
-    - env: MVN_FLAG='-Pthriftserver -Pspark-2.2 -DskipITs'
-    - env: MVN_FLAG='-Pthriftserver -Pspark-2.3-it -DskipTests'
-    - env: MVN_FLAG='-Pthriftserver -Pspark-2.3 -DskipITs'
-
+  - name: "Spark 2.2 Unit Tests"
+    env: MVN_FLAG='-Pthriftserver -DskipITs'
+  - name: "Spark 2.2 ITs"
+    env: MVN_FLAG='-Pthriftserver -DskipTests'
+  - name: "Spark 2.3 Unit Tests"
+    env: MVN_FLAG='-Pspark-2.3 -Pthriftserver -DskipITs'
+  - name: "Spark 2.3 ITs"
+    env: MVN_FLAG='-Pspark-2.3 -Pthriftserver -DskipTests'
 
 jdk:
   - oraclejdk8

diff --git a/README.md b/README.md
@@ -57,12 +57,8 @@ Required python packages for building Livy:
 To run Livy, you will also need a Spark installation. You can get Spark releases at
 https://spark.apache.org/downloads.html.
 
-Livy requires at least Spark 1.6 and supports both Scala 2.10 and 2.11 builds of Spark, Livy
-will automatically pick repl dependencies through detecting the Scala version of Spark.
-
-Livy also supports Spark 2.0+ for both interactive and batch submission, you could seamlessly
-switch to different versions of Spark through ``SPARK_HOME`` configuration, without needing to
-rebuild Livy.
+Livy requires Spark 2.2 or 2.3. You can switch to a different version of Spark by setting the
+``SPARK_HOME`` environment variable in the Livy server process, without needing to rebuild Livy.
 
 
 ## Building Livy
@@ -75,8 +71,9 @@ cd livy
 mvn package
 ```
 
-By default Livy is built against Apache Spark 1.6.2, but the version of Spark used when running
-Livy does not need to match the version used to build Livy. Livy internally uses reflection to
-mitigate the gaps between different Spark versions, also Livy package itself does not
-contain a Spark distribution, so it will work with any supported version of Spark (Spark 1.6+)
-without needing to rebuild against specific version of Spark.
+By default Livy is built against Apache Spark 2.2.0, but the version of Spark used when running
+Livy does not need to match the version used to build Livy. Livy internally handles the differences
+between different Spark versions.
+
+The Livy package itself does not contain a Spark distribution. It will work with any supported
+version of Spark without needing to rebuild.
diff --git a/assembly/assembly.xml b/assembly/assembly.xml
@@ -54,13 +54,6 @@
         <include>*</include>
       </includes>
     </fileSet>
-    <fileSet>
-      <directory>${project.parent.basedir}/repl/scala-2.10/target/jars</directory>
-      <outputDirectory>${assembly.name}/repl_2.10-jars</outputDirectory>
-      <includes>
-        <include>*</include>
-      </includes>
-    </fileSet>
     <fileSet>
       <directory>${project.parent.basedir}/repl/scala-2.11/target/jars</directory>
       <outputDirectory>${assembly.name}/repl_2.11-jars</outputDirectory>

diff --git a/assembly/pom.xml b/assembly/pom.xml
@@ -41,12 +41,6 @@
       <version>${project.version}</version>
     </dependency>
 
-    <dependency>
-      <groupId>${project.groupId}</groupId>
-      <artifactId>livy-repl_2.10</artifactId>
-      <version>${project.version}</version>
-    </dependency>
-
     <dependency>
       <groupId>${project.groupId}</groupId>
       <artifactId>livy-repl_2.11</artifactId>

diff --git a/conf/livy.conf.template b/conf/livy.conf.template
@@ -71,8 +71,8 @@
 # Comma-separated list of Livy REPL jars. By default Livy will upload jars from its installation
 # directory every time a session is started. By caching these files in HDFS, for example, startup
 # time of sessions on YARN can be reduced. Please list all the repl dependencies including
-# livy-repl_2.10 and livy-repl_2.11 jars, Livy will automatically pick the right dependencies in
-# session creation.
+# Scala version-specific livy-repl jars, Livy will automatically pick the right dependencies
+# during session creation.
 # livy.repl.jars =
 
 # Location of PySpark archives. By default Livy will upload the file from SPARK_HOME, but

diff --git a/core/scala-2.10/pom.xml b/core/scala-2.10/pom.xml
diff --git a/coverage/pom.xml b/coverage/pom.xml
@@ -50,24 +50,12 @@
       <version>${project.version}</version>
     </dependency>
 
-    <dependency>
-      <groupId>${project.groupId}</groupId>
-      <artifactId>livy-core_2.10</artifactId>
-      <version>${project.version}</version>
-    </dependency>
-
     <dependency>
       <groupId>${project.groupId}</groupId>
       <artifactId>livy-core_2.11</artifactId>
       <version>${project.version}</version>
     </dependency>
 
-    <dependency>
-      <groupId>${project.groupId}</groupId>
-      <artifactId>livy-repl_2.10</artifactId>
-      <version>${project.version}</version>
-    </dependency>
-
     <dependency>
       <groupId>${project.groupId}</groupId>
       <artifactId>livy-repl_2.11</artifactId>
@@ -86,12 +74,6 @@
       <version>${project.version}</version>
     </dependency>
 
-    <dependency>
-      <groupId>${project.groupId}</groupId>
-      <artifactId>livy-scala-api_2.10</artifactId>
-      <version>${project.version}</version>
-    </dependency>
-
     <dependency>
       <groupId>${project.groupId}</groupId>
       <artifactId>livy-scala-api_2.11</artifactId>

diff --git a/examples/src/main/java/org/apache/livy/examples/PiApp.java b/examples/src/main/java/org/apache/livy/examples/PiApp.java
@@ -65,7 +65,7 @@ public Integer call(Integer v1, Integer v2) {
 
 /**
  * Example execution:
- * java -cp /pathTo/spark-core_2.10-*version*.jar:/pathTo/livy-api-*version*.jar:
+ * java -cp /pathTo/spark-core_2.11-*version*.jar:/pathTo/livy-api-*version*.jar:
  * /pathTo/livy-client-http-*version*.jar:/pathTo/livy-examples-*version*.jar
  * org.apache.livy.examples.PiApp http://livy-host:8998 2
  */

diff --git a/integration-test/src/test/scala/org/apache/livy/test/InteractiveIT.scala b/integration-test/src/test/scala/org/apache/livy/test/InteractiveIT.scala
@@ -44,15 +44,8 @@ class InteractiveIT extends BaseIntegrationTestSuite {
       s.run("throw new IllegalStateException()")
         .verifyError(evalue = ".*java\\.lang\\.IllegalStateException.*")
 
-      // Check if we're running with Spark1 or Spark2, in Spark1 we will use SQLContext, whereas
-      // for Spark2 we will use SparkSession.
-      val entry = if (s.run("spark").result().isLeft) {
-        "spark"
-      } else {
-        "sqlContext"
-      }
       // Verify query submission
-      s.run(s"""val df = $entry.createDataFrame(Seq(("jerry", 20), ("michael", 21)))""")
+      s.run(s"""val df = spark.createDataFrame(Seq(("jerry", 20), ("michael", 21)))""")
         .verifyResult(".*" + Pattern.quote("df: org.apache.spark.sql.DataFrame") + ".*")
       s.run("df.registerTempTable(\"people\")").result()
       s.run("SELECT * FROM people", Some(SQL)).verifyResult(".*\"jerry\",20.*\"michael\",21.*")