[LIVY-457][REPL] Fix SQLContext is not initialized correctly issue

## What changes were proposed in this pull request?

The signature of SQLContext's constructor is changed in Spark2, but we're still using the Spark1's signature, which will throw an exception when using this object.

## How was this patch tested?

UT and local verification.

Author: jerryshao <[email protected]>

Closes #86 from jerryshao/LIVY-457.

(cherry picked from commit cd8b112)
Signed-off-by: jerryshao <[email protected]>

.travis.yml

@@ -51,14 +51,13 @@ cache:
   pip: true
   directories:
     - $HOME/.m2
-    - $(npm config get prefix)
 
 before_install:
   - sudo apt-get -y install python3-pip python-dev
   - sudo apt-get -y install libkrb5-dev
   - sudo apt-get -y remove python-setuptools
-  - sudo pip2 install --upgrade pip "setuptools < 36"
-  - sudo python3 -m pip install --upgrade pip "setuptools < 36"
+  - sudo pip2 install --upgrade "pip < 10.0.0" "setuptools < 36"
+  - sudo python3 -m pip install --upgrade "pip < 10.0.0" "setuptools < 36"
   - sudo pip2 install codecov cloudpickle
   - sudo python3 -m pip install cloudpickle
 

integration-test/src/test/scala/org/apache/livy/test/InteractiveIT.scala

@@ -90,6 +90,10 @@ class InteractiveIT extends BaseIntegrationTestSuite {
       s.run("from pyspark.sql.types import Row").verifyResult("")
       s.run("x = [Row(age=1, name=u'a'), Row(age=2, name=u'b'), Row(age=3, name=u'c')]")
         .verifyResult("")
+      // Check if we're running with Spark2.
+      if (s.run("spark").result().isLeft) {
+        s.run("sqlContext.sparkSession").verifyResult(".*pyspark\\.sql\\.session\\.SparkSession.*")
+      }
       s.run("%table x").verifyResult(".*headers.*type.*name.*data.*")
       s.run("abcde").verifyError(ename = "NameError", evalue = "name 'abcde' is not defined")
       s.run("raise KeyError, 'foo'").verifyError(ename = "KeyError", evalue = "'foo'")

repl/src/main/resources/fake_shell.py

@@ -588,14 +588,17 @@ def main():
             conf = SparkConf(_jvm = gateway.jvm, _jconf = jconf)
             sc = SparkContext(jsc=jsc, gateway=gateway, conf=conf)
             global_dict['sc'] = sc
-            sqlc = SQLContext(sc, jsqlc)
-            global_dict['sqlContext'] = sqlc
 
             if spark_major_version >= "2":
                 from pyspark.sql import SparkSession
                 spark_session = SparkSession(sc, gateway.entry_point.sparkSession())
+                sqlc = SQLContext(sc, spark_session, jsqlc)
+                global_dict['sqlContext'] = sqlc
                 global_dict['spark'] = spark_session
             else:
+                sqlc = SQLContext(sc, jsqlc)
+                global_dict['sqlContext'] = sqlc
+
                 # LIVY-294, need to check whether HiveContext can work properly,
                 # fallback to SQLContext if HiveContext can not be initialized successfully.
                 # Only for spark-1.