FIX: Don't build Hive in assembly unless running Hive tests.

This will make the tests more stable when not running SQL tests.

dev/run-tests

@@ -38,7 +38,7 @@ if [ -n "$AMPLAB_JENKINS" ]; then
   diffs=`git diff --dirstat master | awk '{ print $2; }' | grep "^sql/"`
   if [ -n "$diffs" ]; then
     echo "Detected changes in SQL. Will run Hive test suite."
-    run_sql_tests=true
+    export _RUN_SQL_TESTS=true # exported for PySpark tests
   fi
 fi
 
@@ -62,13 +62,12 @@ echo "========================================================================="
 # echo "q" is needed because sbt on encountering a build file with failure 
 # (either resolution or compilation) prompts the user for input either q, r, 
 # etc to quit or retry. This echo is there to make it not block.
-echo -e "q\n" | SPARK_HIVE=true sbt/sbt clean assembly | \
-  grep -v -e "info.*Resolving" -e "warn.*Merging" -e "info.*Including"
-
-if [ -n "$run_sql_tests" ]; then
-  echo -e "q\n" | SPARK_HIVE=true sbt/sbt test | grep -v -e "info.*Resolving" 
+if [ -n "$_RUN_SQL_TESTS" ]; then
+  echo -e "q\n" | SPARK_HIVE=true sbt/sbt clean assembly test | \
+    grep -v -e "info.*Resolving" -e "warn.*Merging" -e "info.*Including"
 else
-  echo -e "q\n" | sbt/sbt test | grep -v -e "info.*Resolving" 
+  echo -e "q\n" | sbt/sbt clean assembly test | \
+    grep -v -e "info.*Resolving" -e "warn.*Merging" -e "info.*Including"
 fi
 
 echo "========================================================================="

python/run-tests

@@ -49,7 +49,9 @@ function run_test() {
 run_test "pyspark/rdd.py"
 run_test "pyspark/context.py"
 run_test "pyspark/conf.py"
-run_test "pyspark/sql.py"
+if [ -n "$_RUN_SQL_TESTS" ]; then
+  run_test "pyspark/sql.py"
+fi
 run_test "-m doctest pyspark/broadcast.py"
 run_test "-m doctest pyspark/accumulators.py"
 run_test "-m doctest pyspark/serializers.py"