SPARK-1488. Resolve scalac feature warnings during build

For your consideration: scalac currently notes a number of feature warnings during compilation:

```
[warn] there were 65 feature warning(s); re-run with -feature for details
```

Warnings are like:

```
[warn] /Users/srowen/Documents/spark/core/src/main/scala/org/apache/spark/SparkContext.scala:1261: implicit conversion method rddToPairRDDFunctions should be enabled
[warn] by making the implicit value scala.language.implicitConversions visible.
[warn] This can be achieved by adding the import clause 'import scala.language.implicitConversions'
[warn] or by setting the compiler option -language:implicitConversions.
[warn] See the Scala docs for value scala.language.implicitConversions for a discussion
[warn] why the feature should be explicitly enabled.
[warn]   implicit def rddToPairRDDFunctions[K: ClassTag, V: ClassTag](rdd: RDD[(K, V)]) =
[warn]                ^
```

scalac is suggesting that it's just best practice to explicitly enable certain language features by importing them where used.

This PR simply adds the imports it suggests (and squashes one other Java warning along the way). This leaves just deprecation warnings in the build.

Author: Sean Owen <[email protected]>

Closes #404 from srowen/SPARK-1488 and squashes the following commits:

8598980 [Sean Owen] Quiet scalac warnings about language features by explicitly importing language features.
39bc831 [Sean Owen] Enable -feature in scalac to emit language feature warnings
(cherry picked from commit 0247b5c)

Signed-off-by: Patrick Wendell <[email protected]>

bagel/src/test/scala/org/apache/spark/bagel/BagelSuite.scala

@@ -24,6 +24,8 @@ import org.scalatest.time.SpanSugar._
 import org.apache.spark._
 import org.apache.spark.storage.StorageLevel
 
+import scala.language.postfixOps
+
 class TestVertex(val active: Boolean, val age: Int) extends Vertex with Serializable
 class TestMessage(val targetId: String) extends Message[String] with Serializable
 

core/src/main/scala/org/apache/spark/SparkContext.scala

@@ -25,6 +25,7 @@ import java.util.UUID.randomUUID
 import scala.collection.{Map, Set}
 import scala.collection.generic.Growable
 import scala.collection.mutable.{ArrayBuffer, HashMap}
+import scala.language.implicitConversions
 import scala.reflect.{ClassTag, classTag}
 import org.apache.hadoop.conf.Configuration
 import org.apache.hadoop.fs.Path

core/src/main/scala/org/apache/spark/api/java/JavaDoubleRDD.scala

@@ -19,6 +19,7 @@ package org.apache.spark.api.java
 
 import java.lang.{Double => JDouble}
 
+import scala.language.implicitConversions
 import scala.reflect.ClassTag
 
 import org.apache.spark.Partitioner

core/src/main/scala/org/apache/spark/api/java/JavaPairRDD.scala

@@ -21,6 +21,7 @@ import java.util.{Comparator, List => JList}
 import java.lang.{Iterable => JIterable}
 
 import scala.collection.JavaConversions._
+import scala.language.implicitConversions
 import scala.reflect.ClassTag
 
 import com.google.common.base.Optional

core/src/main/scala/org/apache/spark/api/java/JavaRDD.scala

@@ -17,6 +17,7 @@
 
 package org.apache.spark.api.java
 
+import scala.language.implicitConversions
 import scala.reflect.ClassTag
 
 import org.apache.spark._

core/src/main/scala/org/apache/spark/api/java/JavaSparkContext.scala

@@ -22,6 +22,7 @@ import java.util.{Map => JMap}
 
 import scala.collection.JavaConversions
 import scala.collection.JavaConversions._
+import scala.language.implicitConversions
 import scala.reflect.ClassTag
 
 import com.google.common.base.Optional

core/src/main/scala/org/apache/spark/deploy/FaultToleranceTest.scala

@@ -25,6 +25,7 @@ import scala.collection.mutable.ListBuffer
 import scala.concurrent.{Await, future, promise}
 import scala.concurrent.ExecutionContext.Implicits.global
 import scala.concurrent.duration._
+import scala.language.postfixOps
 import scala.sys.process._
 
 import org.json4s._

core/src/main/scala/org/apache/spark/deploy/master/Master.scala

@@ -23,6 +23,7 @@ import java.util.Date
 import scala.collection.mutable.{ArrayBuffer, HashMap, HashSet}
 import scala.concurrent.Await
 import scala.concurrent.duration._
+import scala.language.postfixOps
 import scala.util.Random
 
 import akka.actor._

core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala

@@ -23,6 +23,7 @@ import java.util.Date
 
 import scala.collection.mutable.HashMap
 import scala.concurrent.duration._
+import scala.language.postfixOps
 
 import akka.actor._
 import akka.remote.{DisassociatedEvent, RemotingLifecycleEvent}

core/src/main/scala/org/apache/spark/network/ConnectionManager.scala

@@ -33,6 +33,7 @@ import scala.collection.mutable.SynchronizedQueue
 
 import scala.concurrent.{Await, ExecutionContext, Future, Promise}
 import scala.concurrent.duration._
+import scala.language.postfixOps
 
 import org.apache.spark._
 import org.apache.spark.util.{SystemClock, Utils}

core/src/main/scala/org/apache/spark/rdd/CoGroupedRDD.scala

@@ -20,6 +20,7 @@ package org.apache.spark.rdd
 import java.io.{IOException, ObjectOutputStream}
 
 import scala.collection.mutable.ArrayBuffer
+import scala.language.existentials
 
 import org.apache.spark.{InterruptibleIterator, Partition, Partitioner, SparkEnv, TaskContext}
 import org.apache.spark.{Dependency, OneToOneDependency, ShuffleDependency}

core/src/main/scala/org/apache/spark/rdd/CoalescedRDD.scala

@@ -21,6 +21,7 @@ import java.io.{IOException, ObjectOutputStream}
 
 import scala.collection.mutable
 import scala.collection.mutable.ArrayBuffer
+import scala.language.existentials
 import scala.reflect.ClassTag
 
 import org.apache.spark._

core/src/main/scala/org/apache/spark/scheduler/DAGSchedulerEvent.scala

@@ -20,6 +20,7 @@ package org.apache.spark.scheduler
 import java.util.Properties
 
 import scala.collection.mutable.Map
+import scala.language.existentials
 
 import org.apache.spark._
 import org.apache.spark.executor.TaskMetrics

core/src/main/scala/org/apache/spark/scheduler/ResultTask.scala

@@ -21,6 +21,7 @@ import java.io._
 import java.util.zip.{GZIPInputStream, GZIPOutputStream}
 
 import scala.collection.mutable.HashMap
+import scala.language.existentials
 
 import org.apache.spark._
 import org.apache.spark.rdd.{RDD, RDDCheckpointData}

core/src/main/scala/org/apache/spark/scheduler/ShuffleMapTask.scala

@@ -21,6 +21,7 @@ import java.io._
 import java.util.zip.{GZIPInputStream, GZIPOutputStream}
 
 import scala.collection.mutable.HashMap
+import scala.language.existentials
 
 import org.apache.spark._
 import org.apache.spark.executor.ShuffleWriteMetrics

core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala

@@ -25,6 +25,7 @@ import scala.concurrent.duration._
 import scala.collection.mutable.ArrayBuffer
 import scala.collection.mutable.HashMap
 import scala.collection.mutable.HashSet
+import scala.language.postfixOps
 import scala.util.Random
 
 import org.apache.spark._

core/src/main/scala/org/apache/spark/ui/JettyUtils.scala

@@ -22,6 +22,7 @@ import javax.servlet.DispatcherType
 import javax.servlet.http.{HttpServlet, HttpServletRequest, HttpServletResponse}
 
 import scala.annotation.tailrec
+import scala.language.implicitConversions
 import scala.util.{Failure, Success, Try}
 import scala.xml.Node
 

core/src/main/scala/org/apache/spark/util/TimeStampedWeakValueHashMap.scala

@@ -21,6 +21,7 @@ import java.lang.ref.WeakReference
 import java.util.concurrent.atomic.AtomicInteger
 
 import scala.collection.mutable
+import scala.language.implicitConversions
 
 import org.apache.spark.Logging
 

core/src/main/scala/org/apache/spark/util/Vector.scala

@@ -17,6 +17,7 @@
 
 package org.apache.spark.util
 
+import scala.language.implicitConversions
 import scala.util.Random
 
 import org.apache.spark.util.random.XORShiftRandom

core/src/test/scala/org/apache/spark/ConnectionManagerSuite.scala

@@ -25,7 +25,7 @@ import org.apache.spark.network.{ConnectionManager, Message, ConnectionManagerId
 import scala.concurrent.Await
 import scala.concurrent.TimeoutException
 import scala.concurrent.duration._
-
+import scala.language.postfixOps
 
 /**
   * Test the ConnectionManager with various security settings.

core/src/test/scala/org/apache/spark/ContextCleanerSuite.scala

@@ -20,6 +20,8 @@ package org.apache.spark
 import java.lang.ref.WeakReference
 
 import scala.collection.mutable.{HashSet, SynchronizedSet}
+import scala.language.existentials
+import scala.language.postfixOps
 import scala.util.Random
 
 import org.scalatest.{BeforeAndAfter, FunSuite}

core/src/test/scala/org/apache/spark/DriverSuite.scala

@@ -29,6 +29,8 @@ import org.scalatest.time.SpanSugar._
 
 import org.apache.spark.util.Utils
 
+import scala.language.postfixOps
+
 class DriverSuite extends FunSuite with Timeouts {
 
   test("driver should exit after finishing") {

core/src/test/scala/org/apache/spark/PipedRDDSuite.scala

@@ -19,15 +19,14 @@ package org.apache.spark
 
 import java.io.File
 
-import com.google.common.io.Files
-
 import org.scalatest.FunSuite
 
 import org.apache.spark.rdd.{HadoopRDD, PipedRDD, HadoopPartition}
 import org.apache.hadoop.mapred.{JobConf, TextInputFormat, FileSplit}
 import org.apache.hadoop.fs.Path
 
 import scala.collection.Map
+import scala.language.postfixOps
 import scala.sys.process._
 import scala.util.Try
 

core/src/test/scala/org/apache/spark/rdd/AsyncRDDActionsSuite.scala

@@ -22,6 +22,7 @@ import java.util.concurrent.Semaphore
 import scala.concurrent.{Await, TimeoutException}
 import scala.concurrent.duration.Duration
 import scala.concurrent.ExecutionContext.Implicits.global
+import scala.language.postfixOps
 
 import org.scalatest.{BeforeAndAfterAll, FunSuite}
 import org.scalatest.concurrent.Timeouts

core/src/test/scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala

@@ -19,6 +19,7 @@ package org.apache.spark.scheduler
 
 import scala.Tuple2
 import scala.collection.mutable.{HashSet, HashMap, Map}
+import scala.language.reflectiveCalls
 
 import org.scalatest.{BeforeAndAfter, FunSuite}
 

core/src/test/scala/org/apache/spark/storage/BlockManagerSuite.scala

@@ -33,6 +33,9 @@ import org.apache.spark.scheduler.LiveListenerBus
 import org.apache.spark.serializer.{JavaSerializer, KryoSerializer}
 import org.apache.spark.util.{AkkaUtils, ByteBufferInputStream, SizeEstimator, Utils}
 
+import scala.language.implicitConversions
+import scala.language.postfixOps
+
 class BlockManagerSuite extends FunSuite with BeforeAndAfter with PrivateMethodTester {
   private val conf = new SparkConf(false)
   var store: BlockManager = null

core/src/test/scala/org/apache/spark/storage/DiskBlockManagerSuite.scala

@@ -20,6 +20,7 @@ package org.apache.spark.storage
 import java.io.{File, FileWriter}
 
 import scala.collection.mutable
+import scala.language.reflectiveCalls
 
 import com.google.common.io.Files
 import org.scalatest.{BeforeAndAfterEach, FunSuite}

core/src/test/scala/org/apache/spark/ui/UISuite.scala

@@ -21,6 +21,7 @@ import java.net.ServerSocket
 import javax.servlet.http.HttpServletRequest
 
 import scala.io.Source
+import scala.language.postfixOps
 import scala.util.{Failure, Success, Try}
 
 import org.eclipse.jetty.server.Server

core/src/test/scala/org/apache/spark/util/random/XORShiftRandomSuite.scala

@@ -22,6 +22,8 @@ import org.scalatest.matchers.ShouldMatchers
 
 import org.apache.spark.util.Utils.times
 
+import scala.language.reflectiveCalls
+
 class XORShiftRandomSuite extends FunSuite with ShouldMatchers {
 
   def fixture = new {

examples/src/main/scala/org/apache/spark/streaming/examples/ZeroMQWordCount.scala

@@ -27,6 +27,8 @@ import org.apache.spark.streaming.{Seconds, StreamingContext}
 import org.apache.spark.streaming.StreamingContext._
 import org.apache.spark.streaming.zeromq._
 
+import scala.language.implicitConversions
+
 /**
  * A simple publisher for demonstration purposes, repeatedly publishes random Messages
  * every one second.

graphx/src/main/scala/org/apache/spark/graphx/Graph.scala

@@ -17,6 +17,7 @@
 
 package org.apache.spark.graphx
 
+import scala.language.implicitConversions
 import scala.reflect.ClassTag
 
 import org.apache.spark.graphx.impl._

graphx/src/main/scala/org/apache/spark/graphx/impl/MessageToPartition.scala

@@ -17,6 +17,7 @@
 
 package org.apache.spark.graphx.impl
 
+import scala.language.implicitConversions
 import scala.reflect.{classTag, ClassTag}
 
 import org.apache.spark.Partitioner

graphx/src/main/scala/org/apache/spark/graphx/impl/Serializers.scala

@@ -20,10 +20,11 @@ package org.apache.spark.graphx.impl
 import java.io.{EOFException, InputStream, OutputStream}
 import java.nio.ByteBuffer
 
-import org.apache.spark.SparkConf
 import org.apache.spark.graphx._
 import org.apache.spark.serializer._
 
+import scala.language.existentials
+
 private[graphx]
 class VertexIdMsgSerializer extends Serializer with Serializable {
   override def newInstance(): SerializerInstance = new ShuffleSerializerInstance {

graphx/src/main/scala/org/apache/spark/graphx/util/BytecodeUtils.scala

@@ -20,6 +20,7 @@ package org.apache.spark.graphx.util
 import java.io.{ByteArrayInputStream, ByteArrayOutputStream}
 
 import scala.collection.mutable.HashSet
+import scala.language.existentials
 
 import org.apache.spark.util.Utils
 

mllib/src/main/scala/org/apache/spark/mllib/rdd/RDDFunctions.scala

@@ -17,6 +17,7 @@
 
 package org.apache.spark.mllib.rdd
 
+import scala.language.implicitConversions
 import scala.reflect.ClassTag
 
 import org.apache.spark.rdd.RDD

mllib/src/main/scala/org/apache/spark/mllib/util/MFDataGenerator.scala

@@ -17,6 +17,7 @@
 
 package org.apache.spark.mllib.util
 
+import scala.language.postfixOps
 import scala.util.Random
 
 import org.jblas.DoubleMatrix

mllib/src/test/java/org/apache/spark/mllib/linalg/JavaVectorsSuite.java

@@ -36,6 +36,7 @@ public void denseArrayConstruction() {
 
   @Test
   public void sparseArrayConstruction() {
+    @SuppressWarnings("unchecked")
     Vector v = Vectors.sparse(3, Lists.<Tuple2<Integer, Double>>newArrayList(
         new Tuple2<Integer, Double>(0, 2.0),
         new Tuple2<Integer, Double>(2, 3.0)));

pom.xml

@@ -648,6 +648,7 @@
             <args>
               <arg>-unchecked</arg>
               <arg>-deprecation</arg>
+              <arg>-feature</arg>
             </args>
             <jvmArgs>
               <jvmArg>-Xms1024m</jvmArg>

project/SparkBuild.scala

@@ -162,7 +162,7 @@ object SparkBuild extends Build {
     organization       := "org.apache.spark",
     version            := SPARK_VERSION,
     scalaVersion       := "2.10.4",
-    scalacOptions := Seq("-Xmax-classfile-name", "120", "-unchecked", "-deprecation",
+    scalacOptions := Seq("-Xmax-classfile-name", "120", "-unchecked", "-deprecation", "-feature",
       "-target:" + SCALAC_JVM_VERSION),
     javacOptions := Seq("-target", JAVAC_JVM_VERSION, "-source", JAVAC_JVM_VERSION),
     unmanagedJars in Compile <<= baseDirectory map { base => (base / "lib" ** "*.jar").classpath },

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SqlParser.scala

@@ -17,6 +17,7 @@
 
 package org.apache.spark.sql.catalyst
 
+import scala.language.implicitConversions
 import scala.util.parsing.combinator.lexical.StdLexical
 import scala.util.parsing.combinator.syntactical.StandardTokenParsers
 import scala.util.parsing.input.CharArrayReader.EofCh

streaming/src/main/scala/org/apache/spark/streaming/StreamingContext.scala

@@ -22,6 +22,7 @@ import java.util.concurrent.atomic.AtomicInteger
 
 import scala.collection.Map
 import scala.collection.mutable.Queue
+import scala.language.implicitConversions
 import scala.reflect.ClassTag
 
 import akka.actor.{Props, SupervisorStrategy}

streaming/src/main/scala/org/apache/spark/streaming/api/java/JavaDStream.scala

@@ -23,6 +23,7 @@ import org.apache.spark.api.java.JavaRDD
 import org.apache.spark.storage.StorageLevel
 import org.apache.spark.rdd.RDD
 
+import scala.language.implicitConversions
 import scala.reflect.ClassTag
 import org.apache.spark.streaming.dstream.DStream
 

streaming/src/main/scala/org/apache/spark/streaming/api/java/JavaDStreamLike.scala

@@ -22,6 +22,7 @@ import java.lang.{Long => JLong}
 import java.util.{List => JList}
 
 import scala.collection.JavaConversions._
+import scala.language.implicitConversions
 import scala.reflect.ClassTag
 
 import org.apache.spark.api.java.{JavaPairRDD, JavaRDD, JavaRDDLike}

streaming/src/main/scala/org/apache/spark/streaming/api/java/JavaPairDStream.scala

@@ -21,6 +21,7 @@ import java.lang.{Long => JLong, Iterable => JIterable}
 import java.util.{List => JList}
 
 import scala.collection.JavaConversions._
+import scala.language.implicitConversions
 import scala.reflect.ClassTag
 
 import com.google.common.base.Optional

streaming/src/main/scala/org/apache/spark/streaming/receivers/ActorReceiver.scala

@@ -23,6 +23,7 @@ import akka.actor.{ PossiblyHarmful, OneForOneStrategy }
 import akka.actor.SupervisorStrategy._
 
 import scala.concurrent.duration._
+import scala.language.postfixOps
 import scala.reflect.ClassTag
 
 import org.apache.spark.storage.{StorageLevel, StreamBlockId}

streaming/src/main/scala/org/apache/spark/streaming/scheduler/NetworkInputTracker.scala

@@ -18,6 +18,7 @@
 package org.apache.spark.streaming.scheduler
 
 import scala.collection.mutable.{HashMap, SynchronizedMap, SynchronizedQueue}
+import scala.language.existentials
 
 import akka.actor._
 

streaming/src/test/scala/org/apache/spark/streaming/StreamingContextSuite.scala

@@ -28,6 +28,8 @@ import org.scalatest.concurrent.Timeouts
 import org.scalatest.exceptions.TestFailedDueToTimeoutException
 import org.scalatest.time.SpanSugar._
 
+import scala.language.postfixOps
+
 class StreamingContextSuite extends FunSuite with BeforeAndAfter with Timeouts with Logging {
 
   val master = "local[2]"