SPARK-1387. Update build plugins, avoid plugin version warning, centr…

…alize versions Another handful of small build changes to organize and standardize a bit, and avoid warnings: - Update Maven plugin versions for good measure - Since plugins need maven 3.0.4 already, require it explicitly (<3.0.4 had some bugs anyway) - Use variables to define versions across dependencies where they should move in lock step - ... and make this consistent between Maven/SBT OK, I also updated the JIRA URL while I was at it here. Author: Sean Owen <[email protected]> Closes apache#291 from srowen/SPARK-1387 and squashes the following commits: 461eca1 [Sean Owen] Couldn't resist also updating JIRA location to new one c2d5cc5 [Sean Owen] Update plugins and Maven version; use variables consistently across Maven/SBT to define dependency versions that should stay in step.
pdeyhim · Jun 25, 2014 · 988b698 · 988b698
1 parent 9eda545
commit 988b698
Show file tree

Hide file tree

Showing 10 changed files with 57 additions and 50 deletions.
diff --git a/assembly/pom.xml b/assembly/pom.xml
@@ -208,7 +208,7 @@
           <plugin>
             <groupId>org.codehaus.mojo</groupId>
             <artifactId>buildnumber-maven-plugin</artifactId>
-            <version>1.1</version>
+            <version>1.2</version>
             <executions>
               <execution>
                 <phase>validate</phase>

diff --git a/core/pom.xml b/core/pom.xml
@@ -117,12 +117,10 @@
     <dependency>
       <groupId>com.twitter</groupId>
       <artifactId>chill_${scala.binary.version}</artifactId>
-      <version>0.3.1</version>
     </dependency>
     <dependency>
       <groupId>com.twitter</groupId>
       <artifactId>chill-java</artifactId>
-      <version>0.3.1</version>
     </dependency>
     <dependency>
       <groupId>commons-net</groupId>

diff --git a/dev/audit-release/maven_app_core/pom.xml b/dev/audit-release/maven_app_core/pom.xml
@@ -49,7 +49,7 @@
     <plugins>
       <plugin>
         <artifactId>maven-compiler-plugin</artifactId>
-        <version>2.3.2</version>
+        <version>3.1</version>
       </plugin>
     </plugins>
   </build>

diff --git a/docs/building-with-maven.md b/docs/building-with-maven.md
@@ -6,7 +6,7 @@ title: Building Spark with Maven
 * This will become a table of contents (this text will be scraped).
 {:toc}
 
-Building Spark using Maven Requires Maven 3 (the build process is tested with Maven 3.0.4) and Java 1.6 or newer.
+Building Spark using Maven requires Maven 3.0.4 or newer and Java 1.6 or newer.
 
 
 ## Setting up Maven's Memory Usage ##

diff --git a/examples/pom.xml b/examples/pom.xml
@@ -110,7 +110,7 @@
     <dependency>
       <groupId>org.apache.hbase</groupId>
       <artifactId>hbase</artifactId>
-      <version>0.94.6</version>
+      <version>${hbase.version}</version>
       <exclusions>
         <exclusion>
           <groupId>asm</groupId>

diff --git a/graphx/pom.xml b/graphx/pom.xml
@@ -54,7 +54,7 @@
     <dependency>
       <groupId>org.jblas</groupId>
       <artifactId>jblas</artifactId>
-      <version>1.2.3</version>
+      <version>${jblas.version}</version>
     </dependency>
     <dependency>
       <groupId>org.eclipse.jetty</groupId>

diff --git a/mllib/pom.xml b/mllib/pom.xml
@@ -58,7 +58,7 @@
     <dependency>
       <groupId>org.jblas</groupId>
       <artifactId>jblas</artifactId>
-      <version>1.2.3</version>
+      <version>${jblas.version}</version>
     </dependency>
     <dependency>
       <groupId>org.scalanlp</groupId>

diff --git a/pom.xml b/pom.xml
@@ -54,11 +54,11 @@
   </developers>
   <issueManagement>
     <system>JIRA</system>
-    <url>https://spark-project.atlassian.net/browse/SPARK</url>
+    <url>https://issues.apache.org/jira/browse/SPARK</url>
   </issueManagement>
 
   <prerequisites>
-    <maven>3.0.0</maven>
+    <maven>3.0.4</maven>
   </prerequisites>
 
   <mailingLists>
@@ -123,6 +123,10 @@
     <hbase.version>0.94.6</hbase.version>
     <hive.version>0.12.0</hive.version>
     <parquet.version>1.3.2</parquet.version>
+    <jblas.version>1.2.3</jblas.version>
+    <jetty.version>8.1.14.v20131031</jetty.version>
+    <chill.version>0.3.1</chill.version>
+    <codahale.metrics.version>3.0.0</codahale.metrics.version>
 
     <PermGen>64m</PermGen>
     <MaxPermGen>512m</MaxPermGen>
@@ -192,22 +196,22 @@
       <dependency>
         <groupId>org.eclipse.jetty</groupId>
         <artifactId>jetty-util</artifactId>
-        <version>8.1.14.v20131031</version>
+        <version>${jetty.version}</version>
       </dependency>
       <dependency>
         <groupId>org.eclipse.jetty</groupId>
         <artifactId>jetty-security</artifactId>
-        <version>8.1.14.v20131031</version>
+        <version>${jetty.version}</version>
       </dependency>
       <dependency>
         <groupId>org.eclipse.jetty</groupId>
         <artifactId>jetty-plus</artifactId>
-        <version>8.1.14.v20131031</version>
+        <version>${jetty.version}</version>
       </dependency>
       <dependency>
         <groupId>org.eclipse.jetty</groupId>
         <artifactId>jetty-server</artifactId>
-        <version>8.1.14.v20131031</version>
+        <version>${jetty.version}</version>
       </dependency>
       <dependency>
         <groupId>com.google.guava</groupId>
@@ -273,7 +277,7 @@
       <dependency>
         <groupId>com.twitter</groupId>
         <artifactId>chill_${scala.binary.version}</artifactId>
-        <version>0.3.1</version>
+        <version>${chill.version}</version>
         <exclusions>
           <exclusion>
             <groupId>org.ow2.asm</groupId>
@@ -288,7 +292,7 @@
       <dependency>
         <groupId>com.twitter</groupId>
         <artifactId>chill-java</artifactId>
-        <version>0.3.1</version>
+        <version>${chill.version}</version>
         <exclusions>
           <exclusion>
             <groupId>org.ow2.asm</groupId>
@@ -392,27 +396,27 @@
       <dependency>
         <groupId>com.codahale.metrics</groupId>
         <artifactId>metrics-core</artifactId>
-        <version>3.0.0</version>
+        <version>${codahale.metrics.version}</version>
       </dependency>
       <dependency>
         <groupId>com.codahale.metrics</groupId>
         <artifactId>metrics-jvm</artifactId>
-        <version>3.0.0</version>
+        <version>${codahale.metrics.version}</version>
       </dependency>
       <dependency>
         <groupId>com.codahale.metrics</groupId>
         <artifactId>metrics-json</artifactId>
-        <version>3.0.0</version>
+        <version>${codahale.metrics.version}</version>
       </dependency>
       <dependency>
         <groupId>com.codahale.metrics</groupId>
         <artifactId>metrics-ganglia</artifactId>
-        <version>3.0.0</version>
+        <version>${codahale.metrics.version}</version>
       </dependency>
       <dependency>
         <groupId>com.codahale.metrics</groupId>
         <artifactId>metrics-graphite</artifactId>
-        <version>3.0.0</version>
+        <version>${codahale.metrics.version}</version>
       </dependency>
       <dependency>
         <groupId>org.scala-lang</groupId>
@@ -585,7 +589,7 @@
         <plugin>
           <groupId>org.apache.maven.plugins</groupId>
           <artifactId>maven-enforcer-plugin</artifactId>
-          <version>1.1.1</version>
+          <version>1.3.1</version>
           <executions>
             <execution>
               <id>enforce-versions</id>
@@ -595,7 +599,7 @@
               <configuration>
                 <rules>
                   <requireMavenVersion>
-                    <version>3.0.0</version>
+                    <version>3.0.4</version>
                   </requireMavenVersion>
                   <requireJavaVersion>
                     <version>${java.version}</version>
@@ -608,12 +612,12 @@
         <plugin>
           <groupId>org.codehaus.mojo</groupId>
           <artifactId>build-helper-maven-plugin</artifactId>
-          <version>1.7</version>
+          <version>1.8</version>
         </plugin>
         <plugin>
           <groupId>net.alchim31.maven</groupId>
           <artifactId>scala-maven-plugin</artifactId>
-          <version>3.1.5</version>
+          <version>3.1.6</version>
           <executions>
             <execution>
               <id>scala-compile-first</id>
@@ -674,7 +678,7 @@
         <plugin>
           <groupId>org.apache.maven.plugins</groupId>
           <artifactId>maven-surefire-plugin</artifactId>
-          <version>2.12.4</version>
+          <version>2.17</version>
           <configuration>
             <!-- Uses scalatest instead -->
             <skipTests>true</skipTests>
@@ -713,7 +717,7 @@
         <plugin>
           <groupId>org.apache.maven.plugins</groupId>
           <artifactId>maven-shade-plugin</artifactId>
-          <version>2.0</version>
+          <version>2.2</version>
         </plugin>
         <plugin>
           <groupId>org.apache.maven.plugins</groupId>
@@ -810,7 +814,6 @@
           <plugin>
             <groupId>org.apache.maven.plugins</groupId>
             <artifactId>maven-jar-plugin</artifactId>
-            <version>2.4</version>
             <executions>
               <execution>
                 <goals>

diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
@@ -251,10 +251,10 @@ object SparkBuild extends Build {
 
     libraryDependencies ++= Seq(
         "io.netty"          % "netty-all"      % "4.0.17.Final",
-        "org.eclipse.jetty" % "jetty-server"   % "8.1.14.v20131031",
-        "org.eclipse.jetty" % "jetty-util"     % "8.1.14.v20131031",
-        "org.eclipse.jetty" % "jetty-plus"     % "8.1.14.v20131031",
-        "org.eclipse.jetty" % "jetty-security" % "8.1.14.v20131031",
+        "org.eclipse.jetty" % "jetty-server"   % jettyVersion,
+        "org.eclipse.jetty" % "jetty-util"     % jettyVersion,
+        "org.eclipse.jetty" % "jetty-plus"     % jettyVersion,
+        "org.eclipse.jetty" % "jetty-security" % jettyVersion,
         /** Workaround for SPARK-959. Dependency used by org.eclipse.jetty. Fixed in ivy 2.3.0. */
         "org.eclipse.jetty.orbit" % "javax.servlet" % "3.0.0.v201112011016" artifacts Artifact("javax.servlet", "jar", "jar"),
         "org.scalatest"    %% "scalatest"       % "1.9.1"  % "test",
@@ -279,6 +279,13 @@ object SparkBuild extends Build {
     publishLocalBoth <<= Seq(publishLocal in MavenCompile, publishLocal).dependOn
   ) ++ net.virtualvoid.sbt.graph.Plugin.graphSettings ++ ScalaStyleSettings
 
+  val akkaVersion = "2.2.3-shaded-protobuf"
+  val chillVersion = "0.3.1"
+  val codahaleMetricsVersion = "3.0.0"
+  val jblasVersion = "1.2.3"
+  val jettyVersion = "8.1.14.v20131031"
+  val hiveVersion = "0.12.0"
+  val parquetVersion = "1.3.2"
   val slf4jVersion = "1.7.5"
 
   val excludeNetty = ExclusionRule(organization = "org.jboss.netty")
@@ -312,9 +319,9 @@ object SparkBuild extends Build {
         "commons-daemon"             % "commons-daemon"   % "1.0.10", // workaround for bug HADOOP-9407
         "com.ning"                   % "compress-lzf"     % "1.0.0",
         "org.xerial.snappy"          % "snappy-java"      % "1.0.5",
-        "org.spark-project.akka"    %% "akka-remote"      % "2.2.3-shaded-protobuf"  excludeAll(excludeNetty),
-        "org.spark-project.akka"    %% "akka-slf4j"       % "2.2.3-shaded-protobuf"  excludeAll(excludeNetty),
-        "org.spark-project.akka"    %% "akka-testkit"     % "2.2.3-shaded-protobuf" % "test",
+        "org.spark-project.akka"    %% "akka-remote"      % akkaVersion excludeAll(excludeNetty),
+        "org.spark-project.akka"    %% "akka-slf4j"       % akkaVersion excludeAll(excludeNetty),
+        "org.spark-project.akka"    %% "akka-testkit"     % akkaVersion % "test",
         "org.json4s"                %% "json4s-jackson"   % "3.2.6" excludeAll(excludeScalap),
         "it.unimi.dsi"               % "fastutil"         % "6.4.4",
         "colt"                       % "colt"             % "1.2.0",
@@ -324,12 +331,12 @@ object SparkBuild extends Build {
         "org.apache.derby"           % "derby"            % "10.4.2.0"                     % "test",
         "org.apache.hadoop"          % hadoopClient       % hadoopVersion excludeAll(excludeNetty, excludeAsm, excludeCommonsLogging, excludeSLF4J, excludeOldAsm),
         "org.apache.curator"         % "curator-recipes"  % "2.4.0" excludeAll(excludeNetty),
-        "com.codahale.metrics"       % "metrics-core"     % "3.0.0",
-        "com.codahale.metrics"       % "metrics-jvm"      % "3.0.0",
-        "com.codahale.metrics"       % "metrics-json"     % "3.0.0",
-        "com.codahale.metrics"       % "metrics-graphite" % "3.0.0",
-        "com.twitter"               %% "chill"            % "0.3.1" excludeAll(excludeAsm),
-        "com.twitter"                % "chill-java"       % "0.3.1" excludeAll(excludeAsm),
+        "com.codahale.metrics"       % "metrics-core"     % codahaleMetricsVersion,
+        "com.codahale.metrics"       % "metrics-jvm"      % codahaleMetricsVersion,
+        "com.codahale.metrics"       % "metrics-json"     % codahaleMetricsVersion,
+        "com.codahale.metrics"       % "metrics-graphite" % codahaleMetricsVersion,
+        "com.twitter"               %% "chill"            % chillVersion excludeAll(excludeAsm),
+        "com.twitter"                % "chill-java"       % chillVersion excludeAll(excludeAsm),
         "org.tachyonproject"         % "tachyon"          % "0.4.1-thrift" excludeAll(excludeHadoop, excludeCurator, excludeEclipseJetty, excludePowermock),
         "com.clearspring.analytics"  % "stream"           % "2.5.1"
       ),
@@ -373,7 +380,7 @@ object SparkBuild extends Build {
     name := "spark-graphx",
     previousArtifact := sparkPreviousArtifact("spark-graphx"),
     libraryDependencies ++= Seq(
-      "org.jblas" % "jblas" % "1.2.3"
+      "org.jblas" % "jblas" % jblasVersion
     )
   )
 
@@ -386,7 +393,7 @@ object SparkBuild extends Build {
     name := "spark-mllib",
     previousArtifact := sparkPreviousArtifact("spark-mllib"),
     libraryDependencies ++= Seq(
-      "org.jblas" % "jblas" % "1.2.3",
+      "org.jblas" % "jblas" % jblasVersion,
       "org.scalanlp" %% "breeze" % "0.7"
     )
   )
@@ -406,8 +413,8 @@ object SparkBuild extends Build {
   def sqlCoreSettings = sharedSettings ++ Seq(
     name := "spark-sql",
     libraryDependencies ++= Seq(
-      "com.twitter" % "parquet-column" % "1.3.2",
-      "com.twitter" % "parquet-hadoop" % "1.3.2"
+      "com.twitter" % "parquet-column" % parquetVersion,
+      "com.twitter" % "parquet-hadoop" % parquetVersion
     )
   )
 
@@ -419,9 +426,9 @@ object SparkBuild extends Build {
     jarName in packageDependency <<= version map { v => "spark-hive-assembly-" + v + "-hadoop" + hadoopVersion + "-deps.jar" },
     javaOptions += "-XX:MaxPermSize=1g",
     libraryDependencies ++= Seq(
-      "org.apache.hive" % "hive-metastore" % "0.12.0",
-      "org.apache.hive" % "hive-exec" % "0.12.0",
-      "org.apache.hive" % "hive-serde" % "0.12.0"
+      "org.apache.hive" % "hive-metastore" % hiveVersion,
+      "org.apache.hive" % "hive-exec"      % hiveVersion,
+      "org.apache.hive" % "hive-serde"     % hiveVersion
     ),
     // Multiple queries rely on the TestHive singleton.  See comments there for more details.
     parallelExecution in Test := false,
@@ -560,7 +567,7 @@ object SparkBuild extends Build {
     name := "spark-streaming-zeromq",
     previousArtifact := sparkPreviousArtifact("spark-streaming-zeromq"),
     libraryDependencies ++= Seq(
-      "org.spark-project.akka" %% "akka-zeromq" % "2.2.3-shaded-protobuf" excludeAll(excludeNetty)
+      "org.spark-project.akka" %% "akka-zeromq" % akkaVersion excludeAll(excludeNetty)
     )
   )
 

diff --git a/streaming/pom.xml b/streaming/pom.xml
@@ -96,7 +96,6 @@
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-jar-plugin</artifactId>
-        <version>2.2</version>
         <executions>
           <execution>
             <goals>