Fix project artifact sharing. (#1442)

Embed the MR project into dependent projects by depending on the project and embedding its contents into the jar using the configuration instead of the project's jar task.
elastic · Mar 19, 2020 · b34e112 · b34e112
1 parent 7f36061
commit b34e112
Show file tree

Hide file tree

Showing 5 changed files with 106 additions and 77 deletions.
diff --git a/hive/build.gradle b/hive/build.gradle
@@ -5,16 +5,38 @@ description = "Elasticsearch Hadoop Hive"
 
 evaluationDependsOn(':elasticsearch-hadoop-mr')
 
+configurations {
+    embedded {
+        transitive = false
+        canBeResolved = true
+    }
+    implementation {
+        extendsFrom project.configurations.embedded
+    }
+}
+
 dependencies {
-    provided(project(":elasticsearch-hadoop-mr"))
-    provided(project(path: ":elasticsearch-hadoop-mr", configuration:"compile"))
+    embedded(project(":elasticsearch-hadoop-mr"))
+
+    provided("org.apache.hive:hive-service:$hiveVersion") {
+        exclude module: "log4j-slf4j-impl"
+    }
+    provided("org.apache.hive:hive-exec:$hiveVersion")
+    provided("org.apache.hive:hive-metastore:$hiveVersion")
 
     testImplementation(project(":test:shared"))
+
     itestImplementation(project(":test:shared"))
+    itestImplementation("org.apache.hive:hive-service:$hiveVersion") {
+        exclude module: "log4j-slf4j-impl"
+    }
+    itestImplementation("org.apache.hive:hive-jdbc:$hiveVersion") {
+        exclude module: "log4j-slf4j-impl"
+    }
 }
 
 jar {
-    from(zipTree(project(":elasticsearch-hadoop-mr").jar.archivePath)) {
+    from(project.configurations.embedded.collect { it.isDirectory() ? it : zipTree(it)}) {
         include "org/elasticsearch/hadoop/**"
         include "esh-build.properties"
         include "META-INF/services/*"
@@ -29,18 +51,3 @@ javadoc {
 sourcesJar {
     from project(":elasticsearch-hadoop-mr").sourceSets.main.allJava.srcDirs
 }
-
-dependencies {
-    provided("org.apache.hive:hive-service:$hiveVersion") {
-        exclude module: "log4j-slf4j-impl"
-    }
-    provided("org.apache.hive:hive-exec:$hiveVersion")
-    provided("org.apache.hive:hive-metastore:$hiveVersion")
-
-    itestImplementation("org.apache.hive:hive-service:$hiveVersion") {
-        exclude module: "log4j-slf4j-impl"
-    }
-    itestImplementation("org.apache.hive:hive-jdbc:$hiveVersion") {
-        exclude module: "log4j-slf4j-impl"
-    }
-}
diff --git a/pig/build.gradle b/pig/build.gradle
@@ -5,16 +5,36 @@ description = "Elasticsearch Hadoop Pig"
 
 evaluationDependsOn(':elasticsearch-hadoop-mr')
 
+configurations {
+    embedded {
+        transitive = false
+        canBeResolved = true
+    }
+    implementation {
+        extendsFrom project.configurations.embedded
+    }
+}
+
+ext.pigClassifier = "h2"
+
 dependencies {
-    provided(project(":elasticsearch-hadoop-mr"))
-    provided(project(path: ":elasticsearch-hadoop-mr", configuration:"compile"))
+    embedded(project(":elasticsearch-hadoop-mr"))
+
+    provided("org.apache.pig:pig:$pigVersion:$pigClassifier")
+    provided("joda-time:joda-time:$jodaVersion")
 
     testImplementation(project(":test:shared"))
+    testImplementation("org.apache.pig:pig:$pigVersion:$pigClassifier")
+    testImplementation("joda-time:joda-time:$jodaVersion")
+    testImplementation("com.google.guava:guava:11.0")
+    testImplementation("jline:jline:0.9.94")
+
     itestImplementation(project(":test:shared"))
+    itestImplementation("dk.brics.automaton:automaton:1.11-8")
 }
 
 jar {
-    from(zipTree(project(":elasticsearch-hadoop-mr").jar.archivePath)) {
+    from(project.configurations.embedded.collect { it.isDirectory() ? it : zipTree(it)}) {
         include "org/elasticsearch/hadoop/**"
         include "esh-build.properties"
         include "META-INF/services/*"
@@ -29,18 +49,3 @@ javadoc {
 sourcesJar {
     from project(":elasticsearch-hadoop-mr").sourceSets.main.allJava.srcDirs
 }
-
-ext.pigClassifier = "h2"
-
-dependencies {
-    provided("org.apache.pig:pig:$pigVersion:$pigClassifier")
-    provided("joda-time:joda-time:$jodaVersion")
-
-    testImplementation("org.apache.pig:pig:$pigVersion:$pigClassifier")
-    testImplementation("joda-time:joda-time:$jodaVersion")
-
-    testImplementation("com.google.guava:guava:11.0")
-    testImplementation("jline:jline:0.9.94")
-
-    itestImplementation("dk.brics.automaton:automaton:1.11-8")
-}
diff --git a/spark/sql-13/build.gradle b/spark/sql-13/build.gradle
@@ -13,16 +13,42 @@ variants {
     targetVersions '2.11.12'
 }
 
+println "Compiled using Scala ${project.ext.scalaMajorVersion} [${project.ext.scalaVersion}]"
+String sparkVersion = spark13Version
+
 configurations {
+    embedded {
+        transitive = false
+        canBeResolved = true
+    }
+    implementation {
+        extendsFrom project.configurations.embedded
+    }
     scalaCompilerPlugin {
         defaultDependencies { dependencies ->
             dependencies.add(project.dependencies.create( "com.typesafe.genjavadoc:genjavadoc-plugin_${scalaVersion}:0.13"))
         }
     }
+    testImplementation {
+        exclude group: "org.mortbay.jetty"
+    }
+    itestImplementation {
+        exclude group: "org.mortbay.jetty"
+    }
 }
 
-println "Compiled using Scala ${project.ext.scalaMajorVersion} [${project.ext.scalaVersion}]"
-String sparkVersion = spark13Version
+// deal with the messy conflicts out there
+configurations.all { Configuration conf ->
+    conf.resolutionStrategy {
+        eachDependency { details ->
+            // in a similar vein, change all javax.servlet artifacts to the one used by Spark
+            // otherwise these will lead to SecurityException (signer information wrong)
+            if (details.requested.name.contains("servlet") && !details.requested.name.contains("guice")) {
+                details.useTarget group: "org.eclipse.jetty.orbit", name: "javax.servlet", version: "3.0.0.v201112011016"
+            }
+        }
+    }
+}
 
 tasks.withType(ScalaCompile) { ScalaCompile task ->
     task.sourceCompatibility = project.ext.minimumRuntimeVersion
@@ -88,7 +114,7 @@ eclipse {
 }
 
 dependencies {
-    provided(project(":elasticsearch-hadoop-mr"))
+    embedded(project(":elasticsearch-hadoop-mr"))
 
     api("org.scala-lang:scala-library:${project.ext.scalaVersion}")
     api("org.scala-lang:scala-reflect:${project.ext.scalaVersion}")
@@ -138,7 +164,7 @@ dependencies {
 }
 
 jar {
-    from(zipTree(project(":elasticsearch-hadoop-mr").jar.archivePath)) {
+    from(project.configurations.embedded.collect { it.isDirectory() ? it : zipTree(it)}) {
         include "org/elasticsearch/hadoop/**"
         include "esh-build.properties"
         include "META-INF/services/*"
@@ -160,27 +186,6 @@ scaladoc {
     title = "${rootProject.description} ${version} API"
 }
 
-// deal with the messy conflicts out there
-configurations.all { Configuration conf ->
-    conf.resolutionStrategy {
-        eachDependency { details ->
-            // in a similar vein, change all javax.servlet artifacts to the one used by Spark
-            // otherwise these will lead to SecurityException (signer information wrong)
-            if (details.requested.name.contains("servlet") && !details.requested.name.contains("guice")) {
-                details.useTarget group: "org.eclipse.jetty.orbit", name: "javax.servlet", version: "3.0.0.v201112011016"
-            }
-        }
-    }
-}
-configurations {
-    testImplementation {
-        exclude group: "org.mortbay.jetty"
-    }
-    itestImplementation {
-        exclude group: "org.mortbay.jetty"
-    }
-}
-
 tasks.withType(ScalaCompile) {
     scalaCompileOptions.with {
         additionalParameters = [

diff --git a/spark/sql-20/build.gradle b/spark/sql-20/build.gradle
@@ -14,6 +14,13 @@ variants {
 }
 
 configurations {
+    embedded {
+        transitive = false
+        canBeResolved = true
+    }
+    implementation {
+        extendsFrom project.configurations.embedded
+    }
     scalaCompilerPlugin {
         defaultDependencies { dependencies ->
             dependencies.add(project.dependencies.create( "com.typesafe.genjavadoc:genjavadoc-plugin_${scalaVersion}:0.13"))
@@ -93,7 +100,7 @@ eclipse {
 }
 
 dependencies {
-    provided(project(":elasticsearch-hadoop-mr"))
+    embedded(project(":elasticsearch-hadoop-mr"))
 
     api("org.scala-lang:scala-library:$scalaVersion")
     api("org.scala-lang:scala-reflect:$scalaVersion")
@@ -152,7 +159,7 @@ dependencies {
 }
 
 jar {
-    from(zipTree(project(":elasticsearch-hadoop-mr").jar.archivePath)) {
+    from(project.configurations.embedded.collect { it.isDirectory() ? it : zipTree(it)}) {
         include "org/elasticsearch/hadoop/**"
         include "esh-build.properties"
         include "META-INF/services/*"

diff --git a/storm/build.gradle b/storm/build.gradle
@@ -5,15 +5,32 @@ description = "Elasticsearch Storm"
 
 evaluationDependsOn(':elasticsearch-hadoop-mr')
 
+configurations {
+    embedded {
+        transitive = false
+        canBeResolved = true
+    }
+    implementation {
+        extendsFrom project.configurations.embedded
+    }
+}
+
 dependencies {
-    provided(project(":elasticsearch-hadoop-mr"))
+    embedded(project(":elasticsearch-hadoop-mr"))
+
+    provided("org.apache.storm:storm-core:$stormVersion") {
+        exclude module: "log4j-slf4j-impl"
+    }
 
     testImplementation(project(":test:shared"))
+
     itestImplementation(project(":test:shared"))
+    itestImplementation("com.google.guava:guava:16.0.1")
+    itestImplementation("com.twitter:carbonite:1.4.0")
 }
 
 jar {
-    from(zipTree(project(":elasticsearch-hadoop-mr").jar.archivePath)) {
+    from(project.configurations.embedded.collect { it.isDirectory() ? it : zipTree(it)}) {
         include "org/elasticsearch/hadoop/**"
         include "esh-build.properties"
         include "META-INF/services/*"
@@ -29,16 +46,4 @@ sourcesJar {
     from project(":elasticsearch-hadoop-mr").sourceSets.main.allJava.srcDirs
 }
 
-dependencies {
-    provided("org.apache.storm:storm-core:$stormVersion") {
-        exclude module: "log4j-slf4j-impl"
-    }
-
-    itestImplementation("com.google.guava:guava:16.0.1")
-    itestImplementation("com.twitter:carbonite:1.4.0")
-}
-
-// add itest to Eclipse
-//eclipse.classpath.plusConfigurations += [configurations.itestCompile]
-
 tasks.getByName('integrationTest').enabled = false