From 24b4410db0028a09dc98f1217cb0793f10a83b6a Mon Sep 17 00:00:00 2001 From: James Baiera Date: Thu, 9 Apr 2020 18:22:03 -0400 Subject: [PATCH 1/3] Import sources for javadoc through a dependency Add configurations for importing and exporting source directories Add a sources configuration to add another project's source to the current project's javadoc. Add a sourceElements configuration to register all java source directories in the project under so they may be picked up by other projects. --- .../hadoop/gradle/BuildPlugin.groovy | 53 +++++++++++++++++++ hive/build.gradle | 12 ++--- pig/build.gradle | 12 ++--- spark/sql-13/build.gradle | 18 ++++--- spark/sql-20/build.gradle | 18 ++++--- storm/build.gradle | 12 ++--- 6 files changed, 84 insertions(+), 41 deletions(-) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/hadoop/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/hadoop/gradle/BuildPlugin.groovy index 2b4f2ca40..1a0c83386 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/hadoop/gradle/BuildPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/hadoop/gradle/BuildPlugin.groovy @@ -17,11 +17,15 @@ import org.gradle.api.artifacts.ProjectDependency import org.gradle.api.artifacts.ResolutionStrategy import org.gradle.api.artifacts.maven.MavenPom import org.gradle.api.artifacts.maven.MavenResolver +import org.gradle.api.attributes.LibraryElements +import org.gradle.api.attributes.Usage import org.gradle.api.file.CopySpec +import org.gradle.api.file.FileCollection import org.gradle.api.java.archives.Manifest import org.gradle.api.plugins.JavaPlugin import org.gradle.api.plugins.MavenPlugin import org.gradle.api.plugins.MavenPluginConvention +import org.gradle.api.tasks.SourceSet import org.gradle.api.tasks.SourceSetContainer import org.gradle.api.tasks.TaskProvider import org.gradle.api.tasks.Upload @@ -84,6 +88,37 @@ class BuildPlugin implements Plugin { } private static void configureConfigurations(Project project) { + if (project != project.rootProject) { + // Set up avenues for sharing source files between projects in order to create embedded Javadocs + // Import source configuration + Configuration sources = project.configurations.create("sources") + sources.canBeConsumed = false + sources.canBeResolved = true + sources.attributes { + // Changing USAGE is required when working with Scala projects, otherwise the source dirs get pulled + // into incremental compilation analysis. + attribute(Usage.USAGE_ATTRIBUTE, project.objects.named(Usage, 'java-source')) + attribute(LibraryElements.LIBRARY_ELEMENTS_ATTRIBUTE, project.objects.named(LibraryElements, 'sources')) + } + + // Export source configuration + Configuration sourceElements = project.configurations.create("sourceElements") + sourceElements.canBeConsumed = true + sourceElements.canBeResolved = false + sourceElements.extendsFrom(sources) + sourceElements.attributes { + // Changing USAGE is required when working with Scala projects, otherwise the source dirs get pulled + // into incremental compilation analysis. + attribute(Usage.USAGE_ATTRIBUTE, project.objects.named(Usage, 'java-source')) + attribute(LibraryElements.LIBRARY_ELEMENTS_ATTRIBUTE, project.objects.named(LibraryElements, 'sources')) + } + + // Should this just be implementation? + Configuration javadoc = project.configurations.create("javadoc") + javadoc.canBeConsumed = false + javadoc.canBeResolved = true + } + if (project.path.startsWith(":qa")) { return } @@ -200,6 +235,15 @@ class BuildPlugin implements Plugin { project.sourceCompatibility = '1.8' project.targetCompatibility = '1.8' + // TODO: Remove all root project distribution logic. It should exist in a separate dist project. + if (project != project.rootProject) { + SourceSet mainSourceSet = project.sourceSets.main + FileCollection javaSourceDirs = mainSourceSet.java.sourceDirectories + javaSourceDirs.each { File srcDir -> + project.getArtifacts().add('sourceElements', srcDir) + } + } + JavaCompile compileJava = project.tasks.getByName('compileJava') as JavaCompile compileJava.getOptions().setCompilerArgs(['-Xlint:unchecked', '-Xlint:options']) @@ -235,6 +279,10 @@ class BuildPlugin implements Plugin { sourcesJar.dependsOn(project.tasks.classes) sourcesJar.classifier = 'sources' sourcesJar.from(project.sourceSets.main.allSource) + // TODO: Remove when root project does not handle distribution + if (project != project.rootProject) { + sourcesJar.from(project.configurations.sources) + } // Configure javadoc Javadoc javadoc = project.tasks.getByName('javadoc') as Javadoc @@ -246,6 +294,11 @@ class BuildPlugin implements Plugin { "org/elasticsearch/hadoop/util/**", "org/apache/hadoop/hive/**" ] + // TODO: Remove when root project does not handle distribution + if (project != project.rootProject) { + javadoc.source += project.configurations.sources + javadoc.classpath += project.configurations.javadoc + } // Set javadoc executable to runtime Java (1.8) javadoc.executable = new File(project.ext.runtimeJavaHome, 'bin/javadoc') diff --git a/hive/build.gradle b/hive/build.gradle index 0db0afc18..75dc66151 100644 --- a/hive/build.gradle +++ b/hive/build.gradle @@ -33,6 +33,9 @@ dependencies { itestImplementation("org.apache.hive:hive-jdbc:$hiveVersion") { exclude module: "log4j-slf4j-impl" } + + sources(project(":elasticsearch-hadoop-mr")) + javadoc(project(":elasticsearch-hadoop-mr")) } jar { @@ -42,12 +45,3 @@ jar { include "META-INF/services/*" } } - -javadoc { - source += project(":elasticsearch-hadoop-mr").sourceSets.main.allJava - classpath += files(project(":elasticsearch-hadoop-mr").sourceSets.main.compileClasspath) -} - -sourcesJar { - from project(":elasticsearch-hadoop-mr").sourceSets.main.allJava.srcDirs -} diff --git a/pig/build.gradle b/pig/build.gradle index 45b3f7721..489798068 100644 --- a/pig/build.gradle +++ b/pig/build.gradle @@ -31,6 +31,9 @@ dependencies { itestImplementation(project(":test:shared")) itestImplementation("dk.brics.automaton:automaton:1.11-8") + + sources(project(":elasticsearch-hadoop-mr")) + javadoc(project(":elasticsearch-hadoop-mr")) } jar { @@ -40,12 +43,3 @@ jar { include "META-INF/services/*" } } - -javadoc { - source += project(":elasticsearch-hadoop-mr").sourceSets.main.allJava - classpath += files(project(":elasticsearch-hadoop-mr").sourceSets.main.compileClasspath) -} - -sourcesJar { - from project(":elasticsearch-hadoop-mr").sourceSets.main.allJava.srcDirs -} diff --git a/spark/sql-13/build.gradle b/spark/sql-13/build.gradle index 276abc9bc..0945c3c57 100644 --- a/spark/sql-13/build.gradle +++ b/spark/sql-13/build.gradle @@ -161,6 +161,16 @@ dependencies { testImplementation("org.apache.spark:spark-sql_${project.ext.scalaMajorVersion}:$sparkVersion") { exclude group: 'org.apache.hadoop' } + + sources(project(":elasticsearch-hadoop-mr")) + javadoc(project(":elasticsearch-hadoop-mr")) +} + +// Export generated Java code from the genjavadoc compiler plugin +artifacts { + sourceElements(project.file("$buildDir/generated/java")) { + builtBy compileScala + } } jar { @@ -173,13 +183,7 @@ jar { javadoc { dependsOn compileScala - source += project(":elasticsearch-hadoop-mr").sourceSets.main.allJava source += "$buildDir/generated/java" - classpath += files(project(":elasticsearch-hadoop-mr").sourceSets.main.compileClasspath) -} - -sourcesJar { - from project(":elasticsearch-hadoop-mr").sourceSets.main.allJava.srcDirs } scaladoc { @@ -193,4 +197,4 @@ tasks.withType(ScalaCompile) { "-P:genjavadoc:out=$buildDir/generated/java".toString() ] } -} \ No newline at end of file +} diff --git a/spark/sql-20/build.gradle b/spark/sql-20/build.gradle index 7e2a79d75..1d4f1441f 100644 --- a/spark/sql-20/build.gradle +++ b/spark/sql-20/build.gradle @@ -156,6 +156,16 @@ dependencies { itestImplementation("org.apache.spark:spark-streaming_${project.ext.scalaMajorVersion}:$sparkVersion") { exclude group: 'org.apache.hadoop' } + + sources(project(":elasticsearch-hadoop-mr")) + javadoc(project(":elasticsearch-hadoop-mr")) +} + +// Export generated Java code from the genjavadoc compiler plugin +artifacts { + sourceElements(project.file("$buildDir/generated/java")) { + builtBy compileScala + } } jar { @@ -168,13 +178,7 @@ jar { javadoc { dependsOn compileScala - source += project(":elasticsearch-hadoop-mr").sourceSets.main.allJava source += "$buildDir/generated/java" - classpath += files(project(":elasticsearch-hadoop-mr").sourceSets.main.compileClasspath) -} - -sourcesJar { - from project(":elasticsearch-hadoop-mr").sourceSets.main.allJava.srcDirs } scaladoc { @@ -188,4 +192,4 @@ tasks.withType(ScalaCompile) { "-P:genjavadoc:out=$buildDir/generated/java".toString() ] } -} \ No newline at end of file +} diff --git a/storm/build.gradle b/storm/build.gradle index 8e5896ca4..9250df8d2 100644 --- a/storm/build.gradle +++ b/storm/build.gradle @@ -27,6 +27,9 @@ dependencies { itestImplementation(project(":test:shared")) itestImplementation("com.google.guava:guava:16.0.1") itestImplementation("com.twitter:carbonite:1.4.0") + + sources(project(":elasticsearch-hadoop-mr")) + javadoc(project(":elasticsearch-hadoop-mr")) } jar { @@ -37,13 +40,4 @@ jar { } } -javadoc { - source += project(":elasticsearch-hadoop-mr").sourceSets.main.allJava - classpath += files(project(":elasticsearch-hadoop-mr").sourceSets.main.compileClasspath) -} - -sourcesJar { - from project(":elasticsearch-hadoop-mr").sourceSets.main.allJava.srcDirs -} - tasks.getByName('integrationTest').enabled = false From 94c0133a947fdb1b85e5afd5244cbecf782aabfc Mon Sep 17 00:00:00 2001 From: James Baiera Date: Thu, 9 Apr 2020 18:39:54 -0400 Subject: [PATCH 2/3] Don't need a javadoc configuration. The dependencies for the javadoc classpath have everything already --- .../org/elasticsearch/hadoop/gradle/BuildPlugin.groovy | 6 ------ hive/build.gradle | 1 - pig/build.gradle | 1 - spark/sql-13/build.gradle | 1 - spark/sql-20/build.gradle | 1 - storm/build.gradle | 1 - 6 files changed, 11 deletions(-) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/hadoop/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/hadoop/gradle/BuildPlugin.groovy index 1a0c83386..183944663 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/hadoop/gradle/BuildPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/hadoop/gradle/BuildPlugin.groovy @@ -112,11 +112,6 @@ class BuildPlugin implements Plugin { attribute(Usage.USAGE_ATTRIBUTE, project.objects.named(Usage, 'java-source')) attribute(LibraryElements.LIBRARY_ELEMENTS_ATTRIBUTE, project.objects.named(LibraryElements, 'sources')) } - - // Should this just be implementation? - Configuration javadoc = project.configurations.create("javadoc") - javadoc.canBeConsumed = false - javadoc.canBeResolved = true } if (project.path.startsWith(":qa")) { @@ -297,7 +292,6 @@ class BuildPlugin implements Plugin { // TODO: Remove when root project does not handle distribution if (project != project.rootProject) { javadoc.source += project.configurations.sources - javadoc.classpath += project.configurations.javadoc } // Set javadoc executable to runtime Java (1.8) javadoc.executable = new File(project.ext.runtimeJavaHome, 'bin/javadoc') diff --git a/hive/build.gradle b/hive/build.gradle index 75dc66151..c3810d224 100644 --- a/hive/build.gradle +++ b/hive/build.gradle @@ -35,7 +35,6 @@ dependencies { } sources(project(":elasticsearch-hadoop-mr")) - javadoc(project(":elasticsearch-hadoop-mr")) } jar { diff --git a/pig/build.gradle b/pig/build.gradle index 489798068..f90e99983 100644 --- a/pig/build.gradle +++ b/pig/build.gradle @@ -33,7 +33,6 @@ dependencies { itestImplementation("dk.brics.automaton:automaton:1.11-8") sources(project(":elasticsearch-hadoop-mr")) - javadoc(project(":elasticsearch-hadoop-mr")) } jar { diff --git a/spark/sql-13/build.gradle b/spark/sql-13/build.gradle index 0945c3c57..98e274eaa 100644 --- a/spark/sql-13/build.gradle +++ b/spark/sql-13/build.gradle @@ -163,7 +163,6 @@ dependencies { } sources(project(":elasticsearch-hadoop-mr")) - javadoc(project(":elasticsearch-hadoop-mr")) } // Export generated Java code from the genjavadoc compiler plugin diff --git a/spark/sql-20/build.gradle b/spark/sql-20/build.gradle index 1d4f1441f..16c9541d5 100644 --- a/spark/sql-20/build.gradle +++ b/spark/sql-20/build.gradle @@ -158,7 +158,6 @@ dependencies { } sources(project(":elasticsearch-hadoop-mr")) - javadoc(project(":elasticsearch-hadoop-mr")) } // Export generated Java code from the genjavadoc compiler plugin diff --git a/storm/build.gradle b/storm/build.gradle index 9250df8d2..4f42285d1 100644 --- a/storm/build.gradle +++ b/storm/build.gradle @@ -29,7 +29,6 @@ dependencies { itestImplementation("com.twitter:carbonite:1.4.0") sources(project(":elasticsearch-hadoop-mr")) - javadoc(project(":elasticsearch-hadoop-mr")) } jar { From 1b0ce056a4dc05c1f53053fcc5730c71dc1d37c1 Mon Sep 17 00:00:00 2001 From: James Baiera Date: Fri, 10 Apr 2020 15:47:31 -0400 Subject: [PATCH 3/3] PR Feedback sources -> additionalSources --- .../org/elasticsearch/hadoop/gradle/BuildPlugin.groovy | 6 +++--- hive/build.gradle | 2 +- pig/build.gradle | 4 ++-- spark/sql-13/build.gradle | 4 ++-- spark/sql-20/build.gradle | 2 +- storm/build.gradle | 4 ++-- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/hadoop/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/hadoop/gradle/BuildPlugin.groovy index 183944663..3c8d43022 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/hadoop/gradle/BuildPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/hadoop/gradle/BuildPlugin.groovy @@ -91,7 +91,7 @@ class BuildPlugin implements Plugin { if (project != project.rootProject) { // Set up avenues for sharing source files between projects in order to create embedded Javadocs // Import source configuration - Configuration sources = project.configurations.create("sources") + Configuration sources = project.configurations.create("additionalSources") sources.canBeConsumed = false sources.canBeResolved = true sources.attributes { @@ -276,7 +276,7 @@ class BuildPlugin implements Plugin { sourcesJar.from(project.sourceSets.main.allSource) // TODO: Remove when root project does not handle distribution if (project != project.rootProject) { - sourcesJar.from(project.configurations.sources) + sourcesJar.from(project.configurations.additionalSources) } // Configure javadoc @@ -291,7 +291,7 @@ class BuildPlugin implements Plugin { ] // TODO: Remove when root project does not handle distribution if (project != project.rootProject) { - javadoc.source += project.configurations.sources + javadoc.source = project.files(project.configurations.additionalSources) } // Set javadoc executable to runtime Java (1.8) javadoc.executable = new File(project.ext.runtimeJavaHome, 'bin/javadoc') diff --git a/hive/build.gradle b/hive/build.gradle index c3810d224..3ae60751a 100644 --- a/hive/build.gradle +++ b/hive/build.gradle @@ -34,7 +34,7 @@ dependencies { exclude module: "log4j-slf4j-impl" } - sources(project(":elasticsearch-hadoop-mr")) + additionalSources(project(":elasticsearch-hadoop-mr")) } jar { diff --git a/pig/build.gradle b/pig/build.gradle index f90e99983..8126c3cf3 100644 --- a/pig/build.gradle +++ b/pig/build.gradle @@ -31,8 +31,8 @@ dependencies { itestImplementation(project(":test:shared")) itestImplementation("dk.brics.automaton:automaton:1.11-8") - - sources(project(":elasticsearch-hadoop-mr")) + + additionalSources(project(":elasticsearch-hadoop-mr")) } jar { diff --git a/spark/sql-13/build.gradle b/spark/sql-13/build.gradle index 98e274eaa..8bdba4541 100644 --- a/spark/sql-13/build.gradle +++ b/spark/sql-13/build.gradle @@ -161,8 +161,8 @@ dependencies { testImplementation("org.apache.spark:spark-sql_${project.ext.scalaMajorVersion}:$sparkVersion") { exclude group: 'org.apache.hadoop' } - - sources(project(":elasticsearch-hadoop-mr")) + + additionalSources(project(":elasticsearch-hadoop-mr")) } // Export generated Java code from the genjavadoc compiler plugin diff --git a/spark/sql-20/build.gradle b/spark/sql-20/build.gradle index 16c9541d5..5c2760604 100644 --- a/spark/sql-20/build.gradle +++ b/spark/sql-20/build.gradle @@ -157,7 +157,7 @@ dependencies { exclude group: 'org.apache.hadoop' } - sources(project(":elasticsearch-hadoop-mr")) + additionalSources(project(":elasticsearch-hadoop-mr")) } // Export generated Java code from the genjavadoc compiler plugin diff --git a/storm/build.gradle b/storm/build.gradle index 4f42285d1..d50e58705 100644 --- a/storm/build.gradle +++ b/storm/build.gradle @@ -27,8 +27,8 @@ dependencies { itestImplementation(project(":test:shared")) itestImplementation("com.google.guava:guava:16.0.1") itestImplementation("com.twitter:carbonite:1.4.0") - - sources(project(":elasticsearch-hadoop-mr")) + + additionalSources(project(":elasticsearch-hadoop-mr")) } jar {