Skip to content

Commit

Permalink
Add individual precommit task plugins (#56926) (#57015)
Browse files Browse the repository at this point in the history 
Each precommit task is currently registered inside the shared
PrecommitTasks class. Having a single class with all precommit tasks
makes individualizing which precommit tasks are needed based on type of
project difficult, where we often just disable somet tasks eg for all qa
projects. This commit creates plugins for each precommit task, and moves
the configuration of the task into each plugin. PrecommitTasks remains
for now, and just delegates to each plugin, but will be removed in a
future change.
  • Loading branch information
@rjernst
rjernst committed May 20, 2020
1 parent eece3ee commit 1283065
Show file tree
Hide file tree
Showing 19 changed files with 696 additions and 305 deletions.
17 changes: 1 addition & 16 deletions buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@ import groovy.transform.CompileStatic
import org.apache.commons.io.IOUtils
import org.elasticsearch.gradle.info.BuildParams
import org.elasticsearch.gradle.info.GlobalBuildInfoPlugin
import org.elasticsearch.gradle.precommit.DependencyLicensesTask
import org.elasticsearch.gradle.precommit.PrecommitTasks
import org.elasticsearch.gradle.test.ErrorReportingTestListener
import org.elasticsearch.gradle.testclusters.ElasticsearchCluster
Expand Down Expand Up @@ -51,10 +50,8 @@ import org.gradle.api.credentials.HttpHeaderCredentials
import org.gradle.api.execution.TaskActionListener
import org.gradle.api.plugins.ExtraPropertiesExtension
import org.gradle.api.plugins.JavaPlugin
import org.gradle.api.tasks.TaskProvider
import org.gradle.api.tasks.testing.Test
import org.gradle.authentication.http.HttpHeaderAuthentication
import org.gradle.language.base.plugins.LifecycleBasePlugin
import org.gradle.util.GradleVersion

import java.nio.charset.StandardCharsets
Expand Down Expand Up @@ -98,7 +95,7 @@ class BuildPlugin implements Plugin<Project> {

configureRepositories(project)
project.extensions.getByType(ExtraPropertiesExtension).set('versions', VersionProperties.versions)
configurePrecommit(project)
PrecommitTasks.create(project, true)
configureFips140(project)
}

Expand Down Expand Up @@ -277,18 +274,6 @@ class BuildPlugin implements Plugin<Project> {
throw new GradleException(message)
}
}

private static configurePrecommit(Project project) {
TaskProvider precommit = PrecommitTasks.create(project, true)
project.tasks.named(LifecycleBasePlugin.CHECK_TASK_NAME).configure { it.dependsOn(precommit) }
project.tasks.named(JavaPlugin.TEST_TASK_NAME).configure { it.mustRunAfter(precommit) }
// only require dependency licenses for non-elasticsearch deps
project.tasks.withType(DependencyLicensesTask).named('dependencyLicenses').configure {
it.dependencies = project.configurations.getByName(JavaPlugin.RUNTIME_CLASSPATH_CONFIGURATION_NAME).fileCollection { Dependency dependency ->
dependency.group.startsWith('org.elasticsearch') == false
} - project.configurations.getByName(JavaPlugin.COMPILE_ONLY_CONFIGURATION_NAME)
}
}

private static class TestFailureReportingPlugin implements Plugin<Project> {
@Override
Expand Down
31 changes: 31 additions & 0 deletions ...c/src/main/groovy/org/elasticsearch/gradle/precommit/LicenseHeadersPrecommitPlugin.groovy
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.elasticsearch.gradle.precommit

import org.gradle.api.Project
import org.gradle.api.Task
import org.gradle.api.tasks.TaskProvider

class LicenseHeadersPrecommitPlugin extends PrecommitPlugin {
@Override
TaskProvider<? extends Task> createTask(Project project) {
return project.getTasks().register("licenseHeaders", LicenseHeadersTask.class);
}
}
245 changes: 12 additions & 233 deletions buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,20 +18,8 @@
*/
package org.elasticsearch.gradle.precommit

import de.thetaphi.forbiddenapis.gradle.CheckForbiddenApis
import de.thetaphi.forbiddenapis.gradle.ForbiddenApisPlugin
import org.elasticsearch.gradle.ExportElasticsearchBuildResourcesTask
import org.elasticsearch.gradle.VersionProperties
import org.elasticsearch.gradle.info.BuildParams
import org.elasticsearch.gradle.util.Util
import org.gradle.api.JavaVersion

import org.gradle.api.Project
import org.gradle.api.artifacts.Configuration
import org.gradle.api.file.FileCollection
import org.gradle.api.plugins.JavaBasePlugin
import org.gradle.api.plugins.quality.Checkstyle
import org.gradle.api.tasks.SourceSet
import org.gradle.api.tasks.TaskProvider

/**
* Validation tasks which should be run before committing. These run before tests.
Expand All @@ -40,42 +28,21 @@ class PrecommitTasks {

/** Adds a precommit task, which depends on non-test verification tasks. */

public static TaskProvider create(Project project, boolean includeDependencyLicenses) {
project.configurations.create("forbiddenApisCliJar")
project.dependencies {
forbiddenApisCliJar('de.thetaphi:forbiddenapis:2.7')
}

Configuration jarHellConfig = project.configurations.create("jarHell")
if (BuildParams.internal && project.path.equals(":libs:elasticsearch-core") == false) {
// External plugins will depend on this already via transitive dependencies.
// Internal projects are not all plugins, so make sure the check is available
// we are not doing this for this project itself to avoid jar hell with itself
project.dependencies {
jarHell project.project(":libs:elasticsearch-core")
}
}
public static void create(Project project, boolean includeDependencyLicenses) {

List<TaskProvider> precommitTasks = [
configureCheckstyle(project),
configureForbiddenApisCli(project),
project.tasks.register('forbiddenPatterns', ForbiddenPatternsTask),
project.tasks.register('licenseHeaders', LicenseHeadersTask),
project.tasks.register('filepermissions', FilePermissionsTask),
configureJarHell(project, jarHellConfig),
configureThirdPartyAudit(project),
configureTestingConventions(project)
]
project.pluginManager.apply(CheckstylePrecommitPlugin)
project.pluginManager.apply(ForbiddenApisPrecommitPlugin)
project.pluginManager.apply(JarHellPrecommitPlugin)
project.pluginManager.apply(ForbiddenPatternsPrecommitPlugin)
project.pluginManager.apply(LicenseHeadersPrecommitPlugin)
project.pluginManager.apply(FilePermissionsPrecommitPlugin)
project.pluginManager.apply(ThirdPartyAuditPrecommitPlugin)
project.pluginManager.apply(TestingConventionsPrecommitPlugin)

// tasks with just tests don't need dependency licenses, so this flag makes adding
// the task optional
if (includeDependencyLicenses) {
TaskProvider<DependencyLicensesTask> dependencyLicenses = project.tasks.register('dependencyLicenses', DependencyLicensesTask)
precommitTasks.add(dependencyLicenses)
// we also create the updateShas helper task that is associated with dependencyLicenses
project.tasks.register('updateShas', UpdateShasTask) {
it.parentTask = dependencyLicenses
}
project.pluginManager.apply(DependencyLicensesPrecommitPlugin)
}
if (project.path != ':build-tools') {
/*
Expand All @@ -88,195 +55,7 @@ class PrecommitTasks {
* use the NamingConventionsCheck we break the circular dependency
* here.
*/
precommitTasks.add(configureLoggerUsage(project))
}

// We want to get any compilation error before running the pre-commit checks.
project.sourceSets.all { sourceSet ->
precommitTasks.each { provider ->
provider.configure {
shouldRunAfter(sourceSet.getClassesTaskName())
}
}
}

TaskProvider precommit = project.tasks.register('precommit') {
group = JavaBasePlugin.VERIFICATION_GROUP
description = 'Runs all non-test checks.'
dependsOn = precommitTasks
}

// not all jar projects produce a pom (we don't ship all jars), so a pom validation
// task is only added on some projects, and thus we can't always have a task
// here to add to precommit tasks explicitly. Instead, we apply our internal
// pom validation plugin after the precommit task is created and let the
// plugin add the task if necessary
project.plugins.apply(PomValidationPlugin)

return precommit
}

static TaskProvider configureTestingConventions(Project project) {
return project.getTasks().register("testingConventions", TestingConventionsTasks) {
naming {
Tests {
baseClass "org.apache.lucene.util.LuceneTestCase"
}
IT {
baseClass "org.elasticsearch.test.ESIntegTestCase"
baseClass 'org.elasticsearch.test.rest.ESRestTestCase'
}
}
}
}

private static TaskProvider configureJarHell(Project project, Configuration jarHellConfig) {
return project.tasks.register('jarHell', JarHellTask) { task ->
task.classpath = project.sourceSets.test.runtimeClasspath + jarHellConfig
task.dependsOn(jarHellConfig)
}
}

private static TaskProvider configureThirdPartyAudit(Project project) {
ExportElasticsearchBuildResourcesTask buildResources = project.tasks.getByName('buildResources')
return project.tasks.register('thirdPartyAudit', ThirdPartyAuditTask) { task ->
task.dependsOn(buildResources)
task.signatureFile = buildResources.copy("forbidden/third-party-audit.txt")
task.javaHome = BuildParams.runtimeJavaHome
task.targetCompatibility.set(project.provider({ BuildParams.runtimeJavaVersion }))
}
}

private static TaskProvider configureForbiddenApisCli(Project project) {
project.pluginManager.apply(ForbiddenApisPlugin)
ExportElasticsearchBuildResourcesTask buildResources = project.tasks.getByName('buildResources')
project.tasks.withType(CheckForbiddenApis).configureEach {
dependsOn(buildResources)

assert name.startsWith(ForbiddenApisPlugin.FORBIDDEN_APIS_TASK_NAME)
String sourceSetName
if (ForbiddenApisPlugin.FORBIDDEN_APIS_TASK_NAME.equals(name)) {
sourceSetName = "main"
} else {
//parse out the sourceSetName
char[] chars = name.substring(ForbiddenApisPlugin.FORBIDDEN_APIS_TASK_NAME.length()).toCharArray()
chars[0] = Character.toLowerCase(chars[0])
sourceSetName = new String(chars)
}

SourceSet sourceSet = project.sourceSets.getByName(sourceSetName)
classpath = project.files { sourceSet.runtimeClasspath.plus(sourceSet.compileClasspath) }

targetCompatibility = BuildParams.runtimeJavaVersion.majorVersion
if (BuildParams.runtimeJavaVersion > JavaVersion.VERSION_14) {
// TODO: forbidden apis does not yet support java 15, rethink using runtime version
targetCompatibility = JavaVersion.VERSION_14.majorVersion
}
bundledSignatures = [
"jdk-unsafe", "jdk-deprecated", "jdk-non-portable", "jdk-system-out"
]
signaturesFiles = project.files(
buildResources.copy("forbidden/jdk-signatures.txt"),
buildResources.copy("forbidden/es-all-signatures.txt")
)
suppressAnnotations = ['**.SuppressForbidden']
if (name.endsWith('Test')) {
signaturesFiles += project.files(
buildResources.copy("forbidden/es-test-signatures.txt"),
buildResources.copy("forbidden/http-signatures.txt")
)
} else {
signaturesFiles += project.files(buildResources.copy("forbidden/es-server-signatures.txt"))
}
ext.replaceSignatureFiles = { String... names ->
signaturesFiles = project.files(
names.collect { buildResources.copy("forbidden/${it}.txt") }
)
}
ext.addSignatureFiles = { String... names ->
signaturesFiles += project.files(
names.collect { buildResources.copy("forbidden/${it}.txt") }
)
}
}
TaskProvider forbiddenApis = project.tasks.named("forbiddenApis")
forbiddenApis.configure {
group = ""
}
return forbiddenApis
}

private static TaskProvider configureCheckstyle(Project project) {
// Always copy the checkstyle configuration files to 'buildDir/checkstyle' since the resources could be located in a jar
// file. If the resources are located in a jar, Gradle will fail when it tries to turn the URL into a file
URL checkstyleConfUrl = PrecommitTasks.getResource("/checkstyle.xml")
URL checkstyleSuppressionsUrl = PrecommitTasks.getResource("/checkstyle_suppressions.xml")
File checkstyleDir = new File(project.buildDir, "checkstyle")
File checkstyleSuppressions = new File(checkstyleDir, "checkstyle_suppressions.xml")
File checkstyleConf = new File(checkstyleDir, "checkstyle.xml");
TaskProvider copyCheckstyleConf = project.tasks.register("copyCheckstyleConf")

// configure inputs and outputs so up to date works properly
copyCheckstyleConf.configure {
outputs.files(checkstyleSuppressions, checkstyleConf)
}
if ("jar".equals(checkstyleConfUrl.getProtocol())) {
JarURLConnection jarURLConnection = (JarURLConnection) checkstyleConfUrl.openConnection()
copyCheckstyleConf.configure {
inputs.file(jarURLConnection.getJarFileURL())
}
} else if ("file".equals(checkstyleConfUrl.getProtocol())) {
copyCheckstyleConf.configure {
inputs.files(checkstyleConfUrl.getFile(), checkstyleSuppressionsUrl.getFile())
}
}

copyCheckstyleConf.configure {
doLast {
checkstyleDir.mkdirs()
// withStream will close the output stream and IOGroovyMethods#getBytes reads the InputStream fully and closes it
new FileOutputStream(checkstyleConf).withStream {
it.write(checkstyleConfUrl.openStream().getBytes())
}
new FileOutputStream(checkstyleSuppressions).withStream {
it.write(checkstyleSuppressionsUrl.openStream().getBytes())
}
}
}

TaskProvider checkstyleTask = project.tasks.register('checkstyle') {
dependsOn project.tasks.withType(Checkstyle)
}
// Apply the checkstyle plugin to create `checkstyleMain` and `checkstyleTest`. It only
// creates them if there is main or test code to check and it makes `check` depend
// on them. We also want `precommit` to depend on `checkstyle`.
project.pluginManager.apply('checkstyle')
project.checkstyle {
configDir = checkstyleDir
}
project.dependencies {
checkstyle "com.puppycrawl.tools:checkstyle:${VersionProperties.versions.checkstyle}"
checkstyle project.files(Util.buildSrcCodeSource)
}

project.tasks.withType(Checkstyle).configureEach { task ->
task.dependsOn(copyCheckstyleConf)
task.reports {
html.enabled false
}
}

return checkstyleTask
}

private static TaskProvider configureLoggerUsage(Project project) {
Object dependency = BuildParams.internal ? project.project(':test:logger-usage') :
"org.elasticsearch.test:logger-usage:${VersionProperties.elasticsearch}"

project.configurations.create('loggerUsagePlugin')
project.dependencies.add('loggerUsagePlugin', dependency)
return project.tasks.register('loggerUsageCheck', LoggerUsageTask) {
classpath = project.configurations.loggerUsagePlugin
project.pluginManager.apply(LoggerUsagePrecommitPlugin)
}
}
}
1 change: 0 additions & 1 deletion buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,5 @@ class StandaloneRestTestPlugin implements Plugin<Project> {
idea.module.scopes.put('TEST', [plus: [project.configurations.getByName(JavaPlugin.TEST_RUNTIME_CLASSPATH_CONFIGURATION_NAME)]] as Map<String, Collection<Configuration>>)

PrecommitTasks.create(project, false)
project.tasks.named('check').configure { it.dependsOn(project.tasks.named('precommit')) }
}
}
2 changes: 2 additions & 0 deletions buildSrc/src/main/java/org/elasticsearch/gradle/PublishPlugin.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import groovy.util.Node;
import groovy.util.NodeList;
import org.elasticsearch.gradle.info.BuildParams;
import org.elasticsearch.gradle.precommit.PomValidationPrecommitPlugin;
import org.elasticsearch.gradle.util.Util;
import org.gradle.api.Plugin;
import org.gradle.api.Project;
Expand All @@ -50,6 +51,7 @@ public class PublishPlugin implements Plugin<Project> {
@Override
public void apply(Project project) {
project.getPluginManager().apply("nebula.maven-base-publish");
project.getPluginManager().apply(PomValidationPrecommitPlugin.class);

configureJavadocJar(project);
configureSourcesJar(project);
Expand Down
Loading

0 comments on commit 1283065

Please sign in to comment.