Skip to content

Commit

Permalink
Move common repository configuration to java plugin (#57057)
Browse files Browse the repository at this point in the history 
This commit moves the common maven repository configuration to the ES
java plugin. All java projects need this common set of repos. Note that
the Elastic download and maven repos are removed, as they are not
necessary anymore since distribution download was split into the
DistributionDownloadPlugin.
  • Loading branch information
@rjernst
rjernst authored May 26, 2020
1 parent 86b64e4 commit deafcac
Show file tree
Hide file tree
Showing 6 changed files with 83 additions and 175 deletions.
2 changes: 0 additions & 2 deletions build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,8 +54,6 @@ allprojects {
description = "Elasticsearch subproject ${project.path}"
}

BuildPlugin.configureRepositories(project)

String licenseCommit
if (VersionProperties.elasticsearch.toString().endsWith('-SNAPSHOT')) {
licenseCommit = BuildParams.gitRevision ?: "master" // leniency for non git builds
Expand Down
120 changes: 0 additions & 120 deletions buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@
*/
package org.elasticsearch.gradle


import groovy.transform.CompileStatic
import org.apache.commons.io.IOUtils
import org.elasticsearch.gradle.info.GlobalBuildInfoPlugin
Expand All @@ -44,7 +43,6 @@ import org.gradle.api.plugins.ExtraPropertiesExtension
import org.gradle.api.plugins.JavaPlugin
import org.gradle.api.tasks.bundling.Jar
import org.gradle.api.tasks.testing.Test
import org.gradle.authentication.http.HttpHeaderAuthentication
import org.gradle.util.GradleVersion

import java.nio.charset.StandardCharsets
Expand Down Expand Up @@ -87,7 +85,6 @@ class BuildPlugin implements Plugin<Project> {

project.getTasks().register("buildResources", ExportElasticsearchBuildResourcesTask)

configureRepositories(project)
project.extensions.getByType(ExtraPropertiesExtension).set('versions', VersionProperties.versions)
PrecommitTasks.create(project, true)
configureFips140(project)
Expand Down Expand Up @@ -137,123 +134,6 @@ class BuildPlugin implements Plugin<Project> {
}
}

/**
* Makes dependencies non-transitive.
*
* Gradle allows setting all dependencies as non-transitive very easily.
* Sadly this mechanism does not translate into maven pom generation. In order
* to effectively make the pom act as if it has no transitive dependencies,
* we must exclude each transitive dependency of each direct dependency.
*
* Determining the transitive deps of a dependency which has been resolved as
* non-transitive is difficult because the process of resolving removes the
* transitive deps. To sidestep this issue, we create a configuration per
* direct dependency version. This specially named and unique configuration
* will contain all of the transitive dependencies of this particular
* dependency. We can then use this configuration during pom generation
* to iterate the transitive dependencies and add excludes.
*/
static void configureConfigurations(Project project) {
// we want to test compileOnly deps!
project.configurations.getByName(JavaPlugin.TEST_COMPILE_CONFIGURATION_NAME).extendsFrom(project.configurations.getByName(JavaPlugin.COMPILE_ONLY_CONFIGURATION_NAME))

// we are not shipping these jars, we act like dumb consumers of these things
if (project.path.startsWith(':test:fixtures') || project.path == ':build-tools') {
return
}
// fail on any conflicting dependency versions
project.configurations.all({ Configuration configuration ->
if (configuration.name.endsWith('Fixture')) {
// just a self contained test-fixture configuration, likely transitive and hellacious
return
}
configuration.resolutionStrategy {
failOnVersionConflict()
}
})

// force all dependencies added directly to compile/testCompile to be non-transitive, except for ES itself
Closure disableTransitiveDeps = { Dependency dep ->
if (dep instanceof ModuleDependency && !(dep instanceof ProjectDependency)
&& dep.group.startsWith('org.elasticsearch') == false) {
dep.transitive = false
}
}

project.configurations.getByName(JavaPlugin.COMPILE_CONFIGURATION_NAME).dependencies.all(disableTransitiveDeps)
project.configurations.getByName(JavaPlugin.TEST_COMPILE_CONFIGURATION_NAME).dependencies.all(disableTransitiveDeps)
project.configurations.getByName(JavaPlugin.COMPILE_ONLY_CONFIGURATION_NAME).dependencies.all(disableTransitiveDeps)
project.configurations.getByName(JavaPlugin.RUNTIME_ONLY_CONFIGURATION_NAME).dependencies.all(disableTransitiveDeps)
}

/** Adds repositories used by ES dependencies */
static void configureRepositories(Project project) {
project.getRepositories().all { repository ->
if (repository instanceof MavenArtifactRepository) {
final MavenArtifactRepository maven = (MavenArtifactRepository) repository
assertRepositoryURIIsSecure(maven.name, project.path, maven.getUrl())
repository.getArtifactUrls().each { uri -> assertRepositoryURIIsSecure(maven.name, project.path, uri) }
} else if (repository instanceof IvyArtifactRepository) {
final IvyArtifactRepository ivy = (IvyArtifactRepository) repository
assertRepositoryURIIsSecure(ivy.name, project.path, ivy.getUrl())
}
}
RepositoryHandler repos = project.repositories
if (System.getProperty('repos.mavenLocal') != null) {
// with -Drepos.mavenLocal=true we can force checking the local .m2 repo which is
// useful for development ie. bwc tests where we install stuff in the local repository
// such that we don't have to pass hardcoded files to gradle
repos.mavenLocal()
}
repos.jcenter()
repos.ivy { IvyArtifactRepository repo ->
repo.name = 'elasticsearch'
repo.url = 'https://artifacts.elastic.co/downloads'
repo.patternLayout { IvyPatternRepositoryLayout layout ->
layout.artifact 'elasticsearch/[module]-[revision](-[classifier]).[ext]'
}
// this header is not a credential but we hack the capability to send this header to avoid polluting our download stats
repo.credentials(HttpHeaderCredentials, { HttpHeaderCredentials creds ->
creds.name = 'X-Elastic-No-KPI'
creds.value = '1'
} as Action<HttpHeaderCredentials>)
repo.authentication.create('header', HttpHeaderAuthentication)
}
repos.maven { MavenArtifactRepository repo ->
repo.name = 'elastic'
repo.url = 'https://artifacts.elastic.co/maven'
}
String luceneVersion = VersionProperties.lucene
if (luceneVersion.contains('-snapshot')) {
// extract the revision number from the version with a regex matcher
List<String> matches = (luceneVersion =~ /\w+-snapshot-([a-z0-9]+)/).getAt(0) as List<String>
String revision = matches.get(1)
MavenArtifactRepository luceneRepo = repos.maven { MavenArtifactRepository repo ->
repo.name = 'lucene-snapshots'
repo.url = "https://s3.amazonaws.com/download.elasticsearch.org/lucenesnapshots/${revision}"
}
repos.exclusiveContent { ExclusiveContentRepository exclusiveRepo ->
exclusiveRepo.filter {
it.includeVersionByRegex(/org\.apache\.lucene/, '.*', ".*-snapshot-${revision}")
}
exclusiveRepo.forRepositories(luceneRepo)
}
}
}

static void assertRepositoryURIIsSecure(final String repositoryName, final String projectPath, final URI uri) {
if (uri != null && ["file", "https", "s3"].contains(uri.getScheme()) == false) {
final String message = String.format(
Locale.ROOT,
"repository [%s] on project with path [%s] is not using a secure protocol for artifacts on [%s]",
repositoryName,
projectPath,
uri.toURL())
throw new GradleException(message)
}
}


private static class TestFailureReportingPlugin implements Plugin<Project> {
@Override
void apply(Project project) {
Expand Down
2 changes: 1 addition & 1 deletion buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ class StandaloneRestTestPlugin implements Plugin<Project> {
project.pluginManager.apply(TestClustersPlugin)

project.getTasks().create("buildResources", ExportElasticsearchBuildResourcesTask)
BuildPlugin.configureRepositories(project)
ElasticsearchJavaPlugin.configureRepositories(project)
ElasticsearchJavaPlugin.configureTestTasks(project)
ElasticsearchJavaPlugin.configureInputNormalization(project)
BuildPlugin.configureFips140(project)
Expand Down
79 changes: 79 additions & 0 deletions buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchJavaPlugin.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,9 @@
import org.gradle.api.artifacts.ModuleDependency;
import org.gradle.api.artifacts.ProjectDependency;
import org.gradle.api.artifacts.ResolutionStrategy;
import org.gradle.api.artifacts.dsl.RepositoryHandler;
import org.gradle.api.artifacts.repositories.IvyArtifactRepository;
import org.gradle.api.artifacts.repositories.MavenArtifactRepository;
import org.gradle.api.file.FileCollection;
import org.gradle.api.plugins.BasePlugin;
import org.gradle.api.plugins.JavaPlugin;
Expand All @@ -55,12 +58,18 @@
import java.io.File;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.net.MalformedURLException;
import java.net.URI;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Arrays;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.function.Consumer;
import java.util.function.Function;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import static org.elasticsearch.gradle.util.GradleUtils.maybeConfigure;
import static org.elasticsearch.gradle.util.Util.toStringable;
Expand All @@ -76,6 +85,7 @@ public void apply(Project project) {

project.getPluginManager().apply(JavaPlugin.class);
configureConfigurations(project);
configureRepositories(project);
configureCompile(project);
configureInputNormalization(project);
configureTestTasks(project);
Expand Down Expand Up @@ -136,6 +146,75 @@ public static void configureConfigurations(Project project) {
disableTransitiveDeps.accept(JavaPlugin.RUNTIME_ONLY_CONFIGURATION_NAME);
}

private static final Pattern LUCENE_SNAPSHOT_REGEX = Pattern.compile("\\w+-snapshot-([a-z0-9]+)");

/** Adds repositories used by ES dependencies */
public static void configureRepositories(Project project) {
// ensure all repositories use secure urls
// TODO: remove this with gradle 7.0, which no longer allows insecure urls
project.getRepositories().all(repository -> {
if (repository instanceof MavenArtifactRepository) {
final MavenArtifactRepository maven = (MavenArtifactRepository) repository;
assertRepositoryURIIsSecure(maven.getName(), project.getPath(), maven.getUrl());
for (URI uri : maven.getArtifactUrls()) {
assertRepositoryURIIsSecure(maven.getName(), project.getPath(), uri);
}
} else if (repository instanceof IvyArtifactRepository) {
final IvyArtifactRepository ivy = (IvyArtifactRepository) repository;
assertRepositoryURIIsSecure(ivy.getName(), project.getPath(), ivy.getUrl());
}
});
RepositoryHandler repos = project.getRepositories();
if (System.getProperty("repos.mavenLocal") != null) {
// with -Drepos.mavenLocal=true we can force checking the local .m2 repo which is
// useful for development ie. bwc tests where we install stuff in the local repository
// such that we don't have to pass hardcoded files to gradle
repos.mavenLocal();
}
repos.jcenter();

String luceneVersion = VersionProperties.getLucene();
if (luceneVersion.contains("-snapshot")) {
// extract the revision number from the version with a regex matcher
Matcher matcher = LUCENE_SNAPSHOT_REGEX.matcher(luceneVersion);
if (matcher.find() == false) {
throw new GradleException("Malformed lucene snapshot version: " + luceneVersion);
}
String revision = matcher.group(1);
MavenArtifactRepository luceneRepo = repos.maven(repo -> {
repo.setName("lucene-snapshots");
repo.setUrl("https://s3.amazonaws.com/download.elasticsearch.org/lucenesnapshots/" + revision);
});
repos.exclusiveContent(exclusiveRepo -> {
exclusiveRepo.filter(
descriptor -> descriptor.includeVersionByRegex("org\\.apache\\.lucene", ".*", ".*-snapshot-" + revision)
);
exclusiveRepo.forRepositories(luceneRepo);
});
}
}

private static final List<String> SECURE_URL_SCHEMES = Arrays.asList("file", "https", "s3");

private static void assertRepositoryURIIsSecure(final String repositoryName, final String projectPath, final URI uri) {
if (uri != null && SECURE_URL_SCHEMES.contains(uri.getScheme()) == false) {
String url;
try {
url = uri.toURL().toString();
} catch (MalformedURLException e) {
throw new IllegalStateException(e);
}
final String message = String.format(
Locale.ROOT,
"repository [%s] on project with path [%s] is not using a secure protocol for artifacts on [%s]",
repositoryName,
projectPath,
url
);
throw new GradleException(message);
}
}

/** Adds compiler settings to the project */
public static void configureCompile(Project project) {
project.getExtensions().getExtraProperties().set("compactProfile", "full");
Expand Down
51 changes: 0 additions & 51 deletions buildSrc/src/test/java/org/elasticsearch/gradle/BuildPluginTests.java
View file

This file was deleted.

4 changes: 3 additions & 1 deletion buildSrc/src/test/java/org/elasticsearch/gradle/test/GradleIntegrationTestCase.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@
import java.util.stream.Collectors;
import java.util.stream.Stream;

import static org.hamcrest.CoreMatchers.containsString;

public abstract class GradleIntegrationTestCase extends GradleUnitTestCase {

@Rule
Expand Down Expand Up @@ -71,7 +73,7 @@ protected void assertOutputContains(String output, Set<String> lines) {
}

protected void assertOutputContains(String output, String line) {
assertTrue("Expected the following line in output:\n\n" + line + "\n\nOutput is:\n" + output, output.contains(line));
assertThat("Expected the following line in output:\n\n" + line + "\n\nOutput is:\n" + output, output, containsString(line));
}

protected void assertOutputDoesNotContain(String output, String line) {
Expand Down

0 comments on commit deafcac

Please sign in to comment.