Skip to content

Commit

Permalink
Add snapshot support to distribution download plugin (#47837)
Browse files Browse the repository at this point in the history 
The distribution download plugin which handles finding built
distributions for testing currently only knows how to find locally built
snapshots. When an external Elasticsearch plugin uses build-tools, these
snapshots do not exist. This commit extends the download plugin so it
pulls from the Elastic snapshots service when used outside of the
Elasticsearch repository.

closes #47123
  • Loading branch information
@rjernst
rjernst committed Oct 11, 2019
1 parent 6ab58de commit dc8080e
Show file tree
Hide file tree
Showing 7 changed files with 143 additions and 80 deletions.
68 changes: 43 additions & 25 deletions buildSrc/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import org.elasticsearch.gradle.ElasticsearchDistribution.Flavor;
import org.elasticsearch.gradle.ElasticsearchDistribution.Platform;
import org.elasticsearch.gradle.ElasticsearchDistribution.Type;
import org.elasticsearch.gradle.tool.ClasspathUtils;
import org.gradle.api.GradleException;
import org.gradle.api.NamedDomainObjectContainer;
import org.gradle.api.Plugin;
Expand Down Expand Up @@ -56,7 +57,9 @@ public class DistributionDownloadPlugin implements Plugin<Project> {

private static final String CONTAINER_NAME = "elasticsearch_distributions";
private static final String FAKE_IVY_GROUP = "elasticsearch-distribution";
private static final String FAKE_SNAPSHOT_IVY_GROUP = "elasticsearch-distribution-snapshot";
private static final String DOWNLOAD_REPO_NAME = "elasticsearch-downloads";
private static final String SNAPSHOT_REPO_NAME = "elasticsearch-snapshots";

private BwcVersions bwcVersions;
private NamedDomainObjectContainer<ElasticsearchDistribution> distributionsContainer;
Expand All @@ -72,9 +75,10 @@ public void apply(Project project) {

setupDownloadServiceRepo(project);

ExtraPropertiesExtension extraProperties = project.getExtensions().getExtraProperties();
this.bwcVersions = (BwcVersions) extraProperties.get("bwcVersions");
// TODO: setup snapshot dependency instead of pointing to bwc distribution projects for external projects
if (ClasspathUtils.isElasticsearchProject(project)) {
ExtraPropertiesExtension extraProperties = project.getExtensions().getExtraProperties();
this.bwcVersions = (BwcVersions) extraProperties.get("bwcVersions");
}

project.afterEvaluate(this::setupDistributions);
}
Expand Down Expand Up @@ -148,13 +152,10 @@ private void setupRootDownload(Project rootProject, ElasticsearchDistribution di
}
}

private static void setupDownloadServiceRepo(Project project) {
if (project.getRepositories().findByName(DOWNLOAD_REPO_NAME) != null) {
return;
}
private static void addIvyRepo(Project project, String name, String url, String group) {
project.getRepositories().ivy(ivyRepo -> {
ivyRepo.setName(DOWNLOAD_REPO_NAME);
ivyRepo.setUrl("https://artifacts.elastic.co");
ivyRepo.setName(name);
ivyRepo.setUrl(url);
ivyRepo.metadataSources(IvyArtifactRepository.MetadataSources::artifact);
// this header is not a credential but we hack the capability to send this header to avoid polluting our download stats
ivyRepo.credentials(HttpHeaderCredentials.class, creds -> {
Expand All @@ -163,15 +164,25 @@ private static void setupDownloadServiceRepo(Project project) {
});
ivyRepo.getAuthentication().create("header", HttpHeaderAuthentication.class);
ivyRepo.patternLayout(layout -> layout.artifact("/downloads/elasticsearch/[module]-[revision](-[classifier]).[ext]"));
ivyRepo.content(content -> content.includeGroup(FAKE_IVY_GROUP));
ivyRepo.content(content -> content.includeGroup(group));
});
project.getRepositories().all(repo -> {
if (repo.getName().equals(DOWNLOAD_REPO_NAME) == false) {
if (repo.getName().equals(name) == false) {
// all other repos should ignore the special group name
repo.content(content -> content.excludeGroup(FAKE_IVY_GROUP));
repo.content(content -> content.excludeGroup(group));
}
});
// TODO: need maven repo just for integ-test-zip, but only in external cases
}

private static void setupDownloadServiceRepo(Project project) {
if (project.getRepositories().findByName(DOWNLOAD_REPO_NAME) != null) {
return;
}
addIvyRepo(project, DOWNLOAD_REPO_NAME, "https://artifacts.elastic.co", FAKE_IVY_GROUP);
if (ClasspathUtils.isElasticsearchProject(project) == false) {
// external, so add snapshot repo as well
addIvyRepo(project, SNAPSHOT_REPO_NAME, "https://snapshots.elastic.co", FAKE_SNAPSHOT_IVY_GROUP);
}
}

/**
Expand All @@ -187,25 +198,30 @@ private static void setupDownloadServiceRepo(Project project) {
*/
private Object dependencyNotation(Project project, ElasticsearchDistribution distribution) {

if (Version.fromString(VersionProperties.getElasticsearch()).equals(distribution.getVersion())) {
return projectDependency(project, distributionProjectPath(distribution), "default");
// TODO: snapshot dep when not in ES repo
}
BwcVersions.UnreleasedVersionInfo unreleasedInfo = bwcVersions.unreleasedInfo(distribution.getVersion());
if (unreleasedInfo != null) {
assert distribution.getBundledJdk();
return projectDependency(project, unreleasedInfo.gradleProjectPath, distributionProjectName(distribution));
if (ClasspathUtils.isElasticsearchProject(project)) {
// non-external project, so depend on local build

if (VersionProperties.getElasticsearch().equals(distribution.getVersion())) {
return projectDependency(project, distributionProjectPath(distribution), "default");
}
BwcVersions.UnreleasedVersionInfo unreleasedInfo = bwcVersions.unreleasedInfo(Version.fromString(distribution.getVersion()));
if (unreleasedInfo != null) {
assert distribution.getBundledJdk();
return projectDependency(project, unreleasedInfo.gradleProjectPath, distributionProjectName(distribution));
}
}

if (distribution.getType() == Type.INTEG_TEST_ZIP) {
return "org.elasticsearch.distribution.integ-test-zip:elasticsearch:" + distribution.getVersion();
}


Version distroVersion = Version.fromString(distribution.getVersion());
String extension = distribution.getType().toString();
String classifier = ":x86_64";
if (distribution.getType() == Type.ARCHIVE) {
extension = distribution.getPlatform() == Platform.WINDOWS ? "zip" : "tar.gz";
if (distribution.getVersion().onOrAfter("7.0.0")) {
if (distroVersion.onOrAfter("7.0.0")) {
classifier = ":" + distribution.getPlatform() + "-x86_64";
} else {
classifier = "";
Expand All @@ -214,10 +230,12 @@ private Object dependencyNotation(Project project, ElasticsearchDistribution dis
classifier = ":amd64";
}
String flavor = "";
if (distribution.getFlavor() == Flavor.OSS && distribution.getVersion().onOrAfter("6.3.0")) {
if (distribution.getFlavor() == Flavor.OSS && distroVersion.onOrAfter("6.3.0")) {
flavor = "-oss";
}
return FAKE_IVY_GROUP + ":elasticsearch" + flavor + ":" + distribution.getVersion() + classifier + "@" + extension;

String group = distribution.getVersion().endsWith("-SNAPSHOT") ? FAKE_SNAPSHOT_IVY_GROUP : FAKE_IVY_GROUP;
return group + ":elasticsearch" + flavor + ":" + distribution.getVersion() + classifier + "@" + extension;
}

private static Dependency projectDependency(Project project, String projectPath, String projectConfig) {
Expand Down Expand Up @@ -251,7 +269,7 @@ private static String distributionProjectName(ElasticsearchDistribution distribu
projectName += "no-jdk-";
}
if (distribution.getType() == Type.ARCHIVE) {
if (distribution.getVersion().onOrAfter("7.0.0")) {
if (Version.fromString(distribution.getVersion()).onOrAfter("7.0.0")) {
Platform platform = distribution.getPlatform();
projectName += platform.toString() + (platform == Platform.WINDOWS ? "-zip" : "-tar");
} else {
Expand Down
10 changes: 5 additions & 5 deletions buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchDistribution.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ public String toString() {
final Configuration configuration;
private final Extracted extracted;

private final Property<Version> version;
private final Property<String> version;
private final Property<Type> type;
private final Property<Platform> platform;
private final Property<Flavor> flavor;
Expand All @@ -111,8 +111,7 @@ public String toString() {
Configuration extractedConfiguration) {
this.name = name;
this.configuration = fileConfiguration;
this.version = objectFactory.property(Version.class);
this.version.convention(Version.fromString(VersionProperties.getElasticsearch()));
this.version = objectFactory.property(String.class).convention(VersionProperties.getElasticsearch());
this.type = objectFactory.property(Type.class);
this.type.convention(Type.ARCHIVE);
this.platform = objectFactory.property(Platform.class);
Expand All @@ -125,12 +124,13 @@ public String getName() {
return name;
}

public Version getVersion() {
public String getVersion() {
return version.get();
}

public void setVersion(String version) {
this.version.set(Version.fromString(version));
Version.fromString(version); // ensure the version parses, but don't store as Version since that removes -SNAPSHOT
this.version.set(version);
}

public Platform getPlatform() {
Expand Down
2 changes: 1 addition & 1 deletion buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -180,7 +180,7 @@ public String getName() {

@Internal
public Version getVersion() {
return distributions.get(currentDistro).getVersion();
return Version.fromString(distributions.get(currentDistro).getVersion());
}

@Internal
Expand Down
50 changes: 40 additions & 10 deletions buildSrc/src/test/java/org/elasticsearch/gradle/DistributionDownloadPluginIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@
import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo;

public class DistributionDownloadPluginIT extends GradleIntegrationTestCase {

// TODO: check reuse of root task across projects MOVE TO UNIT TEST
// TODO: future: check integ-test-zip to maven, snapshots to snapshot service for external project

Expand All @@ -48,32 +49,61 @@ public void testCurrent() throws Exception {
"tests.local_distro.project", projectName);
}

public void testCurrentExternal() throws Exception {
checkService(VersionProperties.getElasticsearch(), "archive", "linux", null, null,
"/downloads/elasticsearch/elasticsearch-" + VersionProperties.getElasticsearch() + "-linux-x86_64.tar.gz",
"tests.internal", "false");
}

public void testBwc() throws Exception {
assertExtractedDistro("1.1.0", "archive", "linux", null, null,
"tests.local_distro.config", "zip",
assertExtractedDistro("8.1.0", "archive", "linux", null, null,
"tests.local_distro.config", "linux-tar",
"tests.local_distro.project", ":distribution:bwc:minor",
"tests.current_version", "2.0.0");
"tests.current_version", "8.0.0");
}

public void testBwcExternal() throws Exception {
checkService("8.1.0-SNAPSHOT", "archive", "linux", null, null,
"/downloads/elasticsearch/elasticsearch-8.1.0-SNAPSHOT-linux-x86_64.tar.gz",
"tests.internal", "false",
"tests.current_version", "9.0.0");
}

public void testReleased() throws Exception {
doTestReleased("7.0.0", "/downloads/elasticsearch/elasticsearch-7.0.0-windows-x86_64.zip");
doTestReleased("6.5.0", "/downloads/elasticsearch/elasticsearch-6.5.0.zip");
checkService("7.0.0", "archive", "windows", null, null,
"/downloads/elasticsearch/elasticsearch-7.0.0-windows-x86_64.zip");
checkService("6.5.0", "archive", "windows", null, null,
"/downloads/elasticsearch/elasticsearch-6.5.0.zip");
}

public void testReleasedExternal() throws Exception {
checkService("7.0.0", "archive", "windows", null, null,
"/downloads/elasticsearch/elasticsearch-7.0.0-windows-x86_64.zip",
"tests.internal", "false");
checkService("6.5.0", "archive", "windows", null, null,
"/downloads/elasticsearch/elasticsearch-6.5.0.zip",
"tests.internal", "false");
}

private void doTestReleased(String version, String urlPath) throws IOException {
private void checkService(String version, String type, String platform, String flavor, Boolean bundledJdk,
String urlPath, String... sysProps) throws IOException {
String suffix = urlPath.endsWith("zip") ? "zip" : "tar.gz";
String sourceFile = "src/testKit/distribution-download/distribution/files/fake_elasticsearch." + suffix;
WireMockServer wireMock = new WireMockServer(0);
try {
final byte[] filebytes;
try (InputStream stream =
Files.newInputStream(Paths.get("src/testKit/distribution-download/distribution/files/fake_elasticsearch.zip"))) {
try (InputStream stream = Files.newInputStream(Paths.get(sourceFile))) {
filebytes = stream.readAllBytes();
}
wireMock.stubFor(head(urlEqualTo(urlPath)).willReturn(aResponse().withStatus(200)));
wireMock.stubFor(get(urlEqualTo(urlPath)).willReturn(aResponse().withStatus(200).withBody(filebytes)));
wireMock.start();

assertExtractedDistro(version, "archive", "windows", null, null,
"tests.download_service", wireMock.baseUrl());
List<String> allSysProps = new ArrayList<>();
allSysProps.addAll(Arrays.asList(sysProps));
allSysProps.add("tests.download_service");
allSysProps.add(wireMock.baseUrl());
assertExtractedDistro(version, type, platform, flavor, bundledJdk, allSysProps.toArray(new String[0]));
} catch (Exception e) {
// for debugging
System.err.println("missed requests: " + wireMock.findUnmatchedRequests().getRequests());
Expand Down
Loading

0 comments on commit dc8080e

Please sign in to comment.