From 2b5535c0e50450b6981c3f0199c5b54eb43366c1 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Tue, 22 Jan 2019 14:05:36 -0800 Subject: [PATCH 1/4] Use project dependency instead of substitutions for distributions Currently integration tests which use either bwc snapshot versions or the current version of elasticsearch depend on project substitutions to link to the build of those artifacts. Likewise, vagrant tests use dependency substitutions to get to bwc snapshots of rpm and debs. This commit changes those to depend on the relevant project/configuration and removes the dependency substitutions for distributions we do not publish. --- build.gradle | 21 ----------- .../gradle/test/ClusterFormationTasks.groovy | 18 +++++++--- .../gradle/vagrant/VagrantTestPlugin.groovy | 35 ++++++++++++++----- .../gradle/VersionCollection.java | 20 +++++++++++ distribution/bwc/build.gradle | 31 +++++++++------- 5 files changed, 77 insertions(+), 48 deletions(-) diff --git a/build.gradle b/build.gradle index dade656f78b82..c5611e8b453fb 100644 --- a/build.gradle +++ b/build.gradle @@ -221,14 +221,6 @@ allprojects { "org.elasticsearch.plugin:elasticsearch-scripting-painless-spi:${version}": ':modules:lang-painless:spi', "org.elasticsearch.test:framework:${version}": ':test:framework', "org.elasticsearch.distribution.integ-test-zip:elasticsearch:${version}": ':distribution:archives:integ-test-zip', - "downloads.zip:elasticsearch:${version}": ':distribution:archives:zip', - "downloads.zip:elasticsearch-oss:${version}": ':distribution:archives:oss-zip', - "downloads.tar:elasticsearch:${version}": ':distribution:archives:tar', - "downloads.tar:elasticsearch-oss:${version}": ':distribution:archives:oss-tar', - "downloads.rpm:elasticsearch:${version}": ':distribution:packages:rpm', - "downloads.rpm:elasticsearch-oss:${version}": ':distribution:packages:oss-rpm', - "downloads.deb:elasticsearch:${version}": ':distribution:packages:deb', - "downloads.deb:elasticsearch-oss:${version}": ':distribution:packages:oss-deb', "org.elasticsearch.test:logger-usage:${version}": ':test:logger-usage', "org.elasticsearch.xpack.test:feature-aware:${version}": ':x-pack:test:feature-aware', // for transport client @@ -240,19 +232,6 @@ allprojects { "org.elasticsearch.plugin:percolator-client:${version}": ':modules:percolator', "org.elasticsearch.plugin:rank-eval-client:${version}": ':modules:rank-eval', ] - // substitute unreleased versions with projects that check out and build locally - bwcVersions.forPreviousUnreleased { VersionCollection.UnreleasedVersionInfo unreleasedVersion -> - Version unreleased = unreleasedVersion.version - String snapshotProject = ":distribution:bwc:${unreleasedVersion.gradleProjectName}" - ext.projectSubstitutions["downloads.deb:elasticsearch:${unreleased}"] = snapshotProject - ext.projectSubstitutions["downloads.rpm:elasticsearch:${unreleased}"] = snapshotProject - ext.projectSubstitutions["downloads.zip:elasticsearch:${unreleased}"] = snapshotProject - if (unreleased.onOrAfter('6.3.0')) { - ext.projectSubstitutions["downloads.deb:elasticsearch-oss:${unreleased}"] = snapshotProject - ext.projectSubstitutions["downloads.rpm:elasticsearch-oss:${unreleased}"] = snapshotProject - ext.projectSubstitutions["downloads.zip:elasticsearch-oss:${unreleased}"] = snapshotProject - } - } /* * Gradle only resolve project substitutions during dependency resolution but diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy index 72041b4e628d3..d2552875ffd60 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy @@ -23,6 +23,7 @@ import org.apache.tools.ant.taskdefs.condition.Os import org.elasticsearch.gradle.BuildPlugin import org.elasticsearch.gradle.LoggedExec import org.elasticsearch.gradle.Version +import org.elasticsearch.gradle.VersionCollection import org.elasticsearch.gradle.VersionProperties import org.elasticsearch.gradle.plugin.PluginBuildPlugin import org.elasticsearch.gradle.plugin.PluginPropertiesExtension @@ -184,10 +185,7 @@ class ClusterFormationTasks { throw new GradleException("Unknown distribution: ${distro} in project ${project.path}") } Version version = Version.fromString(elasticsearchVersion) - if (version.before('6.3.0') && distro.startsWith('oss-')) { - distro = distro.substring('oss-'.length()) - } - String group = "downloads.zip" + String group = "downloads.zip" // dummy group, does not matter except for integ-test-zip, it is ignored by the fake ivy repo if (distro.equals("integ-test-zip")) { group = "org.elasticsearch.distribution.integ-test-zip" } @@ -195,7 +193,17 @@ class ClusterFormationTasks { if (distro.equals('oss') && Version.fromString(elasticsearchVersion).onOrAfter('6.3.0')) { artifactName += '-oss' } - project.dependencies.add(configuration.name, "${group}:${artifactName}:${elasticsearchVersion}@zip") + String snapshotProject = distro == 'oss' ? 'oss-zip' : 'zip' + Object dependency + VersionCollection.UnreleasedVersionInfo unreleasedInfo = project.bwcVersions.unreleasedInfo(version) + if (unreleasedInfo != null) { + dependency = project.dependencies.project(path: ":distribution:bwc:${unreleasedInfo.gradleProjectName}", configuration: snapshotProject) + } else if (elasticsearchVersion.equals(VersionProperties.elasticsearch)) { + dependency = project.dependencies.project(path: ":distribution:archives:${snapshotProject}") + } else { + dependency = "${group}:${artifactName}:${elasticsearchVersion}@zip" + } + project.dependencies.add(configuration.name, dependency) } /** Adds a dependency on a different version of the given plugin, which will be retrieved using gradle's dependency resolution */ diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy index dfb825afea255..fec6b2eab7405 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy @@ -4,6 +4,7 @@ import org.apache.tools.ant.taskdefs.condition.Os import org.elasticsearch.gradle.FileContentsTask import org.elasticsearch.gradle.LoggedExec import org.elasticsearch.gradle.Version +import org.elasticsearch.gradle.VersionCollection import org.gradle.api.* import org.gradle.api.artifacts.dsl.RepositoryHandler import org.gradle.api.execution.TaskExecutionAdapter @@ -184,22 +185,38 @@ class VagrantTestPlugin implements Plugin { upgradeFromVersion = Version.fromString(upgradeFromVersionRaw) } + List dependencies = new ArrayList<>() DISTRIBUTIONS.each { // Adds a dependency for the current version - project.dependencies.add(PACKAGING_CONFIGURATION, - project.dependencies.project(path: ":distribution:${it}", configuration: 'default')) + dependencies.add(project.dependencies.project(path: ":distribution:${it}", configuration: 'default')) } - UPGRADE_FROM_ARCHIVES.each { - // The version of elasticsearch that we upgrade *from* - project.dependencies.add(PACKAGING_CONFIGURATION, - "downloads.${it}:elasticsearch:${upgradeFromVersion}@${it}") - if (upgradeFromVersion.onOrAfter('6.3.0')) { - project.dependencies.add(PACKAGING_CONFIGURATION, - "downloads.${it}:elasticsearch-oss:${upgradeFromVersion}@${it}") + // The version of elasticsearch that we upgrade *from* + VersionCollection.UnreleasedVersionInfo unreleasedInfo = project.bwcVersions.unreleasedInfo(upgradeFromVersion) + if (unreleasedInfo != null) { + // handle snapshots pointing to bwc build + UPGRADE_FROM_ARCHIVES.each { + dependencies.add(project.dependencies.project( + path: ":distribution:bwc:${unreleasedInfo.gradleProjectName}", configuration: it)) + if (upgradeFromVersion.onOrAfter('6.3.0')) { + dependencies.add(project.dependencies.project( + path: ":distribution:bwc:${unreleasedInfo.gradleProjectName}", configuration: "oss-${it}")) + } + } + } else { + UPGRADE_FROM_ARCHIVES.each { + // The version of elasticsearch that we upgrade *from* + dependencies.add("downloads.${it}:elasticsearch:${upgradeFromVersion}@${it}") + if (upgradeFromVersion.onOrAfter('6.3.0')) { + dependencies.add("downloads.${it}:elasticsearch-oss:${upgradeFromVersion}@${it}") + } } } + for (Object dependency : dependencies) { + project.dependencies.add(PACKAGING_CONFIGURATION, dependency) + } + project.extensions.esvagrant.upgradeFromVersion = upgradeFromVersion } diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/VersionCollection.java b/buildSrc/src/main/java/org/elasticsearch/gradle/VersionCollection.java index 1cf2fd9e1037c..e2af34dbabdc0 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/VersionCollection.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/VersionCollection.java @@ -20,6 +20,8 @@ import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -86,6 +88,7 @@ public class VersionCollection { private final Version currentVersion; private final Map> groupByMajor; + private final Map unreleased; public class UnreleasedVersionInfo { public final Version version; @@ -129,6 +132,16 @@ protected VersionCollection(List versionLines, Version currentVersionPro assertCurrentVersionMatchesParsed(currentVersionProperty); assertNoOlderThanTwoMajors(); + + Map unreleased = new HashMap<>(); + for (Version unreleasedVersion : getUnreleased()) { + if (unreleasedVersion.equals(currentVersion)) { + continue; + } + unreleased.put(unreleasedVersion, + new UnreleasedVersionInfo(unreleasedVersion, getBranchFor(unreleasedVersion), getGradleProjectNameFor(unreleasedVersion))); + } + this.unreleased = Collections.unmodifiableMap(unreleased); } private void assertNoOlderThanTwoMajors() { @@ -150,6 +163,13 @@ private void assertCurrentVersionMatchesParsed(Version currentVersionProperty) { } } + /** + * Returns info about the unreleased version, or {@code null} if the version is released. + */ + public UnreleasedVersionInfo unreleasedInfo(Version version) { + return unreleased.get(version); + } + public void forPreviousUnreleased(Consumer consumer) { getUnreleased().stream() .filter(version -> version.equals(currentVersion) == false) diff --git a/distribution/bwc/build.gradle b/distribution/bwc/build.gradle index 47f3b78c43f5d..a0acce1ef64eb 100644 --- a/distribution/bwc/build.gradle +++ b/distribution/bwc/build.gradle @@ -120,18 +120,19 @@ bwcVersions.forPreviousUnreleased { VersionCollection.UnreleasedVersionInfo unre } } - List artifactFiles = [] + Map artifactFiles = [:] List projectDirs = [] - for (String project : ['zip', 'deb', 'rpm']) { + List projects = ['zip', 'deb', 'rpm'] + for (String projectName : projects) { String baseDir = "distribution" if (bwcVersion.onOrAfter('6.3.0')) { - baseDir += project == 'zip' ? '/archives' : '/packages' + baseDir += projectName == 'zip' ? '/archives' : '/packages' // add oss variant first - projectDirs.add("${baseDir}/oss-${project}") - artifactFiles.add(file("${checkoutDir}/${baseDir}/oss-${project}/build/distributions/elasticsearch-oss-${bwcVersion}-SNAPSHOT.${project}")) + projectDirs.add("${baseDir}/oss-${projectName}") + artifactFiles.put("oss-" + projectName, file("${checkoutDir}/${baseDir}/oss-${projectName}/build/distributions/elasticsearch-oss-${bwcVersion}-SNAPSHOT.${projectName}")) } - projectDirs.add("${baseDir}/${project}") - artifactFiles.add(file("${checkoutDir}/${baseDir}/${project}/build/distributions/elasticsearch-${bwcVersion}-SNAPSHOT.${project}")) + projectDirs.add("${baseDir}/${projectName}") + artifactFiles.put(projectName, file("${checkoutDir}/${baseDir}/${projectName}/build/distributions/elasticsearch-${bwcVersion}-SNAPSHOT.${projectName}")) } task buildBwcVersion(type: Exec) { @@ -187,7 +188,7 @@ bwcVersions.forPreviousUnreleased { VersionCollection.UnreleasedVersionInfo unre standardOutput = new IndentingOutputStream(System.out, bwcVersion) errorOutput = new IndentingOutputStream(System.err, bwcVersion) doLast { - List missing = artifactFiles.grep { file -> + List missing = artifactFiles.values().grep { file -> false == file.exists() } if (false == missing.empty) { @@ -197,12 +198,16 @@ bwcVersions.forPreviousUnreleased { VersionCollection.UnreleasedVersionInfo unre } } - artifacts { - for (File artifactFile : artifactFiles) { - String artifactName = artifactFile.name.contains('oss') ? 'elasticsearch-oss' : 'elasticsearch' + for (e in artifactFiles) { + String projectName = e.key + File artifactFile = e.value + String artifactFileName = artifactFile.name + String artifactName = artifactFileName.contains('oss') ? 'elasticsearch-oss' : 'elasticsearch' String suffix = artifactFile.toString()[-3..-1] - 'default' file: artifactFile, name: artifactName, type: suffix, builtBy: buildBwcVersion - } + configurations.create(projectName) + artifacts { + it.add(projectName, [file: artifactFile, name: artifactName, type: suffix, builtBy: buildBwcVersion]) + } } // make sure no dependencies were added to assemble; we want it to be a no-op assemble.dependsOn = [] From cafda6d8bb8ac7c6a7a46dd0d4646e0ded5c11ea Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Tue, 22 Jan 2019 17:09:06 -0800 Subject: [PATCH 2/4] add leniency for build-tools --- .../elasticsearch/gradle/test/ClusterFormationTasks.groovy | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy index d2552875ffd60..ad3dee6e8ce1e 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy @@ -195,7 +195,11 @@ class ClusterFormationTasks { } String snapshotProject = distro == 'oss' ? 'oss-zip' : 'zip' Object dependency - VersionCollection.UnreleasedVersionInfo unreleasedInfo = project.bwcVersions.unreleasedInfo(version) + VersionCollection.UnreleasedVersionInfo unreleasedInfo = null + if (project.hasProperty('bwcVersions')) { + // NOTE: leniency is needed for external plugin authors using build-tools. maybe build the version compat info into build-tools? + unreleasedInfo = project.bwcVersions.unreleasedInfo(version) + } if (unreleasedInfo != null) { dependency = project.dependencies.project(path: ":distribution:bwc:${unreleasedInfo.gradleProjectName}", configuration: snapshotProject) } else if (elasticsearchVersion.equals(VersionProperties.elasticsearch)) { From a42e22313a90a42a771fe4607b23314b84f2730c Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Tue, 22 Jan 2019 18:02:48 -0800 Subject: [PATCH 3/4] more leniency... --- .../org/elasticsearch/gradle/test/ClusterFormationTasks.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy index ad3dee6e8ce1e..16502279eb2b4 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy @@ -195,6 +195,7 @@ class ClusterFormationTasks { } String snapshotProject = distro == 'oss' ? 'oss-zip' : 'zip' Object dependency + boolean internalBuild = project.hasProperty('bwcVersions') VersionCollection.UnreleasedVersionInfo unreleasedInfo = null if (project.hasProperty('bwcVersions')) { // NOTE: leniency is needed for external plugin authors using build-tools. maybe build the version compat info into build-tools? @@ -202,7 +203,7 @@ class ClusterFormationTasks { } if (unreleasedInfo != null) { dependency = project.dependencies.project(path: ":distribution:bwc:${unreleasedInfo.gradleProjectName}", configuration: snapshotProject) - } else if (elasticsearchVersion.equals(VersionProperties.elasticsearch)) { + } else if (internalBuild && elasticsearchVersion.equals(VersionProperties.elasticsearch)) { dependency = project.dependencies.project(path: ":distribution:archives:${snapshotProject}") } else { dependency = "${group}:${artifactName}:${elasticsearchVersion}@zip" From f3c0dab2fe8bb8a46ecb67c06f3a04ef443c240e Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Tue, 22 Jan 2019 22:28:33 -0800 Subject: [PATCH 4/4] simplify integ test zip usage --- .../gradle/test/ClusterFormationTasks.groovy | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy index 16502279eb2b4..e38cb854a10b0 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy @@ -172,6 +172,12 @@ class ClusterFormationTasks { /** Adds a dependency on the given distribution */ static void configureDistributionDependency(Project project, String distro, Configuration configuration, String elasticsearchVersion) { + if (distro.equals("integ-test-zip")) { + // short circuit integ test so it doesn't complicate the rest of the distribution setup below + project.dependencies.add(configuration.name, + "org.elasticsearch.distribution.integ-test-zip:elasticsearch:${elasticsearchVersion}@zip") + return + } // TEMP HACK // The oss docs CI build overrides the distro on the command line. This hack handles backcompat until CI is updated. if (distro.equals('oss-zip')) { @@ -181,14 +187,11 @@ class ClusterFormationTasks { distro = 'default' } // END TEMP HACK - if (['integ-test-zip', 'oss', 'default'].contains(distro) == false) { + if (['oss', 'default'].contains(distro) == false) { throw new GradleException("Unknown distribution: ${distro} in project ${project.path}") } Version version = Version.fromString(elasticsearchVersion) String group = "downloads.zip" // dummy group, does not matter except for integ-test-zip, it is ignored by the fake ivy repo - if (distro.equals("integ-test-zip")) { - group = "org.elasticsearch.distribution.integ-test-zip" - } String artifactName = 'elasticsearch' if (distro.equals('oss') && Version.fromString(elasticsearchVersion).onOrAfter('6.3.0')) { artifactName += '-oss'