From e0bea700706453507548c7850ff0f2a957459526 Mon Sep 17 00:00:00 2001 From: Michael Basnight Date: Fri, 9 Feb 2018 14:55:10 -0600 Subject: [PATCH] Generalize BWC logic (#28505) Generalizing BWC building so that there is less code to modify for a release. This ensures we do not need to think about what major or minor version is in the gradle code. It follows the general rules of the elastic release structure. For more information on the rules, see the VersionCollection's javadoc. This also removes the additional bwc snapshots that will never be released, such as 6.0.2, which were being built and tested against every time we ran bwc tests. Additionally, it creates 4 new projects that correspond to the different types of snapshots that may exist for a given version. Its possible to now run those individual tasks to work out bwc logic whereas previously it was impossible and the entire suite of bwc tests had to be run to work out any logic changes in the build tools' bwc project. Please note that if the project does not make sense for the version that is current, that an error will be thrown from that individual project if an attempt is made to run it. This should allow for automating the version bumps as well, since it removes all the hardcoded version logic from the configs. --- build.gradle | 21 +- .../org/elasticsearch/gradle/Version.groovy | 23 +- .../gradle/VersionCollection.groovy | 348 +++++++++++++----- .../gradle/vagrant/VagrantTestPlugin.groovy | 2 +- .../gradle/VersionCollectionTest.groovy | 226 ++++++++++++ distribution/bwc/build.gradle | 21 +- .../maintenance-bugfix-snapshot/build.gradle | 0 .../bwc/next-bugfix-snapshot/build.gradle | 0 .../bwc/next-minor-snapshot/build.gradle | 0 .../bwc/staged-minor-snapshot/build.gradle | 0 qa/full-cluster-restart/build.gradle | 6 +- qa/mixed-cluster/build.gradle | 4 +- qa/query-builder-bwc/build.gradle | 11 +- qa/rolling-upgrade/build.gradle | 4 +- qa/verify-version-constants/build.gradle | 9 +- .../main/java/org/elasticsearch/Version.java | 9 - settings.gradle | 51 +-- test/framework/build.gradle | 4 +- .../org/elasticsearch/test/VersionUtils.java | 99 ++--- .../elasticsearch/test/VersionUtilsTests.java | 42 ++- 20 files changed, 635 insertions(+), 245 deletions(-) create mode 100644 buildSrc/src/test/groovy/org/elasticsearch/gradle/VersionCollectionTest.groovy create mode 100644 distribution/bwc/maintenance-bugfix-snapshot/build.gradle create mode 100644 distribution/bwc/next-bugfix-snapshot/build.gradle create mode 100644 distribution/bwc/next-minor-snapshot/build.gradle create mode 100644 distribution/bwc/staged-minor-snapshot/build.gradle diff --git a/build.gradle b/build.gradle index 487da8926518e..ab5da713d8cac 100644 --- a/build.gradle +++ b/build.gradle @@ -103,7 +103,7 @@ allprojects { isIdea = System.getProperty("idea.active") != null || gradle.startParameter.taskNames.contains('idea') || gradle.startParameter.taskNames.contains('cleanIdea') // for BWC testing - versionCollection = versions + bwcVersions = versions buildMetadata = buildMetadataMap } @@ -122,13 +122,13 @@ task verifyVersions { Set knownVersions = new TreeSet<>(xml.versioning.versions.version.collect { it.text() }.findAll { it ==~ /\d\.\d\.\d/ }.collect { Version.fromString(it) }) // Limit the known versions to those that should be index compatible, and are not future versions - knownVersions = knownVersions.findAll { it.major >= versions.currentVersion.major - 1 && it.before(VersionProperties.elasticsearch) } + knownVersions = knownVersions.findAll { it.major >= bwcVersions.currentVersion.major - 1 && it.before(VersionProperties.elasticsearch) } /* Limit the listed versions to those that have been marked as released. * Versions not marked as released don't get the same testing and we want * to make sure that we flip all unreleased versions to released as soon * as possible after release. */ - Set actualVersions = new TreeSet<>(versions.versionsIndexCompatibleWithCurrent.findAll { false == it.snapshot }) + Set actualVersions = new TreeSet<>(bwcVersions.indexCompatible.findAll { false == it.snapshot }) // Finally, compare! if (knownVersions.equals(actualVersions) == false) { @@ -219,13 +219,14 @@ subprojects { "org.elasticsearch.plugin:rank-eval-client:${version}": ':modules:rank-eval', ] - for (final Version version : versionCollection.versionsIndexCompatibleWithCurrent) { - if (version.branch != null) { - final String snapshotProject = ":distribution:bwc-snapshot-${version.branch}" - project(snapshotProject).ext.bwcVersion = version - ext.projectSubstitutions["org.elasticsearch.distribution.deb:elasticsearch:${version}"] = snapshotProject - ext.projectSubstitutions["org.elasticsearch.distribution.rpm:elasticsearch:${version}"] = snapshotProject - ext.projectSubstitutions["org.elasticsearch.distribution.zip:elasticsearch:${version}"] = snapshotProject + bwcVersions.snapshotProjectNames.each { snapshotName -> + Version snapshot = bwcVersions.getSnapshotForProject(snapshotName) + if (snapshot != null ) { + String snapshotProject = ":distribution:bwc:${snapshotName}" + project(snapshotProject).ext.bwcVersion = snapshot + ext.projectSubstitutions["org.elasticsearch.distribution.deb:elasticsearch:${snapshot}"] = snapshotProject + ext.projectSubstitutions["org.elasticsearch.distribution.rpm:elasticsearch:${snapshot}"] = snapshotProject + ext.projectSubstitutions["org.elasticsearch.distribution.zip:elasticsearch:${snapshot}"] = snapshotProject } } diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/Version.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/Version.groovy index 7dbd943b62d6a..419d3792bb616 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/Version.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/Version.groovy @@ -34,24 +34,29 @@ public class Version { final int revision final int id final boolean snapshot - final String branch /** - * Suffix on the version name. Unlike Version.java the build does not - * consider alphas and betas different versions, it just preserves the - * suffix that the version was declared with in Version.java. + * Suffix on the version name. */ final String suffix public Version(int major, int minor, int revision, - String suffix, boolean snapshot, String branch) { + String suffix, boolean snapshot) { this.major = major this.minor = minor this.revision = revision this.snapshot = snapshot this.suffix = suffix - this.branch = branch - this.id = major * 100000 + minor * 1000 + revision * 10 + - (snapshot ? 1 : 0) + + int suffixOffset = 0 + if (suffix.contains("alpha")) { + suffixOffset += Integer.parseInt(suffix.substring(6)) + } else if (suffix.contains("beta")) { + suffixOffset += 25 + Integer.parseInt(suffix.substring(5)) + } else if (suffix.contains("rc")) { + suffixOffset += 50 + Integer.parseInt(suffix.substring(3)); + } + + this.id = major * 1000000 + minor * 10000 + revision * 100 + suffixOffset } public static Version fromString(String s) { @@ -60,7 +65,7 @@ public class Version { throw new InvalidUserDataException("Invalid version [${s}]") } return new Version(m.group(1) as int, m.group(2) as int, - m.group(3) as int, m.group(4) ?: '', m.group(5) != null, null) + m.group(3) as int, m.group(4) ?: '', m.group(5) != null) } @Override diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/VersionCollection.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/VersionCollection.groovy index a6c151ee59275..6e14777b7c639 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/VersionCollection.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/VersionCollection.groovy @@ -20,164 +20,336 @@ package org.elasticsearch.gradle import org.gradle.api.GradleException +import org.gradle.api.InvalidUserDataException import java.util.regex.Matcher /** * The collection of version constants declared in Version.java, for use in BWC testing. + * + * if major+1 released: released artifacts from $version down to major-1.highestMinor.highestPatch, none of these should be snapshots, period. + * if major+1 unreleased: + * - if released: + * -- caveat 0: snapshot for the major-1.highestMinor.highestPatch + * - if unreleased: + * -- caveat 0: snapshot for the major-1.highestMinor.highestPatch + * -- caveat 1: every same major lower minor branch should also be tested if its released, and if not, its a snapshot. There should only be max 2 of these. + * -- caveat 2: the largest released minor branch before the unreleased minor should also be a snapshot + * -- caveat 3: if the current version is a different major than the previous rules apply to major - 1 of the current version + * + * Please note that the caveat's also correspond with the 4 types of snapshots. + * - Caveat 0 - always maintenanceBugfixSnapshot. + * - Caveat 1 - This is tricky. If caveat 3 applies, the highest matching value is nextMinorSnapshot, if there is another it is the stagedMinorSnapshot. + * If caveat 3 does not apply then the only possible value is the stagedMinorSnapshot. + * - Caveat 2 - always nextBugfixSnapshot + * - Caveat 3 - this only changes the applicability of Caveat 1 + * + * Notes on terminology: + * - The case for major+1 being released is accomplished through the isReleasableBranch value. If this is false, then the branch is no longer + * releasable, meaning not to test against any snapshots. + * - Released is defined as having > 1 suffix-free version in a major.minor series. For instance, only 6.2.0 means unreleased, but a + * 6.2.0 and 6.2.1 mean that 6.2.0 was released already. */ class VersionCollection { private final List versions + Version nextMinorSnapshot + Version stagedMinorSnapshot + Version nextBugfixSnapshot + Version maintenanceBugfixSnapshot + final Version currentVersion + private final TreeSet versionSet = new TreeSet<>() + final List snapshotProjectNames = ['next-minor-snapshot', + 'staged-minor-snapshot', + 'next-bugfix-snapshot', + 'maintenance-bugfix-snapshot'] + + + + // When we roll 8.0 its very likely these will need to be extracted from this class private final boolean buildSnapshot = System.getProperty("build.snapshot", "true") == "true" + private final boolean isReleasableBranch = true /** - * Construct a VersionCollection from the lines of the Version.java file. + * Construct a VersionCollection from the lines of the Version.java file. The basic logic for the following is pretty straight forward. + * @param versionLines The lines of the Version.java file. */ VersionCollection(List versionLines) { List versions = [] + // This class should be converted wholesale to use the treeset for (final String line : versionLines) { final Matcher match = line =~ /\W+public static final Version V_(\d+)_(\d+)_(\d+)(_alpha\d+|_beta\d+|_rc\d+)? .*/ if (match.matches()) { final Version foundVersion = new Version( Integer.parseInt(match.group(1)), Integer.parseInt(match.group(2)), - Integer.parseInt(match.group(3)), (match.group(4) ?: '').replace('_', '-'), false, null) - - if (versions.size() > 0 && foundVersion.onOrBeforeIncludingSuffix(versions[-1])) { - throw new GradleException("Versions.java contains out of order version constants:" + - " ${foundVersion} should come before ${versions[-1]}") - } - - // Only keep the last alpha/beta/rc in the series - if (versions.size() > 0 && versions[-1].id == foundVersion.id) { - versions[-1] = foundVersion - } else { - versions.add(foundVersion) - } + Integer.parseInt(match.group(3)), (match.group(4) ?: '').replace('_', '-'), false) + safeAddToSet(foundVersion) } } - if (versions.empty) { - throw new GradleException("Unexpectedly found no version constants in Versions.java"); + if (versionSet.empty) { + throw new GradleException("Unexpectedly found no version constants in Versions.java") } - /* - * The tip of each minor series (>= 5.6) is unreleased, so they must be built from source (we set branch to non-null), and we set - * the snapshot flag if and only if build.snapshot is true. - */ - Version prevConsideredVersion = null - boolean found6xSnapshot = false - for (final int versionIndex = versions.size() - 1; versionIndex >= 0; versionIndex--) { - final Version currConsideredVersion = versions[versionIndex] - - if (prevConsideredVersion == null - || currConsideredVersion.major != prevConsideredVersion.major - || currConsideredVersion.minor != prevConsideredVersion.minor) { - - // This is a snapshot version. Work out its branch. NB this doesn't name the current branch correctly, but this doesn't - // matter as we don't BWC test against it. - String branch = "${currConsideredVersion.major}.${currConsideredVersion.minor}" - - if (false == found6xSnapshot && currConsideredVersion.major == 6) { - // TODO needs generalising to deal with when 7.x is cut, and when 6.x is deleted, and so on... - branch = "6.x" - found6xSnapshot = true - } + // If the major version has been released, then remove all of the alpha/beta/rc versions that exist in the set + versionSet.removeAll { it.suffix.isEmpty() == false && isMajorReleased(it, versionSet) } - versions[versionIndex] = new Version( - currConsideredVersion.major, currConsideredVersion.minor, - currConsideredVersion.revision, currConsideredVersion.suffix, buildSnapshot, branch) - } + // set currentVersion + Version lastVersion = versionSet.last() + currentVersion = new Version(lastVersion.major, lastVersion.minor, lastVersion.revision, lastVersion.suffix, buildSnapshot) - if (currConsideredVersion.onOrBefore("5.6.0")) { - break - } + // remove all of the potential alpha/beta/rc from the currentVersion + versionSet.removeAll { + it.suffix.isEmpty() == false && + it.major == currentVersion.major && + it.minor == currentVersion.minor && + it.revision == currentVersion.revision } + + // re-add the currentVersion to the set + versionSet.add(currentVersion) - prevConsideredVersion = currConsideredVersion + if (isReleasableBranch) { + if (isReleased(currentVersion)) { + // caveat 0 - if the minor has been released then it only has a maintenance version + // go back 1 version to get the last supported snapshot version of the line, which is a maint bugfix + Version highestMinor = getHighestPreviousMinor(currentVersion.major) + maintenanceBugfixSnapshot = replaceAsSnapshot(highestMinor) + } else { + // caveat 3 - if our currentVersion is a X.0.0, we need to check X-1 minors to see if they are released + if (currentVersion.minor == 0) { + for (Version version: getMinorTips(currentVersion.major - 1)) { + if (isReleased(version) == false) { + // caveat 1 - This should only ever contain 2 non released branches in flight. An example is 6.x is frozen, + // and 6.2 is cut but not yet released there is some simple logic to make sure that in the case of more than 2, + // it will bail. The order is that the minor snapshot is fufilled first, and then the staged minor snapshot + if (nextMinorSnapshot == null) { + // it has not been set yet + nextMinorSnapshot = replaceAsSnapshot(version) + } else if (stagedMinorSnapshot == null) { + stagedMinorSnapshot = replaceAsSnapshot(version) + } else { + throw new GradleException("More than 2 snapshot version existed for the next minor and staged (frozen) minors.") + } + } else { + // caveat 2 - this is the last minor snap for this major, so replace the highest (last) one of these and break + nextBugfixSnapshot = replaceAsSnapshot(version) + // we only care about the largest minor here, so in the case of 6.1 and 6.0, it will only get 6.1 + break + } + } + // caveat 0 - now dip back 2 versions to get the last supported snapshot version of the line + Version highestMinor = getHighestPreviousMinor(currentVersion.major - 1) + maintenanceBugfixSnapshot = replaceAsSnapshot(highestMinor) + } else { + // caveat 3 did not apply. version is not a X.0.0, so we are somewhere on a X.Y line + // only check till minor == 0 of the major + for (Version version: getMinorTips(currentVersion.major)) { + if (isReleased(version) == false) { + // caveat 1 - This should only ever contain 0 or 1 branch in flight. An example is 6.x is frozen, and 6.2 is cut + // but not yet released there is some simple logic to make sure that in the case of more than 1, it will bail + if (stagedMinorSnapshot == null) { + stagedMinorSnapshot = replaceAsSnapshot(version) + } else { + throw new GradleException("More than 1 snapshot version existed for the staged (frozen) minors.") + } + } else { + // caveat 2 - this is the last minor snap for this major, so replace the highest (last) one of these and break + nextBugfixSnapshot = replaceAsSnapshot(version) + // we only care about the largest minor here, so in the case of 6.1 and 6.0, it will only get 6.1 + break + } + } + // caveat 0 - now dip back 1 version to get the last supported snapshot version of the line + Version highestMinor = getHighestPreviousMinor(currentVersion.major) + maintenanceBugfixSnapshot = replaceAsSnapshot(highestMinor) + } + } } - this.versions = Collections.unmodifiableList(versions) + this.versions = Collections.unmodifiableList(versionSet.toList()) } /** * @return The list of versions read from the Version.java file */ List getVersions() { - return Collections.unmodifiableList(versions) + return versions + } + + /** + * Index compat supports 1 previous entire major version. For instance, any 6.x test for this would test all of 5 up to that 6.x version + * + * @return All earlier versions that should be tested for index BWC with the current version. + */ + List getIndexCompatible() { + int actualMajor = (currentVersion.major == 5 ? 2 : currentVersion.major - 1) + return versionSet + .tailSet(Version.fromString("${actualMajor}.0.0")) + .headSet(currentVersion) + .asList() } /** - * @return The latest version in the Version.java file, which must be the current version of the system. + * Ensures the types of snapshot are not null and are also in the index compat list */ - Version getCurrentVersion() { - return versions[-1] + List getSnapshotsIndexCompatible() { + List compatSnapshots = [] + List allCompatVersions = getIndexCompatible() + if (allCompatVersions.contains(nextMinorSnapshot)) { + compatSnapshots.add(nextMinorSnapshot) + } + if (allCompatVersions.contains(stagedMinorSnapshot)) { + compatSnapshots.add(stagedMinorSnapshot) + } + if (allCompatVersions.contains(nextBugfixSnapshot)) { + compatSnapshots.add(nextBugfixSnapshot) + } + if (allCompatVersions.contains(maintenanceBugfixSnapshot)) { + compatSnapshots.add(maintenanceBugfixSnapshot) + } + + return compatSnapshots } /** - * @return The snapshot at the end of the previous minor series in the current major series, or null if this is the first minor series. + * Wire compat supports the last minor of the previous major. For instance, any 6.x test would test 5.6 up to that 6.x version + * + * @return All earlier versions that should be tested for wire BWC with the current version. */ - Version getBWCSnapshotForCurrentMajor() { - return getLastSnapshotWithMajor(currentVersion.major) + List getWireCompatible() { + // Get the last minor of the previous major + Version lowerBound = getHighestPreviousMinor(currentVersion.major) + return versionSet + .tailSet(Version.fromString("${lowerBound.major}.${lowerBound.minor}.0")) + .headSet(currentVersion) + .toList() } /** - * @return The snapshot at the end of the previous major series, which must not be null. + * Ensures the types of snapshot are not null and are also in the wire compat list */ - Version getBWCSnapshotForPreviousMajor() { - Version version = getLastSnapshotWithMajor(currentVersion.major - 1) - assert version != null : "getBWCSnapshotForPreviousMajor(): found no versions in the previous major" - return version + List getSnapshotsWireCompatible() { + List compatSnapshots = [] + List allCompatVersions = getWireCompatible() + if (allCompatVersions.contains(nextMinorSnapshot)) { + compatSnapshots.add(nextMinorSnapshot) + } + if (allCompatVersions.contains(stagedMinorSnapshot)) { + compatSnapshots.add(stagedMinorSnapshot) + } + if (allCompatVersions.contains(nextBugfixSnapshot)) { + compatSnapshots.add(nextBugfixSnapshot) + } + if (allCompatVersions.contains(maintenanceBugfixSnapshot)) { + compatSnapshots.add(maintenanceBugfixSnapshot) + } + // There was no wire compat for the 2.x line + compatSnapshots.removeAll {it.major == 2} + + return compatSnapshots } - private Version getLastSnapshotWithMajor(int targetMajor) { - final String currentVersion = currentVersion.toString() - final int snapshotIndex = versions.findLastIndexOf { - it.major == targetMajor && it.before(currentVersion) && it.snapshot == buildSnapshot + /** + * Grabs the proper snapshot based on the name passed in. These names should correspond with gradle project names under bwc. If you + * are editing this if/else it is only because you added another project under :distribution:bwc. Do not modify this method or its + * reasoning for throwing the exception unless you are sure that it will not harm :distribution:bwc. + */ + Version getSnapshotForProject(String snapshotProjectName) { + if (snapshotProjectName == 'next-minor-snapshot') { + return nextMinorSnapshot + } else if (snapshotProjectName == 'staged-minor-snapshot') { + return stagedMinorSnapshot + } else if (snapshotProjectName == 'maintenance-bugfix-snapshot') { + return maintenanceBugfixSnapshot + } else if (snapshotProjectName == 'next-bugfix-snapshot') { + return nextBugfixSnapshot + } else { + throw new InvalidUserDataException("Unsupported project name ${snapshotProjectName}") } - return snapshotIndex == -1 ? null : versions[snapshotIndex] } - private List versionsOnOrAfterExceptCurrent(Version minVersion) { - final String minVersionString = minVersion.toString() - return Collections.unmodifiableList(versions.findAll { - it.onOrAfter(minVersionString) && it != currentVersion - }) + /** + * Uses basic logic about our releases to determine if this version has been previously released + */ + private boolean isReleased(Version version) { + return version.revision > 0 } /** - * @return All earlier versions that should be tested for index BWC with the current version. + * Validates that the count of non suffixed (alpha/beta/rc) versions in a given major to major+1 is greater than 1. + * This means that there is more than just a major.0.0 or major.0.0-alpha in a branch to signify it has been prevously released. + */ + private boolean isMajorReleased(Version version, TreeSet items) { + return items + .tailSet(Version.fromString("${version.major}.0.0")) + .headSet(Version.fromString("${version.major + 1}.0.0")) + .count { it.suffix.isEmpty() } // count only non suffix'd versions as actual versions that may be released + .intValue() > 1 + } + + /** + * Gets the largest version previous major version based on the nextMajorVersion passed in. + * If you have a list [5.0.2, 5.1.2, 6.0.1, 6.1.1] and pass in 6 for the nextMajorVersion, it will return you 5.1.2 */ - List getVersionsIndexCompatibleWithCurrent() { - final Version firstVersionOfCurrentMajor = versions.find { it.major >= currentVersion.major - 1 } - return versionsOnOrAfterExceptCurrent(firstVersionOfCurrentMajor) + private Version getHighestPreviousMinor(Integer nextMajorVersion) { + return versionSet.headSet(Version.fromString("${nextMajorVersion}.0.0")).last() } - private Version getMinimumWireCompatibilityVersion() { - final int firstIndexOfThisMajor = versions.findIndexOf { it.major == currentVersion.major } - if (firstIndexOfThisMajor == 0) { - return versions[0] + /** + * Helper function for turning a version into a snapshot version, removing and readding it to the tree + */ + private Version replaceAsSnapshot(Version version) { + versionSet.remove(version) + Version snapshotVersion = new Version(version.major, version.minor, version.revision, version.suffix, buildSnapshot) + safeAddToSet(snapshotVersion) + return snapshotVersion + } + + /** + * Safely adds a value to the treeset, or bails if the value already exists. + * @param version + */ + private void safeAddToSet(Version version) { + if (versionSet.add(version) == false) { + throw new GradleException("Versions.java contains duplicate entries for ${version}") } - final Version lastVersionOfEarlierMajor = versions[firstIndexOfThisMajor - 1] - return versions.find { it.major == lastVersionOfEarlierMajor.major && it.minor == lastVersionOfEarlierMajor.minor } } /** - * @return All earlier versions that should be tested for wire BWC with the current version. + * Gets the entire set of major.minor.* given those parameters. */ - List getVersionsWireCompatibleWithCurrent() { - return versionsOnOrAfterExceptCurrent(minimumWireCompatibilityVersion) + private SortedSet getMinorSetForMajor(Integer major, Integer minor) { + return versionSet + .tailSet(Version.fromString("${major}.${minor}.0")) + .headSet(Version.fromString("${major}.${minor + 1}.0")) } /** - * `gradle check` does not run all BWC tests. This defines which tests it does run. - * @return Versions to test for BWC during gradle check. + * Gets the entire set of major.* to the currentVersion */ - List getBasicIntegrationTestVersions() { - // TODO these are the versions checked by `gradle check` for BWC tests. Their choice seems a litle arbitrary. - List result = [BWCSnapshotForPreviousMajor, BWCSnapshotForCurrentMajor] - return Collections.unmodifiableList(result.findAll { it != null }) + private SortedSet getMajorSet(Integer major) { + return versionSet + .tailSet(Version.fromString("${major}.0.0")) + .headSet(currentVersion) + } + + /** + * Gets the tip of each minor set and puts it in a list. + * + * examples: + * [1.0.0, 1.1.0, 1.1.1, 1.2.0, 1.3.1] will return [1.0.0, 1.1.1, 1.2.0, 1.3.1] + * [1.0.0, 1.0.1, 1.0.2, 1.0.3, 1.0.4] will return [1.0.4] + */ + private List getMinorTips(Integer major) { + TreeSet majorSet = getMajorSet(major) + List minorList = new ArrayList<>() + for (int minor = majorSet.last().minor; minor >= 0; minor--) { + TreeSet minorSetInMajor = getMinorSetForMajor(major, minor) + minorList.add(minorSetInMajor.last()) + } + return minorList } } diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy index 82e4ac9b71cd0..669291884f40b 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy @@ -108,7 +108,7 @@ class VagrantTestPlugin implements Plugin { if (upgradeFromVersion == null) { String firstPartOfSeed = project.rootProject.testSeed.tokenize(':').get(0) final long seed = Long.parseUnsignedLong(firstPartOfSeed, 16) - final def indexCompatVersions = project.versionCollection.versionsIndexCompatibleWithCurrent + final def indexCompatVersions = project.bwcVersions.indexCompatible upgradeFromVersion = indexCompatVersions[new Random(seed).nextInt(indexCompatVersions.size())] } diff --git a/buildSrc/src/test/groovy/org/elasticsearch/gradle/VersionCollectionTest.groovy b/buildSrc/src/test/groovy/org/elasticsearch/gradle/VersionCollectionTest.groovy new file mode 100644 index 0000000000000..14f6d1b8523f7 --- /dev/null +++ b/buildSrc/src/test/groovy/org/elasticsearch/gradle/VersionCollectionTest.groovy @@ -0,0 +1,226 @@ +package org.elasticsearch.gradle + +class VersionCollectionTest extends GroovyTestCase { + + String formatVersion(String version) { + return " public static final Version V_${version.replaceAll("\\.", "_")} " + } + def allVersions = [formatVersion('5.0.0'), formatVersion('5.0.0_alpha1'), formatVersion('5.0.0_alpha2'), formatVersion('5.0.0_beta1'), + formatVersion('5.0.0_rc1'),formatVersion('5.0.0_rc2'),formatVersion('5.0.1'), formatVersion('5.0.2'), + formatVersion('5.1.1'), formatVersion('5.1.2'), formatVersion('5.2.0'), formatVersion('5.2.1'), formatVersion('6.0.0'), + formatVersion('6.0.1'), formatVersion('6.1.0'), formatVersion('6.1.1'), formatVersion('6.2.0'), formatVersion('6.3.0'), + formatVersion('7.0.0_alpha1'), formatVersion('7.0.0_alpha2')] + + /** + * This validates the logic of being on a unreleased major branch with a staged major-1.minor sibling. This case happens when a version is + * branched from Major-1.x At the time of this writing 6.2 is unreleased and 6.3 is the 6.x branch. This test simulates the behavior + * from 7.0 perspective, or master at the time of this writing. + */ + void testAgainstMajorUnreleasedWithExistingStagedMinorRelease() { + VersionCollection vc = new VersionCollection(allVersions) + assertNotNull(vc) + assertEquals(vc.nextMinorSnapshot, Version.fromString("6.3.0-SNAPSHOT")) + assertEquals(vc.stagedMinorSnapshot, Version.fromString("6.2.0-SNAPSHOT")) + assertEquals(vc.nextBugfixSnapshot, Version.fromString("6.1.1-SNAPSHOT")) + assertEquals(vc.maintenanceBugfixSnapshot, Version.fromString("5.2.1-SNAPSHOT")) + + vc.indexCompatible.containsAll(vc.versions) + + // This should contain the same list sans the current version + List indexCompatList = [Version.fromString("6.0.0"), Version.fromString("6.0.1"), + Version.fromString("6.1.0"), Version.fromString("6.1.1-SNAPSHOT"), + Version.fromString("6.2.0-SNAPSHOT"), Version.fromString("6.3.0-SNAPSHOT")] + assertTrue(indexCompatList.containsAll(vc.indexCompatible)) + assertTrue(vc.indexCompatible.containsAll(indexCompatList)) + + List wireCompatList = [Version.fromString("6.3.0-SNAPSHOT")] + assertTrue(wireCompatList.containsAll(vc.wireCompatible)) + assertTrue(vc.wireCompatible.containsAll(wireCompatList)) + + assertEquals(vc.snapshotsIndexCompatible.size(), 3) + assertTrue(vc.snapshotsIndexCompatible.contains(Version.fromString("6.3.0-SNAPSHOT"))) + assertTrue(vc.snapshotsIndexCompatible.contains(Version.fromString("6.2.0-SNAPSHOT"))) + assertTrue(vc.snapshotsIndexCompatible.contains(Version.fromString("6.1.1-SNAPSHOT"))) + + assertEquals(vc.snapshotsWireCompatible.size(), 1) + assertEquals(vc.snapshotsWireCompatible.first(), Version.fromString("6.3.0-SNAPSHOT")) + } + + /** + * This validates the logic of being on a unreleased major branch without a staged major-1.minor sibling. This case happens once a staged, + * unreleased minor is released. At the time of this writing 6.2 is unreleased, so adding a 6.2.1 simulates a 6.2 release. This test + * simulates the behavior from 7.0 perspective, or master at the time of this writing. + */ + void testAgainstMajorUnreleasedWithoutStagedMinorRelease() { + List localVersion = allVersions.clone() + localVersion.add(formatVersion('6.2.1')) // release 6.2 + + VersionCollection vc = new VersionCollection(localVersion) + assertNotNull(vc) + assertEquals(vc.nextMinorSnapshot, Version.fromString("6.3.0-SNAPSHOT")) + assertEquals(vc.stagedMinorSnapshot, null) + assertEquals(vc.nextBugfixSnapshot, Version.fromString("6.2.1-SNAPSHOT")) + assertEquals(vc.maintenanceBugfixSnapshot, Version.fromString("5.2.1-SNAPSHOT")) + + vc.indexCompatible.containsAll(vc.versions) + + // This should contain the same list sans the current version + List indexCompatList = [Version.fromString("6.0.0"), Version.fromString("6.0.1"), + Version.fromString("6.1.0"), Version.fromString("6.1.1"), + Version.fromString("6.2.0"), Version.fromString("6.2.1-SNAPSHOT"), + Version.fromString("6.3.0-SNAPSHOT")] + assertTrue(indexCompatList.containsAll(vc.indexCompatible)) + assertTrue(vc.indexCompatible.containsAll(indexCompatList)) + + List wireCompatList = [Version.fromString("6.3.0-SNAPSHOT")] + assertTrue(wireCompatList.containsAll(vc.wireCompatible)) + assertTrue(vc.wireCompatible.containsAll(wireCompatList)) + + assertEquals(vc.snapshotsIndexCompatible.size(), 2) + assertTrue(vc.snapshotsIndexCompatible.contains(Version.fromString("6.3.0-SNAPSHOT"))) + assertTrue(vc.snapshotsIndexCompatible.contains(Version.fromString("6.2.1-SNAPSHOT"))) + + assertEquals(vc.snapshotsWireCompatible.size(), 1) + assertEquals(vc.snapshotsWireCompatible.first(), Version.fromString("6.3.0-SNAPSHOT")) + } + + /** + * This validates the logic of being on a unreleased minor branch with a staged minor sibling. This case happens when a version is + * branched from Major.x At the time of this writing 6.2 is unreleased and 6.3 is the 6.x branch. This test simulates the behavior + * from 6.3 perspective. + */ + void testAgainstMinorReleasedBranch() { + List localVersion = allVersions.clone() + localVersion.removeAll { it.toString().contains('7_0_0')} // remove all the 7.x so that the actual version is 6.3 (6.x) + VersionCollection vc = new VersionCollection(localVersion) + assertNotNull(vc) + assertEquals(vc.nextMinorSnapshot, null) + assertEquals(vc.stagedMinorSnapshot, Version.fromString("6.2.0-SNAPSHOT")) + assertEquals(vc.nextBugfixSnapshot, Version.fromString("6.1.1-SNAPSHOT")) + assertEquals(vc.maintenanceBugfixSnapshot, Version.fromString("5.2.1-SNAPSHOT")) + + // This should contain the same list sans the current version + List indexCompatList = vc.versions.subList(0, vc.versions.size() - 1) + assertTrue(indexCompatList.containsAll(vc.indexCompatible)) + assertTrue(vc.indexCompatible.containsAll(indexCompatList)) + + List wireCompatList = [Version.fromString("5.2.0"), Version.fromString("5.2.1-SNAPSHOT"), Version.fromString("6.0.0"), + Version.fromString("6.0.1"), Version.fromString("6.1.0"), Version.fromString("6.1.1-SNAPSHOT"), + Version.fromString("6.2.0-SNAPSHOT")] + assertTrue(wireCompatList.containsAll(vc.wireCompatible)) + assertTrue(vc.wireCompatible.containsAll(wireCompatList)) + + assertEquals(vc.snapshotsIndexCompatible.size(), 3) + assertTrue(vc.snapshotsIndexCompatible.contains(Version.fromString("6.2.0-SNAPSHOT"))) + assertTrue(vc.snapshotsIndexCompatible.contains(Version.fromString("6.1.1-SNAPSHOT"))) + assertTrue(vc.snapshotsIndexCompatible.contains(Version.fromString("5.2.1-SNAPSHOT"))) + + assertEquals(vc.snapshotsWireCompatible.size(), 3) + assertTrue(vc.snapshotsWireCompatible.contains(Version.fromString("6.2.0-SNAPSHOT"))) + assertTrue(vc.snapshotsWireCompatible.contains(Version.fromString("6.1.1-SNAPSHOT"))) + assertTrue(vc.snapshotsWireCompatible.contains(Version.fromString("5.2.1-SNAPSHOT"))) + } + + /** + * This validates the logic of being on a unreleased minor branch without a staged minor sibling. This case happens once a staged, + * unreleased minor is released. At the time of this writing 6.2 is unreleased, so adding a 6.2.1 simulates a 6.2 release. This test + * simulates the behavior from 6.3 perspective. + */ + void testAgainstMinorReleasedBranchNoStagedMinor() { + List localVersion = allVersions.clone() + // remove all the 7.x and add a 6.2.1 which means 6.2 was released + localVersion.removeAll { it.toString().contains('7_0_0')} + localVersion.add(formatVersion('6.2.1')) + VersionCollection vc = new VersionCollection(localVersion) + assertNotNull(vc) + assertEquals(vc.nextMinorSnapshot, null) + assertEquals(vc.stagedMinorSnapshot, null) + assertEquals(vc.nextBugfixSnapshot, Version.fromString("6.2.1-SNAPSHOT")) + assertEquals(vc.maintenanceBugfixSnapshot, Version.fromString("5.2.1-SNAPSHOT")) + + // This should contain the same list sans the current version + List indexCompatList = vc.versions.subList(0, vc.versions.size() - 1) + assertTrue(indexCompatList.containsAll(vc.indexCompatible)) + assertTrue(vc.indexCompatible.containsAll(indexCompatList)) + + List wireCompatList = [Version.fromString("5.2.0"), Version.fromString("5.2.1-SNAPSHOT"), Version.fromString("6.0.0"), + Version.fromString("6.0.1"), Version.fromString("6.1.0"), Version.fromString("6.1.1"), + Version.fromString("6.2.0"), Version.fromString("6.2.1-SNAPSHOT")] + assertTrue(wireCompatList.containsAll(vc.wireCompatible)) + assertTrue(vc.wireCompatible.containsAll(wireCompatList)) + + assertEquals(vc.snapshotsIndexCompatible.size(), 2) + assertTrue(vc.snapshotsIndexCompatible.contains(Version.fromString("6.2.1-SNAPSHOT"))) + assertTrue(vc.snapshotsIndexCompatible.contains(Version.fromString("5.2.1-SNAPSHOT"))) + + assertEquals(vc.snapshotsWireCompatible.size(), 2) + assertTrue(vc.snapshotsWireCompatible.contains(Version.fromString("6.2.1-SNAPSHOT"))) + assertTrue(vc.snapshotsWireCompatible.contains(Version.fromString("5.2.1-SNAPSHOT"))) + } + + /** + * This validates the logic of being on a released minor branch. At the time of writing, 6.2 is unreleased, so this is equivalent of being + * on 6.1. + */ + void testAgainstOldMinor() { + + List localVersion = allVersions.clone() + // remove the 7 alphas and the ones greater than 6.1 + localVersion.removeAll { it.toString().contains('7_0_0') || it.toString().contains('V_6_2') || it.toString().contains('V_6_3') } + VersionCollection vc = new VersionCollection(localVersion) + assertNotNull(vc) + assertEquals(vc.nextMinorSnapshot, null) + assertEquals(vc.stagedMinorSnapshot, null) + assertEquals(vc.nextBugfixSnapshot, null) + assertEquals(vc.maintenanceBugfixSnapshot, Version.fromString("5.2.1-SNAPSHOT")) + + // This should contain the same list sans the current version + List indexCompatList = vc.versions.subList(0, vc.versions.size() - 1) + assertTrue(indexCompatList.containsAll(vc.indexCompatible)) + assertTrue(vc.indexCompatible.containsAll(indexCompatList)) + + List wireCompatList = [Version.fromString("5.2.0"), Version.fromString("5.2.1-SNAPSHOT"), Version.fromString("6.0.0"), + Version.fromString("6.0.1"), Version.fromString("6.1.0")] + assertTrue(wireCompatList.containsAll(vc.wireCompatible)) + assertTrue(vc.wireCompatible.containsAll(wireCompatList)) + + assertEquals(vc.snapshotsIndexCompatible.size(), 1) + assertTrue(vc.snapshotsIndexCompatible.contains(Version.fromString("5.2.1-SNAPSHOT"))) + + assertEquals(vc.snapshotsWireCompatible.size(), 1) + assertTrue(vc.snapshotsWireCompatible.contains(Version.fromString("5.2.1-SNAPSHOT"))) + } + + /** + * This validates the lower bound of wire compat, which is 5.0. It also validates that the span of 2.x to 5.x if it is decided to port + * this fix all the way to the maint 5.6 release. + */ + void testFloorOfWireCompatVersions() { + List localVersion = [formatVersion('2.0.0'), formatVersion('2.0.1'), formatVersion('2.1.0'), formatVersion('2.1.1'), + formatVersion('5.0.0'), formatVersion('5.0.1'), formatVersion('5.1.0'), formatVersion('5.1.1'), + formatVersion('5.2.0'),formatVersion('5.2.1'),formatVersion('5.3.0'),formatVersion('5.3.1'), + formatVersion('5.3.2')] + VersionCollection vc = new VersionCollection(localVersion) + assertNotNull(vc) + assertEquals(vc.maintenanceBugfixSnapshot, Version.fromString("2.1.1-SNAPSHOT")) + + // This should contain the same list sans the current version + List indexCompatList = vc.versions.subList(0, vc.versions.size() - 1) + assertTrue(indexCompatList.containsAll(vc.indexCompatible)) + assertTrue(vc.indexCompatible.containsAll(indexCompatList)) + + List wireCompatList = [Version.fromString("2.1.0"), Version.fromString("2.1.1-SNAPSHOT"), Version.fromString("5.0.0"), + Version.fromString("5.0.1"), Version.fromString("5.1.0"), + Version.fromString("5.1.1"), Version.fromString("5.2.0"), Version.fromString("5.2.1"), + Version.fromString("5.3.0"), Version.fromString("5.3.1")] + + assertTrue(wireCompatList.containsAll(vc.wireCompatible)) + assertTrue(vc.wireCompatible.containsAll(wireCompatList)) + + assertEquals(vc.snapshotsIndexCompatible.size(), 1) + assertTrue(vc.snapshotsIndexCompatible.contains(Version.fromString("2.1.1-SNAPSHOT"))) + + // ensure none of the 2.x snapshots appear here, as this is the floor of bwc for wire compat + assertEquals(vc.snapshotsWireCompatible.size(), 0) + } +} diff --git a/distribution/bwc/build.gradle b/distribution/bwc/build.gradle index a9a7bd1e0a247..5284d7674a1b3 100644 --- a/distribution/bwc/build.gradle +++ b/distribution/bwc/build.gradle @@ -29,14 +29,21 @@ import java.util.regex.Matcher * tests to test against the next unreleased version, closest to this version, * without relying on snapshots. */ -final Matcher match = project.name =~ /bwc-snapshot-(\d+\.(\d+|x))/ -if (!match.matches()) { - throw new InvalidUserDataException("Unsupport project name ${project.name}") -} -String bwcBranch = match.group(1) +subprojects { + + Version bwcVersion = bwcVersions.getSnapshotForProject(project.name) + if (bwcVersion == null) { + // this project wont do anything + return + } -if (project.hasProperty('bwcVersion')) { - Version bwcVersion = project.ext.bwcVersion + String bwcBranch + if (project.name == 'next-minor-snapshot') { + // this is always a .x series + bwcBranch = "${bwcVersion.major}.x" + } else { + bwcBranch = "${bwcVersion.major}.${bwcVersion.minor}" + } apply plugin: 'distribution' // Not published so no need to assemble diff --git a/distribution/bwc/maintenance-bugfix-snapshot/build.gradle b/distribution/bwc/maintenance-bugfix-snapshot/build.gradle new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/distribution/bwc/next-bugfix-snapshot/build.gradle b/distribution/bwc/next-bugfix-snapshot/build.gradle new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/distribution/bwc/next-minor-snapshot/build.gradle b/distribution/bwc/next-minor-snapshot/build.gradle new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/distribution/bwc/staged-minor-snapshot/build.gradle b/distribution/bwc/staged-minor-snapshot/build.gradle new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/qa/full-cluster-restart/build.gradle b/qa/full-cluster-restart/build.gradle index 123b217a38af3..0449cc3ee71ae 100644 --- a/qa/full-cluster-restart/build.gradle +++ b/qa/full-cluster-restart/build.gradle @@ -19,6 +19,7 @@ import org.elasticsearch.gradle.Version +import org.elasticsearch.gradle.VersionCollection import org.elasticsearch.gradle.test.RestIntegTestTask apply plugin: 'elasticsearch.standalone-test' @@ -30,7 +31,7 @@ task bwcTest { group = 'verification' } -for (Version version : versionCollection.versionsIndexCompatibleWithCurrent) { +for (Version version : bwcVersions.indexCompatible) { String baseName = "v${version}" Task oldClusterTest = tasks.create(name: "${baseName}#oldClusterTest", type: RestIntegTestTask) { @@ -105,10 +106,11 @@ test.enabled = false // no unit tests for rolling upgrades, only the rest integr // basic integ tests includes testing bwc against the most recent version task integTest { if (project.bwc_tests_enabled) { - for (final def version : versionCollection.basicIntegrationTestVersions) { + for (final def version : bwcVersions.snapshotsIndexCompatible) { dependsOn "v${version}#bwcTest" } } } check.dependsOn(integTest) + diff --git a/qa/mixed-cluster/build.gradle b/qa/mixed-cluster/build.gradle index 712fde94b7d31..ca802cd42754f 100644 --- a/qa/mixed-cluster/build.gradle +++ b/qa/mixed-cluster/build.gradle @@ -29,7 +29,7 @@ task bwcTest { group = 'verification' } -for (Version version : versionCollection.versionsWireCompatibleWithCurrent) { +for (Version version : bwcVersions.wireCompatible) { String baseName = "v${version}" Task mixedClusterTest = tasks.create(name: "${baseName}#mixedClusterTest", type: RestIntegTestTask) { @@ -66,7 +66,7 @@ test.enabled = false // no unit tests for rolling upgrades, only the rest integr // basic integ tests includes testing bwc against the most recent version task integTest { if (project.bwc_tests_enabled) { - for (final def version : versionCollection.basicIntegrationTestVersions) { + for (final def version : bwcVersions.snapshotsWireCompatible) { dependsOn "v${version}#bwcTest" } } diff --git a/qa/query-builder-bwc/build.gradle b/qa/query-builder-bwc/build.gradle index 16e9f6298feac..b71fe5159bd87 100644 --- a/qa/query-builder-bwc/build.gradle +++ b/qa/query-builder-bwc/build.gradle @@ -30,7 +30,7 @@ task bwcTest { group = 'verification' } -for (Version version : versionCollection.versionsIndexCompatibleWithCurrent) { +for (Version version : bwcVersions.indexCompatible) { String baseName = "v${version}" Task oldQueryBuilderTest = tasks.create(name: "${baseName}#oldQueryBuilderTest", type: RestIntegTestTask) { @@ -82,11 +82,10 @@ test.enabled = false // no unit tests for rolling upgrades, only the rest integr // basic integ tests includes testing bwc against the most recent version task integTest { - if (project.bwc_tests_enabled) { - for (final def version : versionCollection.basicIntegrationTestVersions) { - dependsOn "v${version}#bwcTest" - } - } + if (project.bwc_tests_enabled) { + final def version = bwcVersions.snapshotsIndexCompatible.first() + dependsOn "v${version}#bwcTest" + } } check.dependsOn(integTest) diff --git a/qa/rolling-upgrade/build.gradle b/qa/rolling-upgrade/build.gradle index 31d056037d865..30e0a311aa673 100644 --- a/qa/rolling-upgrade/build.gradle +++ b/qa/rolling-upgrade/build.gradle @@ -29,7 +29,7 @@ task bwcTest { group = 'verification' } -for (Version version : versionCollection.versionsWireCompatibleWithCurrent) { +for (Version version : bwcVersions.wireCompatible) { String baseName = "v${version}" Task oldClusterTest = tasks.create(name: "${baseName}#oldClusterTest", type: RestIntegTestTask) { @@ -110,7 +110,7 @@ test.enabled = false // no unit tests for rolling upgrades, only the rest integr // basic integ tests includes testing bwc against the most recent version task integTest { if (project.bwc_tests_enabled) { - for (final def version : versionCollection.basicIntegrationTestVersions) { + for (final def version : bwcVersions.snapshotsWireCompatible) { dependsOn "v${version}#bwcTest" } } diff --git a/qa/verify-version-constants/build.gradle b/qa/verify-version-constants/build.gradle index 1d31db6898b7b..2d2135c3a41d9 100644 --- a/qa/verify-version-constants/build.gradle +++ b/qa/verify-version-constants/build.gradle @@ -31,7 +31,7 @@ task bwcTest { group = 'verification' } -for (Version version : versionCollection.versionsIndexCompatibleWithCurrent) { +for (Version version : bwcVersions.indexCompatible) { String baseName = "v${version}" Task oldClusterTest = tasks.create(name: "${baseName}#oldClusterTest", type: RestIntegTestTask) { mustRunAfter(precommit) @@ -57,9 +57,10 @@ for (Version version : versionCollection.versionsIndexCompatibleWithCurrent) { test.enabled = false task integTest { - for (final def version : versionCollection.basicIntegrationTestVersions) { - dependsOn "v${version}#bwcTest" - } + if (project.bwc_tests_enabled) { + final def version = bwcVersions.snapshotsIndexCompatible.first() + dependsOn "v${version}#bwcTest" + } } task verifyDocsLuceneVersion { diff --git a/server/src/main/java/org/elasticsearch/Version.java b/server/src/main/java/org/elasticsearch/Version.java index 84767f2d34d46..e0160f04489d3 100644 --- a/server/src/main/java/org/elasticsearch/Version.java +++ b/server/src/main/java/org/elasticsearch/Version.java @@ -137,9 +137,6 @@ public class Version implements Comparable { public static final int V_6_0_1_ID = 6000199; public static final Version V_6_0_1 = new Version(V_6_0_1_ID, org.apache.lucene.util.Version.LUCENE_7_0_1); - public static final int V_6_0_2_ID = 6000299; - public static final Version V_6_0_2 = - new Version(V_6_0_2_ID, org.apache.lucene.util.Version.LUCENE_7_0_1); public static final int V_6_1_0_ID = 6010099; public static final Version V_6_1_0 = new Version(V_6_1_0_ID, org.apache.lucene.util.Version.LUCENE_7_1_0); public static final int V_6_1_1_ID = 6010199; @@ -148,8 +145,6 @@ public class Version implements Comparable { public static final Version V_6_1_2 = new Version(V_6_1_2_ID, org.apache.lucene.util.Version.LUCENE_7_1_0); public static final int V_6_1_3_ID = 6010399; public static final Version V_6_1_3 = new Version(V_6_1_3_ID, org.apache.lucene.util.Version.LUCENE_7_1_0); - public static final int V_6_1_4_ID = 6010499; - public static final Version V_6_1_4 = new Version(V_6_1_4_ID, org.apache.lucene.util.Version.LUCENE_7_1_0); public static final int V_6_2_0_ID = 6020099; public static final Version V_6_2_0 = new Version(V_6_2_0_ID, org.apache.lucene.util.Version.LUCENE_7_2_1); public static final int V_6_2_1_ID = 6020199; @@ -184,8 +179,6 @@ public static Version fromId(int id) { return V_6_2_1; case V_6_2_0_ID: return V_6_2_0; - case V_6_1_4_ID: - return V_6_1_4; case V_6_1_3_ID: return V_6_1_3; case V_6_1_2_ID: @@ -194,8 +187,6 @@ public static Version fromId(int id) { return V_6_1_1; case V_6_1_0_ID: return V_6_1_0; - case V_6_0_2_ID: - return V_6_0_2; case V_6_0_1_ID: return V_6_0_1; case V_6_0_0_ID: diff --git a/settings.gradle b/settings.gradle index e86f0f5c64498..8a63d37c1057b 100644 --- a/settings.gradle +++ b/settings.gradle @@ -1,3 +1,5 @@ +import org.elasticsearch.gradle.Version + String dirName = rootProject.projectDir.name rootProject.name = dirName @@ -18,6 +20,10 @@ List projects = [ 'distribution:tar', 'distribution:deb', 'distribution:rpm', + 'distribution:bwc:next-minor-snapshot', + 'distribution:bwc:staged-minor-snapshot', + 'distribution:bwc:next-bugfix-snapshot', + 'distribution:bwc:maintenance-bugfix-snapshot', 'distribution:tools:launchers', 'distribution:tools:plugin-cli', 'server', @@ -36,7 +42,7 @@ List projects = [ * of the dir hierarchy to have a build.gradle. Otherwise we would have to iterate * all files/directories in the source tree to find all projects. */ -void addSubProjects(String path, File dir, List projects, List branches) { +void addSubProjects(String path, File dir) { if (dir.isDirectory() == false) return; if (dir.name == 'buildSrc') return; if (new File(dir, 'build.gradle').exists() == false) return; @@ -44,44 +50,28 @@ void addSubProjects(String path, File dir, List projects, List b final String projectName = "${path}:${dir.name}" include projectName - - if (dir.name == 'bwc-snapshot-dummy-projects') { - for (final String branch : branches) { - final String snapshotProjectName = "${projectName}:bwc-snapshot-${branch}" - projects.add(snapshotProjectName) - include snapshotProjectName - project("${snapshotProjectName}").projectDir = dir - } - // TODO do we want to assert that there's nothing else in the bwc directory? - } else { - if (path.isEmpty() || path.startsWith(':example-plugins')) { + if (path.isEmpty() || path.startsWith(':example-plugins')) { project(projectName).projectDir = dir } for (File subdir : dir.listFiles()) { - addSubProjects(projectName, subdir, projects, branches) + addSubProjects(projectName, subdir) } } -} + // include example plugins first, so adding plugin dirs below won't muck with :example-plugins File examplePluginsDir = new File(rootProject.projectDir, 'plugins/examples') for (File example : examplePluginsDir.listFiles()) { if (example.isDirectory() == false) continue; if (example.name.startsWith('build') || example.name.startsWith('.')) continue; - addSubProjects(':example-plugins', example, projects, []) + addSubProjects(':example-plugins', example) } project(':example-plugins').projectDir = new File(rootProject.projectDir, 'plugins/examples') -addSubProjects('', new File(rootProject.projectDir, 'libs'), projects, []) -addSubProjects('', new File(rootProject.projectDir, 'modules'), projects, []) -addSubProjects('', new File(rootProject.projectDir, 'plugins'), projects, []) -addSubProjects('', new File(rootProject.projectDir, 'qa'), projects, []) - -/* Create projects for building BWC snapshot distributions from the heads of other branches */ -final List branches = ['5.6', '6.0', '6.1', '6.2', '6.x'] -for (final String branch : branches) { - projects.add("distribution:bwc-snapshot-${branch}".toString()) -} +addSubProjects('', new File(rootProject.projectDir, 'libs')) +addSubProjects('', new File(rootProject.projectDir, 'modules')) +addSubProjects('', new File(rootProject.projectDir, 'plugins')) +addSubProjects('', new File(rootProject.projectDir, 'qa')) boolean isEclipse = System.getProperty("eclipse.launcher") != null || gradle.startParameter.taskNames.contains('eclipse') || gradle.startParameter.taskNames.contains('cleanEclipse') if (isEclipse) { @@ -97,12 +87,6 @@ include projects.toArray(new String[0]) project(':build-tools').projectDir = new File(rootProject.projectDir, 'buildSrc') -/* The BWC snapshot projects share the same build directory and build file, - * but apply to different backwards compatibility branches. */ -for (final String branch : branches) { - project(":distribution:bwc-snapshot-${branch}").projectDir = new File(rootProject.projectDir, 'distribution/bwc') -} - if (isEclipse) { project(":server").projectDir = new File(rootProject.projectDir, 'server/src/main') project(":server").buildFileName = 'eclipse-build.gradle' @@ -126,7 +110,6 @@ if (isEclipse) { File extraProjects = new File(rootProject.projectDir.parentFile, "${dirName}-extra") if (extraProjects.exists()) { for (File extraProjectDir : extraProjects.listFiles()) { - addSubProjects('', extraProjectDir, projects, branches) + addSubProjects('', extraProjectDir) } -} - +} \ No newline at end of file diff --git a/test/framework/build.gradle b/test/framework/build.gradle index 4e6ef482210b9..193fcb30988c6 100644 --- a/test/framework/build.gradle +++ b/test/framework/build.gradle @@ -71,6 +71,6 @@ task namingConventionsMain(type: org.elasticsearch.gradle.precommit.NamingConven precommit.dependsOn namingConventionsMain test.configure { - systemProperty 'tests.gradle_index_compat_versions', versionCollection.versionsIndexCompatibleWithCurrent.join(',') - systemProperty 'tests.gradle_wire_compat_versions', versionCollection.versionsWireCompatibleWithCurrent.join(',') + systemProperty 'tests.gradle_index_compat_versions', bwcVersions.indexCompatible.join(',') + systemProperty 'tests.gradle_wire_compat_versions', bwcVersions.wireCompatible.join(',') } diff --git a/test/framework/src/main/java/org/elasticsearch/test/VersionUtils.java b/test/framework/src/main/java/org/elasticsearch/test/VersionUtils.java index d452b33f5778f..0e57031459a4a 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/VersionUtils.java +++ b/test/framework/src/main/java/org/elasticsearch/test/VersionUtils.java @@ -24,7 +24,10 @@ import org.elasticsearch.common.Nullable; import org.elasticsearch.common.collect.Tuple; +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Optional; @@ -55,61 +58,59 @@ static Tuple, List> resolveReleasedVersions(Version curre Version last = versions.remove(versions.size() - 1); assert last.equals(current) : "The highest version must be the current one " - + "but was [" + last + "] and current was [" + current + "]"; - - /* In the 5.x series prior to 5.6, unreleased version constants had an - * `_UNRELEASED` suffix, and when making the first release on a minor release - * branch the last, unreleased, version constant from the previous minor branch - * was dropped. After 5.6, there is no `_UNRELEASED` suffix on version constants' - * names and, additionally, they are not dropped when a new minor release branch - * starts. - * - * This means that in 6.x and later series the last release _in each - * minor branch_ is unreleased, whereas in 5.x it's more complicated: There were - * (sometimes, and sometimes multiple) minor branches containing no releases, each - * of which contains a single version constant of the form 5.n.0, and these - * branches always followed a branch that _did_ contain a version of the - * form 5.m.p (p>0). All versions strictly before the last 5.m version are released, - * and all other 5.* versions are unreleased. - */ - - if (current.major == 5 && current.revision != 0) { - /* The current (i.e. latest) version is 5.a.b, b nonzero, which - * means that all other versions are released. */ + + "but was [" + versions.get(versions.size() - 1) + "] and current was [" + current + "]"; + + if (current.revision != 0) { + /* If we are in a stable branch there should be no unreleased version constants + * because we don't expect to release any new versions in older branches. If there + * are extra constants then gradle will yell about it. */ return new Tuple<>(unmodifiableList(versions), singletonList(current)); } - final List unreleased = new ArrayList<>(); - unreleased.add(current); - Version prevConsideredVersion = current; - - for (int i = versions.size() - 1; i >= 0; i--) { - Version currConsideredVersion = versions.get(i); - if (currConsideredVersion.major == 5) { - unreleased.add(currConsideredVersion); - versions.remove(i); - if (currConsideredVersion.revision != 0) { - /* Currently considering the latest version in the 5.x series, - * which is (a) unreleased and (b) the only such. So we're done. */ - break; - } - /* ... else we're on a version of the form 5.n.0, and have not yet - * considered a version of the form 5.n.m (m>0), so this entire branch - * is unreleased, so carry on looking for a branch containing releases. - */ - } else if (currConsideredVersion.major != prevConsideredVersion.major - || currConsideredVersion.minor != prevConsideredVersion.minor) { - /* Have moved to the end of a new minor branch, so this is - * an unreleased version. */ - unreleased.add(currConsideredVersion); - versions.remove(i); + /* If we are on a patch release then we know that at least the version before the + * current one is unreleased. If it is released then gradle would be complaining. */ + int unreleasedIndex = versions.size() - 1; + while (true) { + if (unreleasedIndex < 0) { + throw new IllegalArgumentException("Couldn't find first non-alpha release"); } - prevConsideredVersion = currConsideredVersion; - + /* We don't support backwards compatibility for alphas, betas, and rcs. But + * they were released so we add them to the released list. Usually this doesn't + * matter to consumers, but consumers that do care should filter non-release + * versions. */ + if (versions.get(unreleasedIndex).isRelease()) { + break; + } + unreleasedIndex--; } - Collections.reverse(unreleased); - return new Tuple<>(unmodifiableList(versions), unmodifiableList(unreleased)); + Version unreleased = versions.remove(unreleasedIndex); + if (unreleased.revision == 0) { + /* + * If the last unreleased version is itself a patch release then Gradle enforces that there is yet another unreleased version + * before that. However, we have to skip alpha/betas/RCs too (e.g., consider when the version constants are ..., 5.6.3, 5.6.4, + * 6.0.0-alpha1, ..., 6.0.0-rc1, 6.0.0-rc2, 6.0.0, 6.1.0 on the 6.x branch. In this case, we will have pruned 6.0.0 and 6.1.0 as + * unreleased versions, but we also need to prune 5.6.4. At this point though, unreleasedIndex will be pointing to 6.0.0-rc2, so + * we have to skip backwards until we find a non-alpha/beta/RC again. Then we can prune that version as an unreleased version + * too. + */ + do { + unreleasedIndex--; + } while (versions.get(unreleasedIndex).isRelease() == false); + Version earlierUnreleased = versions.remove(unreleasedIndex); + + // This earlierUnreleased is either the snapshot on the minor branch lower, or its possible its a staged release. If it is a + // staged release, remove it and return it in unreleased as well. + if (earlierUnreleased.revision == 0) { + unreleasedIndex--; + Version actualUnreleasedPreviousMinor = versions.remove(unreleasedIndex); + return new Tuple<>(unmodifiableList(versions), unmodifiableList(Arrays.asList(actualUnreleasedPreviousMinor, + earlierUnreleased, unreleased, current))); + } + + return new Tuple<>(unmodifiableList(versions), unmodifiableList(Arrays.asList(earlierUnreleased, unreleased, current))); + } + return new Tuple<>(unmodifiableList(versions), unmodifiableList(Arrays.asList(unreleased, current))); } private static final List RELEASED_VERSIONS; diff --git a/test/framework/src/test/java/org/elasticsearch/test/VersionUtilsTests.java b/test/framework/src/test/java/org/elasticsearch/test/VersionUtilsTests.java index 9015ad17e357d..e1c3ac82f7865 100644 --- a/test/framework/src/test/java/org/elasticsearch/test/VersionUtilsTests.java +++ b/test/framework/src/test/java/org/elasticsearch/test/VersionUtilsTests.java @@ -269,25 +269,26 @@ public void testResolveReleasedVersionsAtNewMajorRelease() { final List expectedUnreleased; if (Booleans.parseBoolean(System.getProperty("build.snapshot", "true"))) { expectedReleased = Arrays.asList( - TestNewMajorRelease.V_5_6_0, - TestNewMajorRelease.V_5_6_1, - TestNewMajorRelease.V_6_0_0_alpha1, - TestNewMajorRelease.V_6_0_0_alpha2, - TestNewMajorRelease.V_6_0_0_beta1, - TestNewMajorRelease.V_6_0_0_beta2, - TestNewMajorRelease.V_6_0_0); - expectedUnreleased = Arrays.asList(TestNewMajorRelease.V_5_6_2, TestNewMajorRelease.V_6_0_1); + TestNewMajorRelease.V_5_6_0, + TestNewMajorRelease.V_5_6_1, + TestNewMajorRelease.V_5_6_2, + TestNewMajorRelease.V_6_0_0_alpha1, + TestNewMajorRelease.V_6_0_0_alpha2, + TestNewMajorRelease.V_6_0_0_beta1, + TestNewMajorRelease.V_6_0_0_beta2, + TestNewMajorRelease.V_6_0_0); + expectedUnreleased = Arrays.asList(TestNewMajorRelease.V_6_0_1); } else { expectedReleased = Arrays.asList( - TestNewMajorRelease.V_5_6_0, - TestNewMajorRelease.V_5_6_1, - TestNewMajorRelease.V_5_6_2, - TestNewMajorRelease.V_6_0_0_alpha1, - TestNewMajorRelease.V_6_0_0_alpha2, - TestNewMajorRelease.V_6_0_0_beta1, - TestNewMajorRelease.V_6_0_0_beta2, - TestNewMajorRelease.V_6_0_0, - TestNewMajorRelease.V_6_0_1); + TestNewMajorRelease.V_5_6_0, + TestNewMajorRelease.V_5_6_1, + TestNewMajorRelease.V_5_6_2, + TestNewMajorRelease.V_6_0_0_alpha1, + TestNewMajorRelease.V_6_0_0_alpha2, + TestNewMajorRelease.V_6_0_0_beta1, + TestNewMajorRelease.V_6_0_0_beta2, + TestNewMajorRelease.V_6_0_0, + TestNewMajorRelease.V_6_0_1); expectedUnreleased = Collections.emptyList(); } @@ -322,12 +323,13 @@ public void testResolveReleasedVersionsAtVersionBumpIn6x() { expectedReleased = Arrays.asList( TestVersionBumpIn6x.V_5_6_0, TestVersionBumpIn6x.V_5_6_1, + TestVersionBumpIn6x.V_5_6_2, TestVersionBumpIn6x.V_6_0_0_alpha1, TestVersionBumpIn6x.V_6_0_0_alpha2, TestVersionBumpIn6x.V_6_0_0_beta1, TestVersionBumpIn6x.V_6_0_0_beta2, TestVersionBumpIn6x.V_6_0_0); - expectedUnreleased = Arrays.asList(TestVersionBumpIn6x.V_5_6_2, TestVersionBumpIn6x.V_6_0_1, TestVersionBumpIn6x.V_6_1_0); + expectedUnreleased = Arrays.asList(TestVersionBumpIn6x.V_6_0_1, TestVersionBumpIn6x.V_6_1_0); } else { expectedReleased = Arrays.asList( TestVersionBumpIn6x.V_5_6_0, @@ -376,16 +378,16 @@ public void testResolveReleasedVersionsAtNewMinorBranchIn6x() { expectedReleased = Arrays.asList( TestNewMinorBranchIn6x.V_5_6_0, TestNewMinorBranchIn6x.V_5_6_1, + TestNewMinorBranchIn6x.V_5_6_2, TestNewMinorBranchIn6x.V_6_0_0_alpha1, TestNewMinorBranchIn6x.V_6_0_0_alpha2, TestNewMinorBranchIn6x.V_6_0_0_beta1, TestNewMinorBranchIn6x.V_6_0_0_beta2, TestNewMinorBranchIn6x.V_6_0_0, + TestNewMinorBranchIn6x.V_6_0_1, TestNewMinorBranchIn6x.V_6_1_0, TestNewMinorBranchIn6x.V_6_1_1); expectedUnreleased = Arrays.asList( - TestNewMinorBranchIn6x.V_5_6_2, - TestNewMinorBranchIn6x.V_6_0_1, TestNewMinorBranchIn6x.V_6_1_2, TestNewMinorBranchIn6x.V_6_2_0); } else {