Merge branch 'master' into retention-lease-ccr

* master: (27 commits)
  Mute AnalysisModuleTests#testStandardFilterBWC (elastic#38636)
  add geotile_grid ref to asciidoc (elastic#38632)
  Enable Dockerfile from artifacts.elastic.co (elastic#38552)
  Mute FollowerFailOverIT testFailOverOnFollower (elastic#38634)
  Account for a possible rolled over file while reading the audit log file (elastic#34909)
  Mute failure in InternalEngineTests (elastic#38622)
  Fix Issue with Concurrent Snapshot Init + Delete (elastic#38518)
  Refactor ZonedDateTime.now in millis resolution (elastic#38577)
  Mute failing WatchStatusIntegrationTests (elastic#38621)
  Mute failing  ApiKeyIntegTests (elastic#38614)
  [DOCS] Add warning about bypassing ML PUT APIs (elastic#38509)
  Add 7.1 and 8.0 version constants to master (elastic#38514)
  ML: update set_upgrade_mode, add logging (elastic#38372)
  bad formatted JSON object (elastic#38515) (elastic#38525)
  Fix HistoryIntegrationTests timestamp comparsion (elastic#38505)
  SQL: Fix issue with IN not resolving to underlying keyword field (elastic#38440)
  Fix the clock resolution to millis in ScheduledEventTests (elastic#38506)
  Enable BWC after backport recovering leases (elastic#38485)
  Collapse retention lease integration tests (elastic#38483)
  TransportVerifyShardBeforeCloseAction should force a flush (elastic#38401)
  ...

build.gradle

@@ -159,8 +159,8 @@ task verifyVersions {
  * the enabled state of every bwc task. It should be set back to true
  * after the backport of the backcompat code is complete.
  */
-final boolean bwc_tests_enabled = false
-final String bwc_tests_disabled_issue = "https://github.com/elastic/elasticsearch/pull/38435" /* place a PR link here when committing bwc changes */
+final boolean bwc_tests_enabled = true
+final String bwc_tests_disabled_issue = "" /* place a PR link here when committing bwc changes */
 if (bwc_tests_enabled == false) {
   if (bwc_tests_disabled_issue.isEmpty()) {
     throw new GradleException("bwc_tests_disabled_issue must be set when bwc_tests_enabled == false")

buildSrc/src/main/java/org/elasticsearch/gradle/VersionCollection.java

@@ -171,30 +171,38 @@ public UnreleasedVersionInfo unreleasedInfo(Version version) {
     }
 
     public void forPreviousUnreleased(Consumer<UnreleasedVersionInfo> consumer) {
-        getUnreleased().stream()
+        List<UnreleasedVersionInfo> collect = getUnreleased().stream()
             .filter(version -> version.equals(currentVersion) == false)
-            .forEach(version -> consumer.accept(
-                new UnreleasedVersionInfo(
+            .map(version -> new UnreleasedVersionInfo(
                     version,
                     getBranchFor(version),
                     getGradleProjectNameFor(version)
                 )
-            ));
+            )
+            .collect(Collectors.toList());
+
+        collect.forEach(uvi -> consumer.accept(uvi));
     }
 
     private String getGradleProjectNameFor(Version version) {
         if (version.equals(currentVersion)) {
             throw new IllegalArgumentException("The Gradle project to build " + version + " is the current build.");
         }
+
         Map<Integer, List<Version>> releasedMajorGroupedByMinor = getReleasedMajorGroupedByMinor();
 
         if (version.getRevision() == 0) {
-            if (releasedMajorGroupedByMinor
-                .get(releasedMajorGroupedByMinor.keySet().stream().max(Integer::compareTo).orElse(0))
-                .contains(version)) {
-                return "minor";
+            List<Version> unreleasedStagedOrMinor = getUnreleased().stream()
+                .filter(v -> v.getRevision() == 0)
+                .collect(Collectors.toList());
+            if (unreleasedStagedOrMinor.size() > 2) {
+                if (unreleasedStagedOrMinor.get(unreleasedStagedOrMinor.size() - 2).equals(version)) {
+                    return "minor";
+                } else{
+                    return "staged";
+                }
             } else {
-                return "staged";
+                return "minor";
             }
         } else {
             if (releasedMajorGroupedByMinor
@@ -239,8 +247,10 @@ public List<Version> getUnreleased() {
             unreleased.add(getLatestVersionByKey(groupByMinor, greatestMinor - 1));
             if (groupByMinor.getOrDefault(greatestMinor - 1, emptyList()).size() == 1) {
                 // we found that the previous minor is staged but not yet released
-                // in this case, the minor before that has a bugfix
-                unreleased.add(getLatestVersionByKey(groupByMinor, greatestMinor - 2));
+                // in this case, the minor before that has a bugfix, should there be such a minor
+                if (greatestMinor >= 2) {
+                    unreleased.add(getLatestVersionByKey(groupByMinor, greatestMinor - 2));
+                }
             }
         }
 

buildSrc/src/test/java/org/elasticsearch/gradle/VersionCollectionTests.java

@@ -81,6 +81,9 @@ public class VersionCollectionTests extends GradleUnitTestCase {
             "6_0_0", "6_0_1", "6_1_0", "6_1_1", "6_1_2", "6_1_3", "6_1_4", "6_2_0", "6_2_1", "6_2_2", "6_2_3",
             "6_2_4", "6_3_0", "6_3_1", "6_3_2", "6_4_0", "6_4_1", "6_4_2"
         ));
+        sampleVersions.put("7.1.0", asList(
+            "7_1_0", "7_0_0", "6_7_0", "6_6_1", "6_6_0"
+        ));
     }
 
     @Test(expected = IllegalArgumentException.class)
@@ -145,6 +148,11 @@ public void testWireCompatible() {
             singletonList("7.3.0"),
             getVersionCollection("8.0.0").getWireCompatible()
         );
+        assertVersionsEquals(
+            asList("6.7.0", "7.0.0"),
+            getVersionCollection("7.1.0").getWireCompatible()
+        );
+
     }
 
     public void testWireCompatibleUnreleased() {
@@ -171,6 +179,10 @@ public void testWireCompatibleUnreleased() {
             singletonList("7.3.0"),
             getVersionCollection("8.0.0").getUnreleasedWireCompatible()
         );
+        assertVersionsEquals(
+            asList("6.7.0", "7.0.0"),
+            getVersionCollection("7.1.0").getWireCompatible()
+        );
     }
 
     public void testIndexCompatible() {
@@ -286,7 +298,7 @@ public void testGetBranch() {
             getVersionCollection("6.4.2")
         );
         assertUnreleasedBranchNames(
-            asList("5.6", "6.4", "6.5"),
+            asList("5.6", "6.4", "6.x"),
             getVersionCollection("6.6.0")
         );
         assertUnreleasedBranchNames(
@@ -309,13 +321,17 @@ public void testGetGradleProjectName() {
             getVersionCollection("6.4.2")
         );
         assertUnreleasedGradleProjectNames(
-            asList("maintenance", "bugfix", "staged"),
+            asList("maintenance", "bugfix", "minor"),
             getVersionCollection("6.6.0")
         );
         assertUnreleasedGradleProjectNames(
             asList("bugfix", "staged", "minor"),
             getVersionCollection("8.0.0")
         );
+        assertUnreleasedGradleProjectNames(
+            asList("staged", "minor"),
+            getVersionCollection("7.1.0")
+        );
     }
 
     public void testCompareToAuthoritative() {

buildSrc/version.properties

@@ -1,4 +1,4 @@
-elasticsearch     = 7.0.0
+elasticsearch     = 8.0.0
 lucene            = 8.0.0-snapshot-83f9835
 
 # optional dependencies

client/rest-high-level/src/main/java/org/elasticsearch/client/indices/IndexTemplateMetaData.java

@@ -18,27 +18,66 @@
  */
 package org.elasticsearch.client.indices;
 
-import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.cluster.metadata.AliasMetaData;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MappingMetaData;
 import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.mapper.MapperService;
 
 import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
+import java.util.AbstractMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
-import java.util.Set;
+import java.util.stream.Collectors;
+
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
 
 public class IndexTemplateMetaData  {
 
+    @SuppressWarnings("unchecked")
+    private static final ConstructingObjectParser<IndexTemplateMetaData, String> PARSER = new ConstructingObjectParser<>(
+        "IndexTemplateMetaData", true, (a, name) -> {
+        List<Map.Entry<String, AliasMetaData>> alias = (List<Map.Entry<String, AliasMetaData>>) a[5];
+        ImmutableOpenMap<String, AliasMetaData> aliasMap =
+            new ImmutableOpenMap.Builder<String, AliasMetaData>()
+                .putAll(alias.stream().collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)))
+                .build();
+        return new IndexTemplateMetaData(
+            name,
+            (Integer) a[0],
+            (Integer) a[1],
+            (List<String>) a[2],
+            (Settings) a[3],
+            (MappingMetaData) a[4],
+            aliasMap);
+    });
+
+    static {
+        PARSER.declareInt(optionalConstructorArg(), new ParseField("order"));
+        PARSER.declareInt(optionalConstructorArg(), new ParseField("version"));
+        PARSER.declareStringArray(optionalConstructorArg(), new ParseField("index_patterns"));
+        PARSER.declareObject(optionalConstructorArg(), (p, c) -> {
+            Settings.Builder templateSettingsBuilder = Settings.builder();
+            templateSettingsBuilder.put(Settings.fromXContent(p));
+            templateSettingsBuilder.normalizePrefix(IndexMetaData.INDEX_SETTING_PREFIX);
+            return templateSettingsBuilder.build();
+        }, new ParseField("settings"));
+        PARSER.declareObject(optionalConstructorArg(), (p, c) -> {
+            Map<String, Object> mapping = p.map();
+            if (mapping.isEmpty()) {
+                return null;
+            }
+            return new MappingMetaData(MapperService.SINGLE_MAPPING_NAME, mapping);
+        }, new ParseField("mappings"));
+        PARSER.declareNamedObjects(optionalConstructorArg(),
+            (p, c, name) -> new AbstractMap.SimpleEntry<>(name, AliasMetaData.Builder.fromXContent(p)), new ParseField("aliases"));
+    }
 
     private final String name;
 
@@ -125,28 +164,23 @@ public static Builder builder(String name) {
     public boolean equals(Object o) {
         if (this == o) return true;
         if (o == null || getClass() != o.getClass()) return false;
-
         IndexTemplateMetaData that = (IndexTemplateMetaData) o;
-
-        if (order != that.order) return false;
-        if (!Objects.equals(mappings, that.mappings)) return false;
-        if (!name.equals(that.name)) return false;
-        if (!settings.equals(that.settings)) return false;
-        if (!patterns.equals(that.patterns)) return false;
-
-        return Objects.equals(version, that.version);
+        return order == that.order &&
+            Objects.equals(name, that.name) &&
+            Objects.equals(version, that.version) &&
+            Objects.equals(patterns, that.patterns) &&
+            Objects.equals(settings, that.settings) &&
+            Objects.equals(mappings, that.mappings) &&
+            Objects.equals(aliases, that.aliases);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(name, order, version, patterns, settings, mappings);
+        return Objects.hash(name, order, version, patterns, settings, mappings, aliases);
     }
 
     public static class Builder {
 
-        private static final Set<String> VALID_FIELDS = Sets.newHashSet(
-            "template", "order", "mappings", "settings", "index_patterns", "aliases", "version");
-
         private String name;
 
         private int order;
@@ -193,7 +227,6 @@ public Builder patterns(List<String> indexPatterns) {
             return this;
         }
 
-
         public Builder settings(Settings.Builder settings) {
             this.settings = settings.build();
             return this;
@@ -225,76 +258,7 @@ public IndexTemplateMetaData build() {
 
 
         public static IndexTemplateMetaData fromXContent(XContentParser parser, String templateName) throws IOException {
-            Builder builder = new Builder(templateName);
-
-            String currentFieldName = skipTemplateName(parser);
-            XContentParser.Token token;
-            while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-                if (token == XContentParser.Token.FIELD_NAME) {
-                    currentFieldName = parser.currentName();
-                } else if (token == XContentParser.Token.START_OBJECT) {
-                    if ("settings".equals(currentFieldName)) {
-                        Settings.Builder templateSettingsBuilder = Settings.builder();
-                        templateSettingsBuilder.put(Settings.fromXContent(parser));
-                        templateSettingsBuilder.normalizePrefix(IndexMetaData.INDEX_SETTING_PREFIX);
-                        builder.settings(templateSettingsBuilder.build());
-                    } else if ("mappings".equals(currentFieldName)) {
-                        Map<String, Object> mapping = parser.map();
-                        if (mapping.isEmpty() == false) {
-                            MappingMetaData md = new MappingMetaData(MapperService.SINGLE_MAPPING_NAME, mapping);
-                            builder.mapping(md);
-                        }
-                    } else if ("aliases".equals(currentFieldName)) {
-                        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-                            builder.putAlias(AliasMetaData.Builder.fromXContent(parser));
-                        }
-                    } else {
-                        throw new ElasticsearchParseException("unknown key [{}] for index template", currentFieldName);
-                    }
-                } else if (token == XContentParser.Token.START_ARRAY) {
-                    if ("mappings".equals(currentFieldName)) {
-                        // The server-side IndexTemplateMetaData has toXContent impl that can return mappings
-                        // in an array but also a comment saying this never happens with typeless APIs.
-                        throw new ElasticsearchParseException("Invalid response format - "
-                                + "mappings are not expected to be returned in an array", currentFieldName);
-                    } else if ("index_patterns".equals(currentFieldName)) {
-                        List<String> index_patterns = new ArrayList<>();
-                        while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
-                            index_patterns.add(parser.text());
-                        }
-                        builder.patterns(index_patterns);
-                    }
-                } else if (token.isValue()) {
-                    // Prior to 5.1.0, elasticsearch only supported a single index pattern called `template` (#21009)
-                    if("template".equals(currentFieldName)) {
-                        builder.patterns(Collections.singletonList(parser.text()));
-                    } else if ("order".equals(currentFieldName)) {
-                        builder.order(parser.intValue());
-                    } else if ("version".equals(currentFieldName)) {
-                        builder.version(parser.intValue());
-                    }
-                }
-            }
-            return builder.build();
-        }
-
-        private static String skipTemplateName(XContentParser parser) throws IOException {
-            XContentParser.Token token = parser.nextToken();
-            if (token == XContentParser.Token.START_OBJECT) {
-                token = parser.nextToken();
-                if (token == XContentParser.Token.FIELD_NAME) {
-                    String currentFieldName = parser.currentName();
-                    if (VALID_FIELDS.contains(currentFieldName)) {
-                        return currentFieldName;
-                    } else {
-                        // we just hit the template name, which should be ignored and we move on
-                        parser.nextToken();
-                    }
-                }
-            }
-
-            return null;
+            return PARSER.parse(parser, templateName);
         }
     }
-
 }