Merge branch 'master' into delete-searchable-snapshot-with-pending-de…

…letes

.ci/jobs.t/defaults.yml

@@ -53,6 +53,7 @@
           url: https://github.com/elastic/elasticsearch/
       - inject:
           properties-content: |
+            COMPOSE_HTTP_TIMEOUT=120
             JOB_BRANCH=%BRANCH%
             HOME=$JENKINS_HOME
             GRADLEW=./gradlew --parallel --scan --build-cache -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/

build-conventions/src/main/java/org/elasticsearch/gradle/internal/checkstyle/HiddenFieldCheck.java

@@ -71,9 +71,15 @@ public class HiddenFieldCheck extends AbstractCheck {
     /** Control whether to ignore constructor parameters. */
     private boolean ignoreConstructorParameter;
 
+    /** Control whether to ignore variables in constructor bodies. */
+    private boolean ignoreConstructorBody;
+
     /** Control whether to ignore parameters of abstract methods. */
     private boolean ignoreAbstractMethods;
 
+    /** If set, specifies a regex of method names that should be ignored */
+    private String ignoredMethodNames;
+
     @Override
     public int[] getDefaultTokens() {
         return getAcceptableTokens();
@@ -224,7 +230,8 @@ private void processVariable(DetailAST ast) {
 
             if ((frame.containsStaticField(name) || isInstanceField(ast, name))
                 && isMatchingRegexp(name) == false
-                && isIgnoredParam(ast, name) == false) {
+                && isIgnoredParam(ast, name) == false
+                && isIgnoredVariable(ast, name) == false) {
                 log(nameAST, MSG_KEY, name);
             }
         }
@@ -238,7 +245,14 @@ && isIgnoredParam(ast, name) == false) {
      * @return true if parameter is ignored.
      */
     private boolean isIgnoredParam(DetailAST ast, String name) {
-        return isIgnoredSetterParam(ast, name) || isIgnoredConstructorParam(ast) || isIgnoredParamOfAbstractMethod(ast);
+        return isVariableInIgnoredMethod(ast, name)
+            || isIgnoredSetterParam(ast, name)
+            || isIgnoredConstructorParam(ast)
+            || isIgnoredParamOfAbstractMethod(ast);
+    }
+
+    private boolean isIgnoredVariable(DetailAST ast, String name) {
+        return isIgnoredVariableInConstructorBody(ast, name);
     }
 
     /**
@@ -410,6 +424,42 @@ private boolean isIgnoredParamOfAbstractMethod(DetailAST ast) {
         return result;
     }
 
+    /**
+     * Decides whether to ignore an AST node that is the parameter of a method whose
+     * name matches the {@link #ignoredMethodNames} regex, if set.
+     * @param ast the AST to check
+     * @return true is the ast should be ignored because the parameter belongs to a
+     *      method whose name matches the regex.
+     */
+    private boolean isVariableInIgnoredMethod(DetailAST ast, String name) {
+        boolean result = false;
+        if (ignoredMethodNames != null && (ast.getType() == TokenTypes.PARAMETER_DEF || ast.getType() == TokenTypes.VARIABLE_DEF)) {
+            DetailAST method = ast.getParent();
+            while (method != null && method.getType() != TokenTypes.METHOD_DEF) {
+                method = method.getParent();
+            }
+            if (method != null && method.getType() == TokenTypes.METHOD_DEF) {
+                final String methodName = method.findFirstToken(TokenTypes.IDENT).getText();
+                result = methodName.matches(ignoredMethodNames);
+            }
+        }
+        return result;
+    }
+
+    private boolean isIgnoredVariableInConstructorBody(DetailAST ast, String name) {
+        boolean result = false;
+
+        if (ignoreConstructorBody && ast.getType() == TokenTypes.VARIABLE_DEF) {
+            DetailAST method = ast.getParent();
+            while (method != null && method.getType() != TokenTypes.CTOR_DEF) {
+                method = method.getParent();
+            }
+            result = method != null && method.getType() == TokenTypes.CTOR_DEF;
+        }
+
+        return result;
+    }
+
     /**
      * Setter to define the RegExp for names of variables and parameters to ignore.
      *
@@ -463,6 +513,14 @@ public void setIgnoreAbstractMethods(boolean ignoreAbstractMethods) {
         this.ignoreAbstractMethods = ignoreAbstractMethods;
     }
 
+    public void setIgnoredMethodNames(String ignoredMethodNames) {
+        this.ignoredMethodNames = ignoredMethodNames;
+    }
+
+    public void setIgnoreConstructorBody(boolean ignoreConstructorBody) {
+        this.ignoreConstructorBody = ignoreConstructorBody;
+    }
+
     /**
      * Holds the names of static and instance fields of a type.
      */

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java

@@ -97,6 +97,7 @@ public void execute(Task t) {
                 "-Xmx" + System.getProperty("tests.heap.size", "512m"),
                 "-Xms" + System.getProperty("tests.heap.size", "512m"),
                 "--illegal-access=deny",
+                "-Djava.security.manager=allow",
                 // TODO: only open these for mockito when it is modularized
                 "--add-opens=java.base/java.security.cert=ALL-UNNAMED",
                 "--add-opens=java.base/java.nio.channels=ALL-UNNAMED",

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/BreakingChangesGenerator.java

@@ -11,56 +11,118 @@
 import com.google.common.annotations.VisibleForTesting;
 
 import org.elasticsearch.gradle.VersionProperties;
+import org.gradle.api.GradleException;
 
 import java.io.File;
 import java.io.FileWriter;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.util.HashMap;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Set;
 import java.util.TreeMap;
+import java.util.TreeSet;
 import java.util.stream.Collectors;
 
 import static java.util.Comparator.comparing;
 import static java.util.stream.Collectors.groupingBy;
+import static java.util.stream.Collectors.toCollection;
 
 /**
- * Generates the page that lists the breaking changes and deprecations for a minor version release.
+ * Generates the page that contains an index into the breaking changes and lists deprecations for a minor version release,
+ * and the individual pages for each breaking area.
  */
 public class BreakingChangesGenerator {
 
-    static void update(File templateFile, File outputFile, List<ChangelogEntry> entries) throws IOException {
-        try (FileWriter output = new FileWriter(outputFile)) {
+    // Needs to match `changelog-schema.json`
+    private static final List<String> BREAKING_AREAS = List.of(
+        "Cluster and node setting",
+        "Command line tool",
+        "Index setting",
+        "JVM option",
+        "Java API",
+        "Logging",
+        "Mapping",
+        "Packaging",
+        "Painless",
+        "REST API",
+        "System requirement",
+        "Transform"
+    );
+
+    static void update(
+        File indexTemplateFile,
+        File indexOutputFile,
+        File outputDirectory,
+        File areaTemplateFile,
+        List<ChangelogEntry> entries
+    ) throws IOException {
+        if (outputDirectory.exists()) {
+            if (outputDirectory.isDirectory() == false) {
+                throw new GradleException("Path [" + outputDirectory + "] exists but isn't a directory!");
+            }
+        } else {
+            Files.createDirectory(outputDirectory.toPath());
+        }
+
+        try (FileWriter output = new FileWriter(indexOutputFile)) {
             output.write(
-                generateFile(QualifiedVersion.of(VersionProperties.getElasticsearch()), Files.readString(templateFile.toPath()), entries)
+                generateIndexFile(
+                    QualifiedVersion.of(VersionProperties.getElasticsearch()),
+                    Files.readString(indexTemplateFile.toPath()),
+                    entries
+                )
             );
         }
-    }
 
-    @VisibleForTesting
-    static String generateFile(QualifiedVersion version, String template, List<ChangelogEntry> entries) throws IOException {
+        String areaTemplate = Files.readString(areaTemplateFile.toPath());
 
-        final Map<Boolean, Map<String, List<ChangelogEntry.Breaking>>> breakingChangesByNotabilityByArea = entries.stream()
-            .map(ChangelogEntry::getBreaking)
-            .filter(Objects::nonNull)
-            .sorted(comparing(ChangelogEntry.Breaking::getTitle))
-            .collect(
-                groupingBy(
-                    ChangelogEntry.Breaking::isNotable,
-                    groupingBy(ChangelogEntry.Breaking::getArea, TreeMap::new, Collectors.toList())
-                )
-            );
+        for (String breakingArea : BREAKING_AREAS) {
+            final List<ChangelogEntry.Breaking> entriesForArea = entries.stream()
+                .map(ChangelogEntry::getBreaking)
+                .filter(entry -> entry != null && breakingArea.equals(entry.getArea()))
+                .collect(Collectors.toList());
+
+            if (entriesForArea.isEmpty()) {
+                continue;
+            }
+
+            final String outputFilename = breakingArea.toLowerCase(Locale.ROOT).replaceFirst(" and", "").replaceAll(" ", "-")
+                + "-changes.asciidoc";
+
+            try (FileWriter output = new FileWriter(outputDirectory.toPath().resolve(outputFilename).toFile())) {
+                output.write(
+                    generateBreakingAreaFile(
+                        QualifiedVersion.of(VersionProperties.getElasticsearch()),
+                        areaTemplate,
+                        breakingArea,
+                        entriesForArea
+                    )
+                );
+            }
+        }
+    }
 
+    @VisibleForTesting
+    static String generateIndexFile(QualifiedVersion version, String template, List<ChangelogEntry> entries) throws IOException {
         final Map<String, List<ChangelogEntry.Deprecation>> deprecationsByArea = entries.stream()
             .map(ChangelogEntry::getDeprecation)
             .filter(Objects::nonNull)
             .sorted(comparing(ChangelogEntry.Deprecation::getTitle))
             .collect(groupingBy(ChangelogEntry.Deprecation::getArea, TreeMap::new, Collectors.toList()));
 
+        final List<String> breakingIncludeList = entries.stream()
+            .filter(each -> each.getBreaking() != null)
+            .map(each -> each.getBreaking().getArea().toLowerCase(Locale.ROOT).replaceFirst(" and", "").replaceAll(" ", "-"))
+            .distinct()
+            .sorted()
+            .toList();
+
         final Map<String, Object> bindings = new HashMap<>();
-        bindings.put("breakingChangesByNotabilityByArea", breakingChangesByNotabilityByArea);
+        bindings.put("breakingIncludeList", breakingIncludeList);
         bindings.put("deprecationsByArea", deprecationsByArea);
         bindings.put("isElasticsearchSnapshot", version.isSnapshot());
         bindings.put("majorDotMinor", version.getMajor() + "." + version.getMinor());
@@ -70,4 +132,28 @@ static String generateFile(QualifiedVersion version, String template, List<Chang
 
         return TemplateUtils.render(template, bindings);
     }
+
+    @VisibleForTesting
+    static String generateBreakingAreaFile(
+        QualifiedVersion version,
+        String template,
+        String breakingArea,
+        List<ChangelogEntry.Breaking> entriesForArea
+    ) throws IOException {
+        final Map<Boolean, Set<ChangelogEntry.Breaking>> breakingEntriesByNotability = entriesForArea.stream()
+            .collect(
+                groupingBy(
+                    ChangelogEntry.Breaking::isNotable,
+                    toCollection(() -> new TreeSet<>(comparing(ChangelogEntry.Breaking::getTitle)))
+                )
+            );
+
+        final Map<String, Object> bindings = new HashMap<>();
+        bindings.put("breakingArea", breakingArea);
+        bindings.put("breakingEntriesByNotability", breakingEntriesByNotability);
+        bindings.put("breakingAreaAnchor", breakingArea.toLowerCase(Locale.ROOT).replaceFirst(" and", "").replaceAll(" ", "_"));
+        bindings.put("majorMinor", String.valueOf(version.getMajor()) + version.getMinor());
+
+        return TemplateUtils.render(template, bindings);
+    }
 }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ChangelogEntry.java

@@ -215,6 +215,7 @@ public static class Breaking {
         private String details;
         private String impact;
         private boolean notable;
+        private boolean essSettingChange;
 
         public String getArea() {
             return area;
@@ -260,6 +261,14 @@ public String getAnchor() {
             return generatedAnchor(this.title);
         }
 
+        public boolean isEssSettingChange() {
+            return essSettingChange;
+        }
+
+        public void setEssSettingChange(boolean essSettingChange) {
+            this.essSettingChange = essSettingChange;
+        }
+
         @Override
         public boolean equals(Object o) {
             if (this == o) {
@@ -273,23 +282,25 @@ public boolean equals(Object o) {
                 && Objects.equals(area, breaking.area)
                 && Objects.equals(title, breaking.title)
                 && Objects.equals(details, breaking.details)
-                && Objects.equals(impact, breaking.impact);
+                && Objects.equals(impact, breaking.impact)
+                && Objects.equals(essSettingChange, breaking.essSettingChange);
         }
 
         @Override
         public int hashCode() {
-            return Objects.hash(area, title, details, impact, notable);
+            return Objects.hash(area, title, details, impact, notable, essSettingChange);
         }
 
         @Override
         public String toString() {
             return String.format(
-                "Breaking{area='%s', title='%s', details='%s', impact='%s', isNotable=%s}",
+                "Breaking{area='%s', title='%s', details='%s', impact='%s', notable=%s, essSettingChange=%s}",
                 area,
                 title,
                 details,
                 impact,
-                notable
+                notable,
+                essSettingChange
             );
         }
     }
@@ -351,7 +362,7 @@ public String toString() {
     }
 
     private static String generatedAnchor(String input) {
-        final List<String> excludes = List.of("the", "is", "a");
+        final List<String> excludes = List.of("the", "is", "a", "and");
 
         final String[] words = input.toLowerCase(Locale.ROOT)
             .replaceAll("[^\\w]+", "_")