New release notes generator tasks (#71125)

Part of #67335.

Add tasks for generating release notes, using information stored in files
in the repository:

   * `generateReleaseNotes` - generates new release notes, release
     highlights and breaking changes
   * `validateChangelogs` - validates that all the changelog YAML files are
     well-formed (confirm to schema, have required fields depending on the
     `type` value)

I also changed `Version` to allow a `v` prefix in relaxed mode

build-tools-internal/build.gradle

@@ -110,6 +110,10 @@ gradlePlugin {
       id = 'elasticsearch.jdk-download'
       implementationClass = 'org.elasticsearch.gradle.internal.JdkDownloadPlugin'
     }
+    releaseTools {
+      id = 'elasticsearch.release-tools'
+      implementationClass = 'org.elasticsearch.gradle.internal.release.ReleaseToolsPlugin'
+    }
     repositories {
       id = 'elasticsearch.repositories'
       implementationClass = 'org.elasticsearch.gradle.internal.RepositoriesSetupPlugin'

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ValidateJsonAgainstSchemaTask.java

@@ -20,9 +20,11 @@
 import org.gradle.api.file.FileCollection;
 import org.gradle.api.tasks.InputFile;
 import org.gradle.api.tasks.InputFiles;
+import org.gradle.api.tasks.Internal;
 import org.gradle.api.tasks.OutputFile;
 import org.gradle.api.tasks.TaskAction;
 import org.gradle.work.ChangeType;
+import org.gradle.work.FileChange;
 import org.gradle.work.Incremental;
 import org.gradle.work.InputChanges;
 
@@ -42,8 +44,6 @@
  * Incremental task to validate a set of JSON files against against a schema.
  */
 public class ValidateJsonAgainstSchemaTask extends DefaultTask {
-
-    private final ObjectMapper mapper = new ObjectMapper();
     private File jsonSchema;
     private File report;
     private FileCollection inputFiles;
@@ -76,28 +76,36 @@ public File getReport() {
         return this.report;
     }
 
+    @Internal
+    protected ObjectMapper getMapper() {
+        return new ObjectMapper();
+    }
+
+    @Internal
+    protected String getFileType() {
+        return "JSON";
+    }
+
     @TaskAction
     public void validate(InputChanges inputChanges) throws IOException {
-        File jsonSchemaOnDisk = getJsonSchema();
-        getLogger().debug("JSON schema : [{}]", jsonSchemaOnDisk.getAbsolutePath());
-        SchemaValidatorsConfig config = new SchemaValidatorsConfig();
-        JsonSchemaFactory factory = JsonSchemaFactory.getInstance(SpecVersion.VersionFlag.V7);
-        JsonSchema jsonSchema = factory.getSchema(mapper.readTree(jsonSchemaOnDisk), config);
-        Map<File, Set<String>> errors = new LinkedHashMap<>();
+        final File jsonSchemaOnDisk = getJsonSchema();
+        final JsonSchema jsonSchema = buildSchemaObject(jsonSchemaOnDisk);
+
+        final Map<File, Set<String>> errors = new LinkedHashMap<>();
+        final ObjectMapper mapper = this.getMapper();
+
         // incrementally evaluate input files
+        // validate all files and hold on to errors for a complete report if there are failures
         StreamSupport.stream(inputChanges.getFileChanges(getInputFiles()).spliterator(), false)
             .filter(f -> f.getChangeType() != ChangeType.REMOVED)
-            .forEach(fileChange -> {
-                File file = fileChange.getFile();
-                if (file.isDirectory() == false) {
-                    // validate all files and hold on to errors for a complete report if there are failures
-                    getLogger().debug("Validating JSON [{}]", file.getName());
-                    try {
-                        Set<ValidationMessage> validationMessages = jsonSchema.validate(mapper.readTree(file));
-                        maybeLogAndCollectError(validationMessages, errors, file);
-                    } catch (IOException e) {
-                        throw new UncheckedIOException(e);
-                    }
+            .map(FileChange::getFile)
+            .filter(file -> file.isDirectory() == false)
+            .forEach(file -> {
+                try {
+                    Set<ValidationMessage> validationMessages = jsonSchema.validate(mapper.readTree(file));
+                    maybeLogAndCollectError(validationMessages, errors, file);
+                } catch (IOException e) {
+                    throw new UncheckedIOException(e);
                 }
             });
         if (errors.isEmpty()) {
@@ -119,9 +127,17 @@ public void validate(InputChanges inputChanges) throws IOException {
         }
     }
 
+    private JsonSchema buildSchemaObject(File jsonSchemaOnDisk) throws IOException {
+        final ObjectMapper jsonMapper = new ObjectMapper();
+        final SchemaValidatorsConfig config = new SchemaValidatorsConfig();
+        final JsonSchemaFactory factory = JsonSchemaFactory.getInstance(SpecVersion.VersionFlag.V7);
+        return factory.getSchema(jsonMapper.readTree(jsonSchemaOnDisk), config);
+    }
+
     private void maybeLogAndCollectError(Set<ValidationMessage> messages, Map<File, Set<String>> errors, File file) {
+        final String fileType = getFileType();
         for (ValidationMessage message : messages) {
-            getLogger().error("[validate JSON][ERROR][{}][{}]", file.getName(), message.toString());
+            getLogger().error("[validate {}][ERROR][{}][{}]", fileType, file.getName(), message.toString());
             errors.computeIfAbsent(file, k -> new LinkedHashSet<>())
                 .add(String.format("%s: %s", file.getAbsolutePath(), message.toString()));
         }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ValidateYamlAgainstSchemaTask.java

@@ -0,0 +1,26 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.gradle.internal.precommit;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
+
+/**
+ * Incremental task to validate a set of YAML files against against a schema.
+ */
+public class ValidateYamlAgainstSchemaTask extends ValidateJsonAgainstSchemaTask {
+    @Override
+    protected String getFileType() {
+        return "YAML";
+    }
+
+    protected ObjectMapper getMapper() {
+        return new ObjectMapper(new YAMLFactory());
+    }
+}

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/BreakingChangesGenerator.java

@@ -0,0 +1,75 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.gradle.internal.release;
+
+import groovy.text.SimpleTemplateEngine;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import org.elasticsearch.gradle.Version;
+import org.elasticsearch.gradle.VersionProperties;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.TreeMap;
+import java.util.stream.Collectors;
+
+/**
+ * Generates the page that lists the breaking changes and deprecations for a minor version release.
+ */
+public class BreakingChangesGenerator {
+
+    static void update(File templateFile, File outputFile, List<ChangelogEntry> entries) throws IOException {
+        try (FileWriter output = new FileWriter(outputFile)) {
+            generateFile(Files.readString(templateFile.toPath()), output, entries);
+        }
+    }
+
+    @VisibleForTesting
+    private static void generateFile(String template, FileWriter outputWriter, List<ChangelogEntry> entries) throws IOException {
+        final Version version = VersionProperties.getElasticsearchVersion();
+
+        final Map<Boolean, Map<String, List<ChangelogEntry.Breaking>>> breakingChangesByNotabilityByArea = entries.stream()
+            .map(ChangelogEntry::getBreaking)
+            .filter(Objects::nonNull)
+            .collect(
+                Collectors.groupingBy(
+                    ChangelogEntry.Breaking::isNotable,
+                    Collectors.groupingBy(ChangelogEntry.Breaking::getArea, TreeMap::new, Collectors.toList())
+                )
+            );
+
+        final Map<String, List<ChangelogEntry.Deprecation>> deprecationsByArea = entries.stream()
+            .map(ChangelogEntry::getDeprecation)
+            .filter(Objects::nonNull)
+            .collect(Collectors.groupingBy(ChangelogEntry.Deprecation::getArea, TreeMap::new, Collectors.toList()));
+
+        final Map<String, Object> bindings = new HashMap<>();
+        bindings.put("breakingChangesByNotabilityByArea", breakingChangesByNotabilityByArea);
+        bindings.put("deprecationsByArea", deprecationsByArea);
+        bindings.put("isElasticsearchSnapshot", VersionProperties.isElasticsearchSnapshot());
+        bindings.put("majorDotMinor", version.getMajor() + "." + version.getMinor());
+        bindings.put("majorMinor", String.valueOf(version.getMajor()) + version.getMinor());
+        bindings.put("nextMajor", (version.getMajor() + 1) + ".0");
+        bindings.put("version", version);
+
+        try {
+            final SimpleTemplateEngine engine = new SimpleTemplateEngine();
+            engine.createTemplate(template).make(bindings).writeTo(outputWriter);
+        } catch (ClassNotFoundException e) {
+            throw new RuntimeException(e);
+        }
+    }
+}