Merge branch 'upstream/master' into rollup-support-cancel

* upstream/master: (38 commits)
  Simplify map copying (elastic#88432)
  Make DiffableUtils.diff implementation agnostic (elastic#88403)
  Ingest: Start separating Metadata from IngestSourceAndMetadata (elastic#88401)
  Move runtime fields base scripts out of scripting fields api package. (elastic#88488)
  Enable TRACE Logging for test and increase timeout (elastic#88477)
  Mute ReactiveStorageIT#testScaleDuringSplitOrClone (elastic#88480)
  Track the count of failed invocations since last successful policy snapshot (elastic#88398)
  Avoid noisy exceptions on data nodes when aborting snapshots (elastic#88476)
  Fix ReactiveStorageDeciderServiceTests testNodeSizeForDataBelowLowWatermark (elastic#88452)
  INFO logging of snapshot restore and completion (elastic#88257)
  unmute test (elastic#88454)
  Updatable API keys - noop check (elastic#88346)
  Corrected an incomplete sentence. (elastic#86542)
  Use consistent shard map type in IndexService (elastic#88465)
  Stop registering TestGeoShapeFieldMapperPlugin in ESIntegTestCase (elastic#88460)
  TSDB: RollupShardIndexer logging improvements (elastic#88416)
  Audit API key ID when create or grant API keys (elastic#88456)
  Bound random negative size test in SearchSourceBuilderTests#testNegativeSizeErrors (elastic#88457)
  Updatable API keys - logging audit trail event (elastic#88276)
  Polish reworked LoggedExec task (elastic#88424)
  ...

# Conflicts:
#	x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/v2/RollupShardIndexer.java

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalBwcGitPluginFuncTest.groovy

@@ -14,8 +14,6 @@ import org.gradle.testkit.runner.TaskOutcome
 class InternalBwcGitPluginFuncTest extends AbstractGitAwareGradleFuncTest {
 
     def setup() {
-        // using LoggedExec is not cc compatible
-        configurationCacheCompatible = false
         internalBuild()
         buildFile << """
             import org.elasticsearch.gradle.Version;

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPluginFuncTest.groovy

@@ -17,12 +17,11 @@ import spock.lang.Unroll
 /*
  * Test is ignored on ARM since this test case tests the ability to build certain older BWC branches that we don't support on ARM
  */
-
 @IgnoreIf({ Architecture.current() == Architecture.AARCH64 })
 class InternalDistributionBwcSetupPluginFuncTest extends AbstractGitAwareGradleFuncTest {
 
     def setup() {
-        // used LoggedExec task is not configuration cache compatible and
+        // Cannot serialize BwcSetupExtension containing project object
         configurationCacheCompatible = false
         internalBuild()
         buildFile << """
@@ -119,4 +118,5 @@ class InternalDistributionBwcSetupPluginFuncTest extends AbstractGitAwareGradleF
         result.output.contains("nested folder /distribution/bwc/minor/build/bwc/checkout-8.0/" +
                         "distribution/archives/darwin-tar/build/install/elasticsearch-8.0.0-SNAPSHOT")
     }
+
 }

build-tools-internal/src/main/groovy/org/elasticsearch/gradle/internal/AntFixtureStop.groovy

@@ -12,22 +12,20 @@ import org.apache.tools.ant.taskdefs.condition.Os
 import org.elasticsearch.gradle.LoggedExec
 import org.elasticsearch.gradle.internal.test.AntFixture
 import org.gradle.api.file.FileSystemOperations
+import org.gradle.api.file.ProjectLayout
 import org.gradle.api.tasks.Internal
+import org.gradle.process.ExecOperations
 
 import javax.inject.Inject
 
-class AntFixtureStop extends LoggedExec implements FixtureStop {
+abstract class AntFixtureStop extends LoggedExec implements FixtureStop {
 
     @Internal
     AntFixture fixture
 
-    @Internal
-    FileSystemOperations fileSystemOperations
-
     @Inject
-    AntFixtureStop(FileSystemOperations fileSystemOperations) {
-        super(fileSystemOperations)
-        this.fileSystemOperations = fileSystemOperations
+    AntFixtureStop(ProjectLayout projectLayout, ExecOperations execOperations, FileSystemOperations fileSystemOperations) {
+       super(projectLayout, execOperations, fileSystemOperations)
     }
 
     void setFixture(AntFixture fixture) {
@@ -40,10 +38,10 @@ class AntFixtureStop extends LoggedExec implements FixtureStop {
         }
 
         if (Os.isFamily(Os.FAMILY_WINDOWS)) {
-            executable = 'Taskkill'
+            getExecutable().set('Taskkill')
             args('/PID', pid, '/F')
         } else {
-            executable = 'kill'
+            getExecutable().set('kill')
             args('-9', pid)
         }
         doLast {

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcSetupExtension.java

@@ -15,15 +15,12 @@
 import org.gradle.api.Action;
 import org.gradle.api.GradleException;
 import org.gradle.api.Project;
-import org.gradle.api.Task;
 import org.gradle.api.logging.LogLevel;
 import org.gradle.api.provider.Provider;
 import org.gradle.api.tasks.TaskProvider;
 
 import java.io.File;
 import java.io.IOException;
-import java.io.OutputStream;
-import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Locale;
@@ -64,26 +61,21 @@ private TaskProvider<LoggedExec> createRunBwcGradleTask(Project project, String
         return project.getTasks().register(name, LoggedExec.class, loggedExec -> {
             loggedExec.dependsOn("checkoutBwcBranch");
             loggedExec.usesService(bwcTaskThrottleProvider);
-            loggedExec.setSpoolOutput(true);
-            loggedExec.setWorkingDir(checkoutDir.get());
-            loggedExec.doFirst(new Action<Task>() {
-                @Override
-                public void execute(Task t) {
-                    // Execution time so that the checkouts are available
-                    String compilerVersionInfoPath = minimumCompilerVersionPath(unreleasedVersionInfo.get().version());
-                    String minimumCompilerVersion = readFromFile(new File(checkoutDir.get(), compilerVersionInfoPath));
-                    loggedExec.environment("JAVA_HOME", getJavaHome(Integer.parseInt(minimumCompilerVersion)));
-                }
-            });
+            loggedExec.getWorkingDir().set(checkoutDir.get());
+
+            loggedExec.getEnvironment().put("JAVA_HOME", unreleasedVersionInfo.zip(checkoutDir, (version, checkoutDir) -> {
+                String minimumCompilerVersion = readFromFile(new File(checkoutDir, minimumCompilerVersionPath(version.version())));
+                return getJavaHome(Integer.parseInt(minimumCompilerVersion));
+            }));
 
             if (Os.isFamily(Os.FAMILY_WINDOWS)) {
-                loggedExec.executable("cmd");
+                loggedExec.getExecutable().set("cmd");
                 loggedExec.args("/C", "call", new File(checkoutDir.get(), "gradlew").toString());
             } else {
-                loggedExec.executable(new File(checkoutDir.get(), "gradlew").toString());
+                loggedExec.getExecutable().set(new File(checkoutDir.get(), "gradlew").toString());
             }
 
-            loggedExec.args("-g", project.getGradle().getGradleUserHomeDir());
+            loggedExec.args("-g", project.getGradle().getGradleUserHomeDir().toString());
             if (project.getGradle().getStartParameter().isOffline()) {
                 loggedExec.args("--offline");
             }
@@ -93,8 +85,7 @@ public void execute(Task t) {
                 loggedExec.args("-Dorg.elasticsearch.build.cache.url=" + buildCacheUrl);
             }
 
-            loggedExec.args("-Dbuild.snapshot=true");
-            loggedExec.args("-Dscan.tag.NESTED");
+            loggedExec.args("-Dbuild.snapshot=true", "-Dscan.tag.NESTED");
             final LogLevel logLevel = project.getGradle().getStartParameter().getLogLevel();
             List<LogLevel> nonDefaultLogLevels = Arrays.asList(LogLevel.QUIET, LogLevel.WARN, LogLevel.INFO, LogLevel.DEBUG);
             if (nonDefaultLogLevels.contains(logLevel)) {
@@ -110,8 +101,7 @@ public void execute(Task t) {
             if (project.getGradle().getStartParameter().isParallelProjectExecutionEnabled()) {
                 loggedExec.args("--parallel");
             }
-            loggedExec.setStandardOutput(new IndentingOutputStream(System.out, unreleasedVersionInfo.get().version()));
-            loggedExec.setErrorOutput(new IndentingOutputStream(System.err, unreleasedVersionInfo.get().version()));
+            loggedExec.getIndentingConsoleOutput().set(unreleasedVersionInfo.map(v -> v.version().toString()));
             configAction.execute(loggedExec);
         });
     }
@@ -122,32 +112,6 @@ private String minimumCompilerVersionPath(Version bwcVersion) {
             : "buildSrc/" + MINIMUM_COMPILER_VERSION_PATH;
     }
 
-    private static class IndentingOutputStream extends OutputStream {
-
-        public final byte[] indent;
-        private final OutputStream delegate;
-
-        IndentingOutputStream(OutputStream delegate, Object version) {
-            this.delegate = delegate;
-            indent = (" [" + version + "] ").getBytes(StandardCharsets.UTF_8);
-        }
-
-        @Override
-        public void write(int b) throws IOException {
-            int[] arr = { b };
-            write(arr, 0, 1);
-        }
-
-        public void write(int[] bytes, int offset, int length) throws IOException {
-            for (int i = 0; i < bytes.length; i++) {
-                delegate.write(bytes[i]);
-                if (bytes[i] == '\n') {
-                    delegate.write(indent);
-                }
-            }
-        }
-    }
-
     private static String readFromFile(File file) {
         try {
             return FileUtils.readFileToString(file).trim();

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalBwcGitPlugin.java

@@ -69,29 +69,27 @@ public void apply(Project project) {
         TaskContainer tasks = project.getTasks();
         TaskProvider<LoggedExec> createCloneTaskProvider = tasks.register("createClone", LoggedExec.class, createClone -> {
             createClone.onlyIf(task -> this.gitExtension.getCheckoutDir().get().exists() == false);
-            createClone.setCommandLine(asList("git", "clone", buildLayout.getRootDirectory(), gitExtension.getCheckoutDir().get()));
+            createClone.commandLine("git", "clone", buildLayout.getRootDirectory(), gitExtension.getCheckoutDir().get());
         });
 
         ExtraPropertiesExtension extraProperties = project.getExtensions().getExtraProperties();
         TaskProvider<LoggedExec> findRemoteTaskProvider = tasks.register("findRemote", LoggedExec.class, findRemote -> {
             findRemote.dependsOn(createCloneTaskProvider);
-            // TODO Gradle should provide property based configuration here
-            findRemote.setWorkingDir(gitExtension.getCheckoutDir().get());
-            findRemote.setCommandLine(asList("git", "remote", "-v"));
-            ByteArrayOutputStream output = new ByteArrayOutputStream();
-            findRemote.setStandardOutput(output);
-            findRemote.doLast(t -> { extraProperties.set("remoteExists", isRemoteAvailable(remote, output)); });
+            findRemote.getWorkingDir().set(gitExtension.getCheckoutDir());
+            findRemote.commandLine("git", "remote", "-v");
+            findRemote.getCaptureOutput().set(true);
+            findRemote.doLast(t -> { extraProperties.set("remoteExists", isRemoteAvailable(remote, findRemote.getOutput())); });
         });
 
         TaskProvider<LoggedExec> addRemoteTaskProvider = tasks.register("addRemote", LoggedExec.class, addRemote -> {
             addRemote.dependsOn(findRemoteTaskProvider);
             addRemote.onlyIf(task -> ((boolean) extraProperties.get("remoteExists")) == false);
-            addRemote.setWorkingDir(gitExtension.getCheckoutDir().get());
+            addRemote.getWorkingDir().set(gitExtension.getCheckoutDir().get());
             String remoteRepo = remote.get();
             // for testing only we can override the base remote url
             String remoteRepoUrl = providerFactory.systemProperty("testRemoteRepo")
                 .getOrElse("https://github.com/" + remoteRepo + "/elasticsearch.git");
-            addRemote.setCommandLine(asList("git", "remote", "add", remoteRepo, remoteRepoUrl));
+            addRemote.commandLine("git", "remote", "add", remoteRepo, remoteRepoUrl);
         });
 
         boolean isOffline = project.getGradle().getStartParameter().isOffline();
@@ -107,8 +105,8 @@ public void apply(Project project) {
             });
             fetchLatest.onlyIf(t -> isOffline == false && gitFetchLatest.get());
             fetchLatest.dependsOn(addRemoteTaskProvider);
-            fetchLatest.setWorkingDir(gitExtension.getCheckoutDir().get());
-            fetchLatest.setCommandLine(asList("git", "fetch", "--all"));
+            fetchLatest.getWorkingDir().set(gitExtension.getCheckoutDir().get());
+            fetchLatest.commandLine("git", "fetch", "--all");
         });
 
         String projectPath = project.getPath();
@@ -210,7 +208,7 @@ private String execInCheckoutDir(Action<ExecSpec> execSpecConfig) {
         return os.toString().trim();
     }
 
-    private static boolean isRemoteAvailable(Provider<String> remote, ByteArrayOutputStream output) {
-        return new String(output.toByteArray()).lines().anyMatch(l -> l.contains(remote.get() + "\t"));
+    private static boolean isRemoteAvailable(Provider<String> remote, String output) {
+        return output.lines().anyMatch(l -> l.contains(remote.get() + "\t"));
     }
 }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPlugin.java

@@ -241,9 +241,9 @@ static void createBuildBwcTask(
                 return BuildParams.isCi()
                     && (gitBranch == null || gitBranch.endsWith("master") == false || gitBranch.endsWith("main") == false);
             });
-            c.args(projectPath.replace('/', ':') + ":" + assembleTaskName);
+            c.getArgs().add(projectPath.replace('/', ':') + ":" + assembleTaskName);
             if (project.getGradle().getStartParameter().isBuildCacheEnabled()) {
-                c.args("--build-cache");
+                c.getArgs().add("--build-cache");
             }
             c.doLast(new Action<Task>() {
                 @Override