Merge branch 'main' into esql-data-type-conversion-2

build-tools-internal/build.gradle

@@ -179,6 +179,10 @@ gradlePlugin {
       id = 'elasticsearch.test.fixtures'
       implementationClass = 'org.elasticsearch.gradle.internal.testfixtures.TestFixturesPlugin'
     }
+    deployTestFixtures {
+      id = 'elasticsearch.deploy-test-fixtures'
+      implementationClass = 'org.elasticsearch.gradle.internal.testfixtures.TestFixturesDeployPlugin'
+    }
     testBase {
       id = 'elasticsearch.test-base'
       implementationClass = 'org.elasticsearch.gradle.internal.ElasticsearchTestBasePlugin'

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchBuildCompletePlugin.java

@@ -28,7 +28,12 @@
 import org.gradle.api.tasks.Input;
 import org.jetbrains.annotations.NotNull;
 
-import java.io.*;
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.OutputStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.ArrayList;
@@ -210,12 +215,15 @@ private static void createBuildArchiveTar(List<File> files, File projectDir, Fil
                         throw new IOException("Support only file!");
                     }
 
+                    long entrySize = Files.size(path);
                     TarArchiveEntry tarEntry = new TarArchiveEntry(path.toFile(), calculateArchivePath(path, projectPath));
-                    tarEntry.setSize(Files.size(path));
+                    tarEntry.setSize(entrySize);
                     tOut.putArchiveEntry(tarEntry);
 
                     // copy file to TarArchiveOutputStream
-                    Files.copy(path, tOut);
+                    try (BufferedInputStream bin = new BufferedInputStream(Files.newInputStream(path))) {
+                        IOUtils.copyLarge(bin, tOut, 0, entrySize);
+                    }
                     tOut.closeArchiveEntry();
 
                 }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/MrjarPlugin.java

@@ -60,48 +60,30 @@ public void apply(Project project) {
         project.getPluginManager().apply(ElasticsearchJavaBasePlugin.class);
         var javaExtension = project.getExtensions().getByType(JavaPluginExtension.class);
 
-        var srcDir = project.getProjectDir().toPath().resolve("src");
-        List<Integer> mainVersions = new ArrayList<>();
-        try (var subdirStream = Files.list(srcDir)) {
-            for (Path sourceset : subdirStream.toList()) {
-                assert Files.isDirectory(sourceset);
-                String sourcesetName = sourceset.getFileName().toString();
-                Matcher sourcesetMatcher = MRJAR_SOURCESET_PATTERN.matcher(sourcesetName);
-                if (sourcesetMatcher.matches()) {
-                    mainVersions.add(Integer.parseInt(sourcesetMatcher.group(1)));
-                }
-            }
-        } catch (IOException e) {
-            throw new UncheckedIOException(e);
-        }
-
-        Collections.sort(mainVersions);
-        List<String> parentSourceSets = new ArrayList<>();
-        parentSourceSets.add(SourceSet.MAIN_SOURCE_SET_NAME);
+        List<Integer> mainVersions = findSourceVersions(project);
+        List<String> mainSourceSets = new ArrayList<>();
+        mainSourceSets.add(SourceSet.MAIN_SOURCE_SET_NAME);
+        List<String> testSourceSets = new ArrayList<>(mainSourceSets);
+        testSourceSets.add(SourceSet.TEST_SOURCE_SET_NAME);
         for (int javaVersion : mainVersions) {
-            String sourcesetName = "main" + javaVersion;
-            addMrjarSourceset(project, javaExtension, sourcesetName, parentSourceSets, javaVersion);
-            parentSourceSets.add(sourcesetName);
+            String mainSourceSetName = SourceSet.MAIN_SOURCE_SET_NAME + javaVersion;
+            SourceSet mainSourceSet = addSourceSet(project, javaExtension, mainSourceSetName, mainSourceSets, javaVersion);
+            configureSourceSetInJar(project, mainSourceSet, javaVersion);
+            mainSourceSets.add(mainSourceSetName);
+            testSourceSets.add(mainSourceSetName);
+
+            String testSourceSetName = SourceSet.TEST_SOURCE_SET_NAME + javaVersion;
+            SourceSet testSourceSet = addSourceSet(project, javaExtension, testSourceSetName, testSourceSets, javaVersion);
+            testSourceSets.add(testSourceSetName);
+            createTestTask(project, testSourceSet, javaVersion, mainSourceSets);
         }
-    }
 
-    private void addMrjarSourceset(
-        Project project,
-        JavaPluginExtension javaExtension,
-        String sourcesetName,
-        List<String> parentSourceSets,
-        int javaVersion
-    ) {
-        SourceSet sourceSet = javaExtension.getSourceSets().maybeCreate(sourcesetName);
-        for (String parentSourceSetName : parentSourceSets) {
-            GradleUtils.extendSourceSet(project, parentSourceSetName, sourcesetName);
-        }
+        configureMrjar(project);
+    }
 
+    private void configureMrjar(Project project) {
         var jarTask = project.getTasks().withType(Jar.class).named(JavaPlugin.JAR_TASK_NAME);
-        jarTask.configure(task -> {
-            task.into("META-INF/versions/" + javaVersion, copySpec -> copySpec.from(sourceSet.getOutput()));
-            task.manifest(manifest -> { manifest.attributes(Map.of("Multi-Release", "true")); });
-        });
+        jarTask.configure(task -> { task.manifest(manifest -> { manifest.attributes(Map.of("Multi-Release", "true")); }); });
 
         project.getTasks().withType(Test.class).named(JavaPlugin.TEST_TASK_NAME).configure(testTask -> {
             testTask.dependsOn(jarTask);
@@ -111,6 +93,19 @@ private void addMrjarSourceset(
             FileCollection testRuntime = sourceSets.getByName(SourceSet.TEST_SOURCE_SET_NAME).getRuntimeClasspath();
             testTask.setClasspath(testRuntime.minus(mainRuntime).plus(project.files(jarTask)));
         });
+    }
+
+    private SourceSet addSourceSet(
+        Project project,
+        JavaPluginExtension javaExtension,
+        String sourceSetName,
+        List<String> parentSourceSets,
+        int javaVersion
+    ) {
+        SourceSet sourceSet = javaExtension.getSourceSets().maybeCreate(sourceSetName);
+        for (String parentSourceSetName : parentSourceSets) {
+            GradleUtils.extendSourceSet(project, parentSourceSetName, sourceSetName);
+        }
 
         project.getTasks().withType(JavaCompile.class).named(sourceSet.getCompileJavaTaskName()).configure(compileTask -> {
             compileTask.getJavaCompiler()
@@ -132,6 +127,55 @@ private void addMrjarSourceset(
         project.getTasks().withType(CheckForbiddenApisTask.class).named(forbiddenApisTaskName).configure(forbiddenApisTask -> {
             forbiddenApisTask.setIgnoreMissingClasses(true);
         });
+
+        return sourceSet;
+    }
+
+    private void configureSourceSetInJar(Project project, SourceSet sourceSet, int javaVersion) {
+        var jarTask = project.getTasks().withType(Jar.class).named(JavaPlugin.JAR_TASK_NAME);
+        jarTask.configure(task -> task.into("META-INF/versions/" + javaVersion, copySpec -> copySpec.from(sourceSet.getOutput())));
+    }
+
+    private void createTestTask(Project project, SourceSet sourceSet, int javaVersion, List<String> mainSourceSets) {
+        var jarTask = project.getTasks().withType(Jar.class).named(JavaPlugin.JAR_TASK_NAME);
+        var testTaskProvider = project.getTasks().register(JavaPlugin.TEST_TASK_NAME + javaVersion, Test.class);
+        testTaskProvider.configure(testTask -> {
+            testTask.dependsOn(jarTask);
+
+            SourceSetContainer sourceSets = GradleUtils.getJavaSourceSets(project);
+            FileCollection testRuntime = sourceSet.getRuntimeClasspath();
+            for (String mainSourceSetName : mainSourceSets) {
+                FileCollection mainRuntime = sourceSets.getByName(mainSourceSetName).getOutput();
+                testRuntime = testRuntime.minus(mainRuntime);
+            }
+            testTask.setClasspath(testRuntime.plus(project.files(jarTask)));
+            testTask.setTestClassesDirs(sourceSet.getOutput().getClassesDirs());
+
+            testTask.getJavaLauncher()
+                .set(javaToolchains.launcherFor(spec -> spec.getLanguageVersion().set(JavaLanguageVersion.of(javaVersion))));
+        });
+
+        project.getTasks().named("check").configure(checkTask -> checkTask.dependsOn(testTaskProvider));
+    }
+
+    private static List<Integer> findSourceVersions(Project project) {
+        var srcDir = project.getProjectDir().toPath().resolve("src");
+        List<Integer> versions = new ArrayList<>();
+        try (var subdirStream = Files.list(srcDir)) {
+            for (Path sourceSetPath : subdirStream.toList()) {
+                assert Files.isDirectory(sourceSetPath);
+                String sourcesetName = sourceSetPath.getFileName().toString();
+                Matcher sourcesetMatcher = MRJAR_SOURCESET_PATTERN.matcher(sourcesetName);
+                if (sourcesetMatcher.matches()) {
+                    versions.add(Integer.parseInt(sourcesetMatcher.group(1)));
+                }
+            }
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+
+        Collections.sort(versions);
+        return versions;
     }
 
     private static void stripPreviewFromFiles(Path compileDir) {

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/testfixtures/TestFixtureDeployment.java

@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.gradle.internal.testfixtures;
+
+import org.gradle.api.Named;
+import org.gradle.api.provider.ListProperty;
+import org.gradle.api.provider.Property;
+
+import java.io.File;
+
+public abstract class TestFixtureDeployment implements Named {
+
+    private final String name;
+
+    public TestFixtureDeployment(String name) {
+        this.name = name;
+    }
+
+    @Override
+    public String getName() {
+        return name;
+    }
+
+    public abstract Property<String> getDockerRegistry();
+
+    public abstract Property<File> getDockerContext();
+
+    public abstract Property<String> getVersion();
+
+    public abstract ListProperty<String> getBaseImages();
+}

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/testfixtures/TestFixturesDeployPlugin.java

@@ -0,0 +1,60 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.gradle.internal.testfixtures;
+
+import org.apache.commons.lang.StringUtils;
+import org.elasticsearch.gradle.Architecture;
+import org.elasticsearch.gradle.internal.docker.DockerBuildTask;
+import org.elasticsearch.gradle.internal.info.BuildParams;
+import org.gradle.api.NamedDomainObjectContainer;
+import org.gradle.api.Plugin;
+import org.gradle.api.Project;
+
+import java.util.Arrays;
+import java.util.List;
+
+public class TestFixturesDeployPlugin implements Plugin<Project> {
+
+    public static final String DEPLOY_FIXTURE_TASK_NAME = "deployFixtureDockerImages";
+    private static String DEFAULT_DOCKER_REGISTRY = "docker.elastic.co/elasticsearch-dev";
+
+    @Override
+    public void apply(Project project) {
+        NamedDomainObjectContainer<TestFixtureDeployment> fixtures = project.container(TestFixtureDeployment.class);
+        project.getExtensions().add("dockerFixtures", fixtures);
+        registerDeployTaskPerFixture(project, fixtures);
+        project.getTasks().register(DEPLOY_FIXTURE_TASK_NAME, task -> task.dependsOn(project.getTasks().withType(DockerBuildTask.class)));
+    }
+
+    private static void registerDeployTaskPerFixture(Project project, NamedDomainObjectContainer<TestFixtureDeployment> fixtures) {
+        fixtures.all(
+            fixture -> project.getTasks()
+                .register("deploy" + StringUtils.capitalize(fixture.getName()) + "DockerImage", DockerBuildTask.class, task -> {
+                    task.getDockerContext().fileValue(fixture.getDockerContext().get());
+                    List<String> baseImages = fixture.getBaseImages().get();
+                    if (baseImages.isEmpty() == false) {
+                        task.setBaseImages(baseImages.toArray(new String[baseImages.size()]));
+                    }
+                    task.setNoCache(BuildParams.isCi());
+                    task.setTags(
+                        new String[] {
+                            resolveTargetDockerRegistry(fixture) + "/" + fixture.getName() + "-fixture:" + fixture.getVersion().get() }
+                    );
+                    task.getPush().set(BuildParams.isCi());
+                    task.getPlatforms().addAll(Arrays.stream(Architecture.values()).map(a -> a.dockerPlatform).toList());
+                    task.setGroup("Deploy TestFixtures");
+                    task.setDescription("Deploys the " + fixture.getName() + " test fixture");
+                })
+        );
+    }
+
+    private static String resolveTargetDockerRegistry(TestFixtureDeployment fixture) {
+        return fixture.getDockerRegistry().getOrElse(DEFAULT_DOCKER_REGISTRY);
+    }
+}

docs/changelog/105501.yaml

@@ -0,0 +1,5 @@
+pr: 105501
+summary: Support non-keyword dimensions as routing fields in TSDB
+area: TSDB
+type: enhancement
+issues: []

docs/changelog/106065.yaml

@@ -0,0 +1,6 @@
+pr: 106065
+summary: "ESQL: Values aggregation function"
+area: ES|QL
+type: feature
+issues:
+ - 103600

docs/changelog/106396.yaml

@@ -0,0 +1,6 @@
+pr: 106396
+summary: "Check preTags and postTags params for empty values"
+area: Highlighting
+type: bug
+issues:
+  - 69009

docs/changelog/106563.yaml

@@ -0,0 +1,5 @@
+pr: 106563
+summary: Improve short-circuiting downsample execution
+area: TSDB
+type: enhancement
+issues: []

docs/changelog/106574.yaml

@@ -0,0 +1,6 @@
+pr: 106574
+summary: Fix `_reset` API when called with `force=true` on a failed transform
+area: Transform
+type: bug
+issues:
+ - 106573

docs/changelog/106575.yaml

@@ -0,0 +1,5 @@
+pr: 106575
+summary: Unable to retrieve multiple stored field values
+area: "Search"
+type: bug
+issues: []

docs/changelog/97072.yaml

@@ -0,0 +1,5 @@
+pr: 97072
+summary: Log when update AffixSetting using addAffixMapUpdateConsumer
+area: Infra/Logging
+type: bug
+issues: []

docs/reference/data-streams/lifecycle/apis/get-lifecycle.asciidoc

@@ -130,14 +130,18 @@ The response will look like the following:
       "name": "my-data-stream-1",
       "lifecycle": {
         "enabled": true,
-        "data_retention": "7d"
+        "data_retention": "7d",
+        "effective_retention": "7d",
+        "retention_determined_by": "data_stream_configuration"
       }
     },
     {
       "name": "my-data-stream-2",
       "lifecycle": {
         "enabled": true,
-        "data_retention": "7d"
+        "data_retention": "7d",
+        "effective_retention": "7d",
+        "retention_determined_by": "data_stream_configuration"
       }
     }
   ]

docs/reference/data-streams/lifecycle/tutorial-manage-existing-data-stream.asciidoc

@@ -74,7 +74,9 @@ The response will look like:
       "generation_time": "6.84s",                       <9>
       "lifecycle": {
         "enabled": true,
-        "data_retention": "30d"                         <10>
+        "data_retention": "30d",
+        "effective_retention": "30d"                    <10>
+        "retention_determined_by": "data_stream_configuration"
       }
     }
   }