Merge branch 'main' of https://github.com/elastic/elasticsearch into …

…trained-models-metadata

.backportrc.json

@@ -1,9 +1,9 @@
 {
   "upstream" : "elastic/elasticsearch",
-  "targetBranchChoices" : [ "main", "8.13", "8.12", "8.11", "8.10", "8.9", "8.8", "8.7", "8.6", "8.5", "8.4", "8.3", "8.2", "8.1", "8.0", "7.17", "6.8" ],
+  "targetBranchChoices" : [ "main", "8.14", "8.13", "8.12", "8.11", "8.10", "8.9", "8.8", "8.7", "8.6", "8.5", "8.4", "8.3", "8.2", "8.1", "8.0", "7.17", "6.8" ],
   "targetPRLabels" : [ "backport" ],
   "branchLabelMapping" : {
-    "^v8.14.0$" : "main",
+    "^v8.15.0$" : "main",
     "^v(\\d+).(\\d+).\\d+(?:-(?:alpha|beta|rc)\\d+)?$" : "$1.$2"
   }
 }

.buildkite/pipelines/intake.yml

@@ -48,7 +48,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["7.17.21", "8.13.3", "8.14.0"]
+            BWC_VERSION: ["7.17.21", "8.13.3", "8.14.0", "8.15.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.buildkite/pipelines/lucene-snapshot/run-tests.yml

@@ -32,6 +32,14 @@ steps:
       image: family/elasticsearch-ubuntu-2004
       machineType: custom-32-98304
       buildDirectory: /dev/shm/bk
+  - label: part4
+    command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart4
+    timeout_in_minutes: 300
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2004
+      machineType: custom-32-98304
+      buildDirectory: /dev/shm/bk
   - group: bwc-snapshots
     steps:
       - label: "{{matrix.BWC_VERSION}} / bwc-snapshots"

.buildkite/pipelines/periodic-packaging.yml

@@ -561,6 +561,22 @@ steps:
         env:
           BWC_VERSION: 8.14.0
 
+      - label: "{{matrix.image}} / 8.15.0 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.15.0
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            image:
+              - rocky-8
+              - ubuntu-2004
+        agents:
+          provider: gcp
+          image: family/elasticsearch-{{matrix.image}}
+          machineType: custom-16-32768
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 8.15.0
+
   - group: packaging-tests-windows
     steps:
       - label: "{{matrix.image}} / packaging-tests-windows"

.buildkite/pipelines/periodic-platform-support.yml

@@ -47,6 +47,7 @@ steps:
               - checkPart1
               - checkPart2
               - checkPart3
+              - checkPart4
               - checkRestCompat
         agents:
           provider: gcp
@@ -70,6 +71,7 @@ steps:
               - checkPart1
               - checkPart2
               - checkPart3
+              - checkPart4
               - checkRestCompat
         agents:
           provider: aws

.buildkite/pipelines/periodic.template.yml

@@ -49,6 +49,7 @@ steps:
               - checkPart1
               - checkPart2
               - checkPart3
+              - checkPart4
               - checkRestCompat
         agents:
           provider: gcp
@@ -89,6 +90,7 @@ steps:
               - checkPart1
               - checkPart2
               - checkPart3
+              - checkPart4
               - checkRestCompat
         agents:
           provider: gcp

.buildkite/pipelines/periodic.yml

@@ -332,6 +332,16 @@ steps:
           buildDirectory: /dev/shm/bk
         env:
           BWC_VERSION: 8.14.0
+      - label: 8.15.0 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.15.0#bwcTest
+        timeout_in_minutes: 300
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2004
+          machineType: n1-standard-32
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 8.15.0
   - label: concurrent-search-tests
     command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dtests.jvm.argline=-Des.concurrent_search=true -Des.concurrent_search=true functionalTests
     timeout_in_minutes: 420
@@ -380,6 +390,7 @@ steps:
               - checkPart1
               - checkPart2
               - checkPart3
+              - checkPart4
               - checkRestCompat
         agents:
           provider: gcp
@@ -396,7 +407,7 @@ steps:
           setup:
             ES_RUNTIME_JAVA:
               - openjdk17
-            BWC_VERSION: ["7.17.21", "8.13.3", "8.14.0"]
+            BWC_VERSION: ["7.17.21", "8.13.3", "8.14.0", "8.15.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
@@ -420,6 +431,7 @@ steps:
               - checkPart1
               - checkPart2
               - checkPart3
+              - checkPart4
               - checkRestCompat
         agents:
           provider: gcp
@@ -438,7 +450,7 @@ steps:
               - graalvm-ce17
               - openjdk17
               - openjdk21
-            BWC_VERSION: ["7.17.21", "8.13.3", "8.14.0"]
+            BWC_VERSION: ["7.17.21", "8.13.3", "8.14.0", "8.15.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.ci/bwcVersions

@@ -32,3 +32,4 @@ BWC_VERSION:
   - "8.12.2"
   - "8.13.3"
   - "8.14.0"
+  - "8.15.0"

.ci/snapshotBwcVersions

@@ -2,3 +2,4 @@ BWC_VERSION:
   - "7.17.21"
   - "8.13.3"
   - "8.14.0"
+  - "8.15.0"

branches.json

@@ -4,6 +4,9 @@
     {
       "branch": "main"
     },
+    {
+      "branch": "8.14"
+    },
     {
       "branch": "8.13"
     },

build-tools-internal/muted-tests.yml

@@ -0,0 +1,24 @@
+tests:
+# Examples:
+#
+#  Mute a single test case in a YAML test suite:
+#  - class: org.elasticsearch.analysis.common.CommonAnalysisClientYamlTestSuiteIT
+#    method: test {yaml=analysis-common/30_tokenizers/letter}
+#    issue: https://github.com/elastic/elasticsearch/...
+#
+#  Mute several methods of a Java test:
+#  - class: org.elasticsearch.common.CharArraysTests
+#    methods:
+#      - testCharsBeginsWith
+#      - testCharsToBytes
+#      - testConstantTimeEquals
+#    issue: https://github.com/elastic/elasticsearch/...
+#
+#  Mute an entire test class:
+#  - class: org.elasticsearch.common.unit.TimeValueTests
+#    issue: https://github.com/elastic/elasticsearch/...
+#
+#  Mute a single method in a test class:
+#  - class: org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToIPTests
+#    method: testCrankyEvaluateBlockWithoutNulls
+#    issue: https://github.com/elastic/elasticsearch/...

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaBasePlugin.java

@@ -12,6 +12,7 @@
 import org.elasticsearch.gradle.internal.conventions.precommit.PrecommitTaskPlugin;
 import org.elasticsearch.gradle.internal.info.BuildParams;
 import org.elasticsearch.gradle.internal.info.GlobalBuildInfoPlugin;
+import org.elasticsearch.gradle.internal.test.MutedTestPlugin;
 import org.elasticsearch.gradle.internal.test.TestUtil;
 import org.elasticsearch.gradle.test.SystemPropertyCommandLineArgumentProvider;
 import org.elasticsearch.gradle.util.GradleUtils;
@@ -62,6 +63,7 @@ public void apply(Project project) {
         project.getPluginManager().apply(RepositoriesSetupPlugin.class);
         project.getPluginManager().apply(ElasticsearchTestBasePlugin.class);
         project.getPluginManager().apply(PrecommitTaskPlugin.class);
+        project.getPluginManager().apply(MutedTestPlugin.class);
 
         configureConfigurations(project);
         configureCompile(project);

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/MutedTestPlugin.java

@@ -0,0 +1,41 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.gradle.internal.test;
+
+import org.elasticsearch.gradle.internal.conventions.util.Util;
+import org.elasticsearch.gradle.internal.info.BuildParams;
+import org.gradle.api.Plugin;
+import org.gradle.api.Project;
+import org.gradle.api.provider.Provider;
+import org.gradle.api.tasks.testing.Test;
+
+import java.io.File;
+
+public class MutedTestPlugin implements Plugin<Project> {
+    @Override
+    public void apply(Project project) {
+        File infoPath = new File(Util.locateElasticsearchWorkspace(project.getGradle()), "build-tools-internal");
+        Provider<MutedTestsBuildService> mutedTestsProvider = project.getGradle()
+            .getSharedServices()
+            .registerIfAbsent("mutedTests", MutedTestsBuildService.class, spec -> {
+                spec.getParameters().getInfoPath().set(infoPath);
+            });
+
+        project.getTasks().withType(Test.class).configureEach(test -> {
+            test.filter(filter -> {
+                for (String exclude : mutedTestsProvider.get().getExcludePatterns()) {
+                    filter.excludeTestsMatching(exclude);
+                }
+
+                // Don't fail when all tests are ignored when running in CI
+                filter.setFailOnNoMatchingTests(BuildParams.isCi() == false);
+            });
+        });
+    }
+}

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/MutedTestsBuildService.java

@@ -0,0 +1,141 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.gradle.internal.test;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
+
+import org.gradle.api.file.RegularFileProperty;
+import org.gradle.api.services.BuildService;
+import org.gradle.api.services.BuildServiceParameters;
+
+import java.io.BufferedInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.UncheckedIOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+public abstract class MutedTestsBuildService implements BuildService<MutedTestsBuildService.Params> {
+    private final List<String> excludePatterns;
+
+    public MutedTestsBuildService() {
+        File infoPath = getParameters().getInfoPath().get().getAsFile();
+        File mutedTestsFile = new File(infoPath, "muted-tests.yml");
+        try (InputStream is = new BufferedInputStream(new FileInputStream(mutedTestsFile))) {
+            ObjectMapper objectMapper = new ObjectMapper(new YAMLFactory());
+            List<MutedTest> mutedTests = objectMapper.readValue(is, MutedTests.class).getTests();
+            excludePatterns = buildExcludePatterns(mutedTests == null ? Collections.emptyList() : mutedTests);
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+
+    public List<String> getExcludePatterns() {
+        return excludePatterns;
+    }
+
+    private static List<String> buildExcludePatterns(List<MutedTest> mutedTests) {
+        List<String> excludes = new ArrayList<>();
+        if (mutedTests.isEmpty() == false) {
+            for (MutedTestsBuildService.MutedTest mutedTest : mutedTests) {
+                if (mutedTest.getClassName() != null && mutedTest.getMethods().isEmpty() == false) {
+                    for (String method : mutedTest.getMethods()) {
+                        // Tests that use the randomized runner and parameters end up looking like this:
+                        // test {yaml=analysis-common/30_tokenizers/letter}
+                        // We need to detect this and handle them a little bit different than non-parameterized tests, because of some
+                        // quirks in the randomized runner
+                        int index = method.indexOf(" {");
+                        String methodWithoutParams = index >= 0 ? method.substring(0, index) : method;
+                        String paramString = index >= 0 ? method.substring(index) : null;
+
+                        excludes.add(mutedTest.getClassName() + "." + method);
+
+                        if (paramString != null) {
+                            // Because of randomized runner quirks, we need skip the test method by itself whenever we want to skip a test
+                            // that has parameters
+                            // This is because the runner has *two* separate checks that can cause the test to end up getting executed, so
+                            // we need filters that cover both checks
+                            excludes.add(mutedTest.getClassName() + "." + methodWithoutParams);
+                        } else {
+                            // We need to add the following, in case we're skipping an entire class of parameterized tests
+                            excludes.add(mutedTest.getClassName() + "." + method + " *");
+                        }
+                    }
+                } else if (mutedTest.getClassName() != null) {
+                    excludes.add(mutedTest.getClassName() + ".*");
+                }
+            }
+        }
+
+        return excludes;
+    }
+
+    public interface Params extends BuildServiceParameters {
+        RegularFileProperty getInfoPath();
+    }
+
+    public static class MutedTest {
+        private final String className;
+        private final String method;
+        private final List<String> methods;
+        private final String issue;
+
+        @JsonCreator
+        public MutedTest(
+            @JsonProperty("class") String className,
+            @JsonProperty("method") String method,
+            @JsonProperty("methods") List<String> methods,
+            @JsonProperty("issue") String issue
+        ) {
+            this.className = className;
+            this.method = method;
+            this.methods = methods;
+            this.issue = issue;
+        }
+
+        public List<String> getMethods() {
+            List<String> allMethods = new ArrayList<>();
+            if (methods != null) {
+                allMethods.addAll(methods);
+            }
+            if (method != null) {
+                allMethods.add(method);
+            }
+
+            return allMethods;
+        }
+
+        public String getClassName() {
+            return className;
+        }
+
+        public String getIssue() {
+            return issue;
+        }
+    }
+
+    private static class MutedTests {
+        private final List<MutedTest> tests;
+
+        @JsonCreator
+        MutedTests(@JsonProperty("tests") List<MutedTest> tests) {
+            this.tests = tests;
+        }
+
+        public List<MutedTest> getTests() {
+            return tests;
+        }
+    }
+}