Merge branch 'main' into esql-binary-comparators-part-1

.buildkite/pipelines/intake.yml

@@ -48,7 +48,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["7.17.18", "8.12.1", "8.13.0"]
+            BWC_VERSION: ["7.17.19", "8.12.2", "8.13.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.buildkite/pipelines/periodic-packaging.yml

@@ -1121,6 +1121,22 @@ steps:
         env:
           BWC_VERSION: 7.17.18
 
+      - label: "{{matrix.image}} / 7.17.19 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v7.17.19
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            image:
+              - rocky-8
+              - ubuntu-2004
+        agents:
+          provider: gcp
+          image: family/elasticsearch-{{matrix.image}}
+          machineType: custom-16-32768
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 7.17.19
+
       - label: "{{matrix.image}} / 8.0.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.0.0
         timeout_in_minutes: 300
@@ -1841,6 +1857,22 @@ steps:
         env:
           BWC_VERSION: 8.12.1
 
+      - label: "{{matrix.image}} / 8.12.2 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.12.2
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            image:
+              - rocky-8
+              - ubuntu-2004
+        agents:
+          provider: gcp
+          image: family/elasticsearch-{{matrix.image}}
+          machineType: custom-16-32768
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 8.12.2
+
       - label: "{{matrix.image}} / 8.13.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.13.0
         timeout_in_minutes: 300

.buildkite/pipelines/periodic.yml

@@ -682,6 +682,16 @@ steps:
           buildDirectory: /dev/shm/bk
         env:
           BWC_VERSION: 7.17.18
+      - label: 7.17.19 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v7.17.19#bwcTest
+        timeout_in_minutes: 300
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2004
+          machineType: n1-standard-32
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 7.17.19
       - label: 8.0.0 / bwc
         command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.0.0#bwcTest
         timeout_in_minutes: 300
@@ -1132,6 +1142,16 @@ steps:
           buildDirectory: /dev/shm/bk
         env:
           BWC_VERSION: 8.12.1
+      - label: 8.12.2 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.12.2#bwcTest
+        timeout_in_minutes: 300
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2004
+          machineType: n1-standard-32
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 8.12.2
       - label: 8.13.0 / bwc
         command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.13.0#bwcTest
         timeout_in_minutes: 300

.buildkite/pull-requests.json

@@ -11,7 +11,7 @@
       "set_commit_status": false,
       "build_on_commit": true,
       "build_on_comment": true,
-      "trigger_comment_regex": "(run\\W+elasticsearch-ci.+)|(^\\s*(buildkite\\s*)?test\\s+this(\\s+please)?)",
+      "trigger_comment_regex": "(run\\W+elasticsearch-ci.+)|(^\\s*((buildkite|@elastic(search)?machine)\\s*)?test\\s+this(\\s+please)?)",
       "cancel_intermediate_builds": true,
       "cancel_intermediate_builds_on_comment": false
     },

.buildkite/scripts/pull-request/__snapshots__/pipeline.test.ts.snap

@@ -201,3 +201,111 @@ exports[`generatePipelines should generate correct pipeline when using a trigger
   },
 ]
 `;
+
+exports[`generatePipelines should generate correct pipelines with a non-docs change and @elasticmachine 1`] = `
+[
+  {
+    "name": "bwc-snapshots",
+    "pipeline": {
+      "steps": [
+        {
+          "group": "bwc-snapshots",
+          "steps": [
+            {
+              "agents": {
+                "buildDirectory": "/dev/shm/bk",
+                "image": "family/elasticsearch-ubuntu-2004",
+                "machineType": "custom-32-98304",
+                "provider": "gcp",
+              },
+              "command": ".ci/scripts/run-gradle.sh -Dignore.tests.seed v{{matrix.BWC_VERSION}}#bwcTest",
+              "env": {
+                "BWC_VERSION": "{{matrix.BWC_VERSION}}",
+              },
+              "label": "{{matrix.BWC_VERSION}} / bwc-snapshots",
+              "matrix": {
+                "setup": {
+                  "BWC_VERSION": [
+                    "7.17.14",
+                    "8.10.3",
+                    "8.11.0",
+                  ],
+                },
+              },
+              "timeout_in_minutes": 300,
+            },
+          ],
+        },
+      ],
+    },
+  },
+  {
+    "name": "using-defaults",
+    "pipeline": {
+      "env": {
+        "CUSTOM_ENV_VAR": "value",
+      },
+      "steps": [
+        {
+          "command": "echo 'hello world'",
+          "label": "test-step",
+        },
+      ],
+    },
+  },
+]
+`;
+
+exports[`generatePipelines should generate correct pipelines with a non-docs change and @elasticsearchmachine 1`] = `
+[
+  {
+    "name": "bwc-snapshots",
+    "pipeline": {
+      "steps": [
+        {
+          "group": "bwc-snapshots",
+          "steps": [
+            {
+              "agents": {
+                "buildDirectory": "/dev/shm/bk",
+                "image": "family/elasticsearch-ubuntu-2004",
+                "machineType": "custom-32-98304",
+                "provider": "gcp",
+              },
+              "command": ".ci/scripts/run-gradle.sh -Dignore.tests.seed v{{matrix.BWC_VERSION}}#bwcTest",
+              "env": {
+                "BWC_VERSION": "{{matrix.BWC_VERSION}}",
+              },
+              "label": "{{matrix.BWC_VERSION}} / bwc-snapshots",
+              "matrix": {
+                "setup": {
+                  "BWC_VERSION": [
+                    "7.17.14",
+                    "8.10.3",
+                    "8.11.0",
+                  ],
+                },
+              },
+              "timeout_in_minutes": 300,
+            },
+          ],
+        },
+      ],
+    },
+  },
+  {
+    "name": "using-defaults",
+    "pipeline": {
+      "env": {
+        "CUSTOM_ENV_VAR": "value",
+      },
+      "steps": [
+        {
+          "command": "echo 'hello world'",
+          "label": "test-step",
+        },
+      ],
+    },
+  },
+]
+`;

.buildkite/scripts/pull-request/pipeline.test.ts

@@ -13,11 +13,11 @@ describe("generatePipelines", () => {
   });
 
   // Helper for testing pipeline generations that should be the same when using the overall ci trigger comment "buildkite test this"
-  const testWithTriggerCheck = (directory: string, changedFiles?: string[]) => {
+  const testWithTriggerCheck = (directory: string, changedFiles?: string[], comment = "buildkite test this") => {
     const pipelines = generatePipelines(directory, changedFiles);
     expect(pipelines).toMatchSnapshot();
 
-    process.env["GITHUB_PR_TRIGGER_COMMENT"] = "buildkite test this";
+    process.env["GITHUB_PR_TRIGGER_COMMENT"] = comment;
     const pipelinesWithTriggerComment = generatePipelines(directory, changedFiles);
     expect(pipelinesWithTriggerComment).toEqual(pipelines);
   };
@@ -42,4 +42,20 @@ describe("generatePipelines", () => {
     const pipelines = generatePipelines(`${import.meta.dir}/mocks/pipelines`, ["build.gradle"]);
     expect(pipelines).toMatchSnapshot();
   });
+
+  test("should generate correct pipelines with a non-docs change and @elasticmachine", () => {
+    testWithTriggerCheck(
+      `${import.meta.dir}/mocks/pipelines`,
+      ["build.gradle", "docs/README.asciidoc"],
+      "@elasticmachine test this please"
+    );
+  });
+
+  test("should generate correct pipelines with a non-docs change and @elasticsearchmachine", () => {
+    testWithTriggerCheck(
+      `${import.meta.dir}/mocks/pipelines`,
+      ["build.gradle", "docs/README.asciidoc"],
+      "@elasticsearchmachine test this please"
+    );
+  });
 });

.buildkite/scripts/pull-request/pipeline.ts

@@ -148,7 +148,9 @@ export const generatePipelines = (
   // However, if we're using the overall CI trigger "[buildkite] test this [please]", we should use the regular filters above
   if (
     process.env["GITHUB_PR_TRIGGER_COMMENT"] &&
-    !process.env["GITHUB_PR_TRIGGER_COMMENT"].match(/^\s*(buildkite\s*)?test\s+this(\s+please)?/i)
+    !process.env["GITHUB_PR_TRIGGER_COMMENT"].match(
+      /^\s*((@elastic(search)?machine|buildkite)\s*)?test\s+this(\s+please)?/i
+    )
   ) {
     filters = [triggerCommentCheck];
   }

.ci/bwcVersions

@@ -67,6 +67,7 @@ BWC_VERSION:
   - "7.17.16"
   - "7.17.17"
   - "7.17.18"
+  - "7.17.19"
   - "8.0.0"
   - "8.0.1"
   - "8.1.0"
@@ -112,4 +113,5 @@ BWC_VERSION:
   - "8.11.4"
   - "8.12.0"
   - "8.12.1"
+  - "8.12.2"
   - "8.13.0"

.ci/snapshotBwcVersions

@@ -1,4 +1,4 @@
 BWC_VERSION:
-  - "7.17.18"
-  - "8.12.1"
+  - "7.17.19"
+  - "8.12.2"
   - "8.13.0"

build-tools-internal/build.gradle

@@ -143,6 +143,10 @@ gradlePlugin {
       id = 'elasticsearch.mrjar'
       implementationClass = 'org.elasticsearch.gradle.internal.MrjarPlugin'
     }
+    embeddedProvider {
+      id = 'elasticsearch.embedded-providers'
+      implementationClass = 'org.elasticsearch.gradle.internal.EmbeddedProviderPlugin'
+    }
     releaseTools {
       id = 'elasticsearch.release-tools'
       implementationClass = 'org.elasticsearch.gradle.internal.release.ReleaseToolsPlugin'

build-tools-internal/src/main/groovy/elasticsearch.ide.gradle

@@ -114,7 +114,7 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') {
     description = 'Builds artifacts needed as dependency for IDE modules'
     dependsOn([':client:rest-high-level:shadowJar',
       ':plugins:repository-hdfs:hadoop-client-api:shadowJar',
-      ':libs:elasticsearch-x-content:generateProviderImpl',
+      ':libs:elasticsearch-x-content:generateImplProviderImpl',
       ':x-pack:plugin:esql:compute:ann:jar',
       ':x-pack:plugin:esql:compute:gen:jar',
       ':server:generateModulesList',

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaBasePlugin.java

@@ -26,14 +26,26 @@
 import org.gradle.api.tasks.compile.CompileOptions;
 import org.gradle.api.tasks.compile.GroovyCompile;
 import org.gradle.api.tasks.compile.JavaCompile;
+import org.gradle.jvm.toolchain.JavaLanguageVersion;
+import org.gradle.jvm.toolchain.JavaToolchainService;
 
 import java.util.List;
 
+import javax.inject.Inject;
+
 /**
  * A wrapper around Gradle's Java Base plugin that applies our
  * common configuration for production code.
  */
 public class ElasticsearchJavaBasePlugin implements Plugin<Project> {
+
+    private final JavaToolchainService javaToolchains;
+
+    @Inject
+    ElasticsearchJavaBasePlugin(JavaToolchainService javaToolchains) {
+        this.javaToolchains = javaToolchains;
+    }
+
     @Override
     public void apply(Project project) {
         // make sure the global build info plugin is applied to the root project
@@ -103,7 +115,7 @@ private static void disableTransitiveDependenciesForSourceSet(Project project, S
     /**
      * Adds compiler settings to the project
      */
-    public static void configureCompile(Project project) {
+    public void configureCompile(Project project) {
         project.getExtensions().getExtraProperties().set("compactProfile", "full");
         JavaPluginExtension java = project.getExtensions().getByType(JavaPluginExtension.class);
         if (BuildParams.getJavaToolChainSpec().isPresent()) {
@@ -112,6 +124,10 @@ public static void configureCompile(Project project) {
         java.setSourceCompatibility(BuildParams.getMinimumRuntimeVersion());
         java.setTargetCompatibility(BuildParams.getMinimumRuntimeVersion());
         project.getTasks().withType(JavaCompile.class).configureEach(compileTask -> {
+            compileTask.getJavaCompiler().set(javaToolchains.compilerFor(spec -> {
+                spec.getLanguageVersion().set(JavaLanguageVersion.of(BuildParams.getMinimumRuntimeVersion().getMajorVersion()));
+            }));
+
             CompileOptions compileOptions = compileTask.getOptions();
             /*
              * -path because gradle will send in paths that don't always exist.