Merge upstream main

.buildkite/pipelines/dra-workflow.yml

@@ -7,3 +7,13 @@ steps:
       image: family/elasticsearch-ubuntu-2204
       machineType: custom-32-98304
       buildDirectory: /dev/shm/bk
+  - wait
+  # The hadoop build depends on the ES artifact
+  # So let's trigger the hadoop build any time we build a new staging artifact
+  - trigger: elasticsearch-hadoop-dra-workflow
+    async: true
+    build:
+      branch: "${BUILDKITE_BRANCH}"
+      env:
+        DRA_WORKFLOW: staging
+    if: build.env('DRA_WORKFLOW') == 'staging'

.buildkite/pipelines/intake.yml

@@ -48,7 +48,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["7.17.17", "8.11.4", "8.12.0", "8.13.0"]
+            BWC_VERSION: ["7.17.17", "8.11.5", "8.12.0", "8.13.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.buildkite/pipelines/periodic-packaging.yml

@@ -1793,6 +1793,22 @@ steps:
         env:
           BWC_VERSION: 8.11.4
 
+      - label: "{{matrix.image}} / 8.11.5 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.11.5
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            image:
+              - rocky-8
+              - ubuntu-2004
+        agents:
+          provider: gcp
+          image: family/elasticsearch-{{matrix.image}}
+          machineType: custom-16-32768
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 8.11.5
+
       - label: "{{matrix.image}} / 8.12.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.12.0
         timeout_in_minutes: 300

.buildkite/pipelines/periodic-platform-support.yml

@@ -89,6 +89,7 @@ steps:
           setup:
             image:
               - amazonlinux-2023
+              - amazonlinux-2
         agents:
           provider: aws
           imagePrefix: elasticsearch-{{matrix.image}}

.buildkite/pipelines/periodic.template.yml

@@ -2,7 +2,7 @@ steps:
   - group: bwc
     steps: $BWC_STEPS
   - label: concurrent-search-tests
-    command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dtests.jvm.argline=-Des.concurrent_search=true -Des.concurrent_search=true check
+    command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dtests.jvm.argline=-Des.concurrent_search=true -Des.concurrent_search=true functionalTests
     timeout_in_minutes: 420
     agents:
       provider: gcp
@@ -97,7 +97,7 @@ steps:
       diskSizeGb: 350
       machineType: custom-32-98304
   - label: single-processor-node-tests
-    command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dtests.configure_test_clusters_with_one_processor=true check
+    command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dtests.configure_test_clusters_with_one_processor=true functionalTests
     timeout_in_minutes: 420
     agents:
       provider: gcp
@@ -182,13 +182,13 @@ steps:
       machineType: n2-standard-8
       buildDirectory: /dev/shm/bk
     if: build.branch == "main" || build.branch == "7.17"
-  - label: Check branch consistency
+  - label: check-branch-consistency
     command: .ci/scripts/run-gradle.sh branchConsistency
     timeout_in_minutes: 15
     agents:
       provider: gcp
       image: family/elasticsearch-ubuntu-2004
       machineType: n2-standard-2
-  - label: Check branch protection rules
+  - label: check-branch-protection-rules
     command: .buildkite/scripts/branch-protection.sh
     timeout_in_minutes: 5

.buildkite/pipelines/periodic.yml

@@ -1102,6 +1102,16 @@ steps:
           buildDirectory: /dev/shm/bk
         env:
           BWC_VERSION: 8.11.4
+      - label: 8.11.5 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.11.5#bwcTest
+        timeout_in_minutes: 300
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2004
+          machineType: n1-standard-32
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 8.11.5
       - label: 8.12.0 / bwc
         command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.12.0#bwcTest
         timeout_in_minutes: 300
@@ -1123,7 +1133,7 @@ steps:
         env:
           BWC_VERSION: 8.13.0
   - label: concurrent-search-tests
-    command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dtests.jvm.argline=-Des.concurrent_search=true -Des.concurrent_search=true check
+    command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dtests.jvm.argline=-Des.concurrent_search=true -Des.concurrent_search=true functionalTests
     timeout_in_minutes: 420
     agents:
       provider: gcp
@@ -1218,7 +1228,7 @@ steps:
       diskSizeGb: 350
       machineType: custom-32-98304
   - label: single-processor-node-tests
-    command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dtests.configure_test_clusters_with_one_processor=true check
+    command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dtests.configure_test_clusters_with_one_processor=true functionalTests
     timeout_in_minutes: 420
     agents:
       provider: gcp
@@ -1303,13 +1313,13 @@ steps:
       machineType: n2-standard-8
       buildDirectory: /dev/shm/bk
     if: build.branch == "main" || build.branch == "7.17"
-  - label: Check branch consistency
+  - label: check-branch-consistency
     command: .ci/scripts/run-gradle.sh branchConsistency
     timeout_in_minutes: 15
     agents:
       provider: gcp
       image: family/elasticsearch-ubuntu-2004
       machineType: n2-standard-2
-  - label: Check branch protection rules
+  - label: check-branch-protection-rules
     command: .buildkite/scripts/branch-protection.sh
     timeout_in_minutes: 5

.buildkite/scripts/encryption-at-rest.sh

@@ -22,4 +22,4 @@ touch .output.log
 rm -Rf "$WORKSPACE"
 ln -s "$PWD" "$WORKSPACE"
 
-.ci/scripts/run-gradle.sh -Dbwc.checkout.align=true check
+.ci/scripts/run-gradle.sh -Dbwc.checkout.align=true functionalTests

.buildkite/scripts/release-tests.sh

@@ -20,4 +20,4 @@ curl --fail -o "${ML_IVY_REPO}/maven/org/elasticsearch/ml/ml-cpp/${ES_VERSION}/m
 curl --fail -o "${ML_IVY_REPO}/maven/org/elasticsearch/ml/ml-cpp/${ES_VERSION}/ml-cpp-${ES_VERSION}.zip" https://artifacts-snapshot.elastic.co/ml-cpp/${ES_VERSION}-SNAPSHOT/downloads/ml-cpp/ml-cpp-${ES_VERSION}-SNAPSHOT.zip
 
 .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dbuild.snapshot=false -Dbuild.ml_cpp.repo=file://${ML_IVY_REPO} \
-  -Dtests.jvm.argline=-Dbuild.snapshot=false -Dlicense.key=${WORKSPACE}/x-pack/license-tools/src/test/resources/public.key -Dbuild.id=deadbeef build
+  -Dtests.jvm.argline=-Dbuild.snapshot=false -Dlicense.key=${WORKSPACE}/x-pack/license-tools/src/test/resources/public.key -Dbuild.id=deadbeef assemble functionalTests

.ci/bwcVersions

@@ -109,5 +109,6 @@ BWC_VERSION:
   - "8.11.2"
   - "8.11.3"
   - "8.11.4"
+  - "8.11.5"
   - "8.12.0"
   - "8.13.0"

.ci/snapshotBwcVersions

@@ -1,5 +1,5 @@
 BWC_VERSION:
   - "7.17.17"
-  - "8.11.4"
+  - "8.11.5"
   - "8.12.0"
   - "8.13.0"

build-tools-internal/build.gradle

@@ -255,7 +255,7 @@ dependencies {
     // ensuring brought asm version brought in by spock is up-to-date
     testImplementation buildLibs.asm
     integTestImplementation buildLibs.asm
-    integTestImplementation('org.ow2.asm:asm:9.5')
+    integTestImplementation('org.ow2.asm:asm:9.6')
     api("org.yaml:snakeyaml") {
       version { strictly(versions.snakeyaml) }
     }

build-tools-internal/src/main/groovy/elasticsearch.build-scan.gradle

@@ -98,7 +98,8 @@ buildScan {
       def branch = System.getenv('BUILDKITE_PULL_REQUEST_BASE_BRANCH') ?: System.getenv('BUILDKITE_BRANCH')
       def repoMatcher = System.getenv('BUILDKITE_REPO') =~ /(https:\/\/github\.com\/|git@github\.com:)(\S+)\.git/
       def repository = repoMatcher.matches() ? repoMatcher.group(2) : "<unknown>"
-      def jobName = (System.getenv('BUILDKITE_LABEL') ?: '').replaceAll(/[^a-zA-Z0-9_\-]+/, ' ').trim().replaceAll(' ', '_').toLowerCase()
+      def jobLabel = System.getenv('BUILDKITE_LABEL') ?: ''
+      def jobName = safeName(jobLabel)
 
       tag 'CI'
       link 'CI Build', "${buildKiteUrl}#${System.getenv('BUILDKITE_JOB_ID')}"
@@ -111,6 +112,11 @@ buildScan {
 
       value 'Job Name', jobName
       tag jobName
+      if (jobLabel.contains("/")) {
+        jobLabel.split("/").collect {safeName(it) }.each {matrix ->
+          tag matrix
+        }
+      }
 
       if (branch) {
         tag branch
@@ -161,3 +167,7 @@ buildScan {
     }
   }
 }
+
+static def safeName(String string) {
+  return string.replaceAll(/[^a-zA-Z0-9_\-\.]+/, ' ').trim().replaceAll(' ', '_').toLowerCase()
+}

build-tools-internal/src/main/groovy/elasticsearch.fips.gradle

@@ -79,7 +79,6 @@ if (BuildParams.inFipsJvm) {
           // with no x-pack. Tests having security explicitly enabled/disabled will override this setting
           setting 'xpack.security.enabled', 'false'
           setting 'xpack.security.fips_mode.enabled', 'true'
-          setting 'xpack.security.fips_mode.required_providers', '["BCFIPS", "BCJSSE"]'
           setting 'xpack.license.self_generated.type', 'trial'
           setting 'xpack.security.authc.password_hashing.algorithm', 'pbkdf2_stretch'
           keystorePassword 'keystore-password'

build-tools-internal/src/main/groovy/elasticsearch.runtime-jdk-provision.gradle

@@ -47,15 +47,6 @@ configure(allprojects) {
             }
         }
     }
-
-    project.plugins.withType(RestTestBasePlugin) {
-      tasks.withType(StandaloneRestIntegTestTask).configureEach {
-        if (BuildParams.getIsRuntimeJavaHomeSet() == false) {
-          nonInputProperties.systemProperty("tests.runtime.java", "${-> launcher.map { it.metadata.installationPath.asFile.path }.get()}")
-        }
-      }
-    }
-
     project.plugins.withType(ThirdPartyAuditPrecommitPlugin) {
       project.getTasks().withType(ThirdPartyAuditTask.class).configureEach {
         if (BuildParams.getIsRuntimeJavaHomeSet() == false) {

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestTestBasePlugin.java

@@ -63,7 +63,6 @@
 public class RestTestBasePlugin implements Plugin<Project> {
 
     private static final String TESTS_MAX_PARALLEL_FORKS_SYSPROP = "tests.max.parallel.forks";
-    private static final String TESTS_RUNTIME_JAVA_SYSPROP = "tests.runtime.java";
     private static final String DEFAULT_DISTRIBUTION_SYSPROP = "tests.default.distribution";
     private static final String INTEG_TEST_DISTRIBUTION_SYSPROP = "tests.integ-test.distribution";
     private static final String BWC_SNAPSHOT_DISTRIBUTION_SYSPROP_PREFIX = "tests.snapshot.distribution.";
@@ -189,7 +188,6 @@ public void apply(Project project) {
             // Wire up integ-test distribution by default for all test tasks
             FileCollection extracted = integTestDistro.getExtracted();
             nonInputSystemProperties.systemProperty(INTEG_TEST_DISTRIBUTION_SYSPROP, () -> extracted.getSingleFile().getPath());
-            nonInputSystemProperties.systemProperty(TESTS_RUNTIME_JAVA_SYSPROP, BuildParams.getRuntimeJavaHome());
 
             // Add `usesDefaultDistribution()` extension method to test tasks to indicate they require the default distro
             task.getExtensions().getExtraProperties().set("usesDefaultDistribution", new Closure<Void>(task) {

build-tools-internal/src/main/resources/checkstyle.xml

@@ -194,5 +194,12 @@
             value="Do not negate boolean expressions with '!', but check explicitly with '== false' as it is more explicit"/>
     </module>
 
+    <module name="NeedBraces">
+        <!-- We have many generated equals classes with LITERAL_IF and no braces, so we don't check IF or ELSE -->
+        <property name="tokens" value="LITERAL_DO,LITERAL_FOR,LITERAL_WHILE"/>
+        <property name="allowSingleLineStatement" value="false"/>
+        <property name="allowEmptyLoopBody" value="true"/>
+    </module>
+
   </module>
 </module>

build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/match/RemoveMatchTests.java

@@ -9,11 +9,8 @@
 package org.elasticsearch.gradle.internal.test.rest.transform.match;
 
 import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.ObjectReader;
 import com.fasterxml.jackson.databind.node.ArrayNode;
 import com.fasterxml.jackson.databind.node.ObjectNode;
-import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
 
 import org.elasticsearch.gradle.internal.test.rest.transform.TransformTests;
 import org.hamcrest.CoreMatchers;
@@ -29,10 +26,6 @@
 
 public class RemoveMatchTests extends TransformTests {
 
-    private static final YAMLFactory YAML_FACTORY = new YAMLFactory();
-    private static final ObjectMapper MAPPER = new ObjectMapper(YAML_FACTORY);
-    private static final ObjectReader READER = MAPPER.readerFor(ObjectNode.class);
-
     @Test
     public void testRemoveAll() throws Exception {
         String testName = "/rest/transform/match/match_original.yml";

build-tools/src/main/java/org/elasticsearch/gradle/testclusters/RunTask.java

@@ -201,7 +201,7 @@ public void beforeStart() {
                     try {
                         mockServer.start();
                         node.setting("telemetry.metrics.enabled", "true");
-                        node.setting("tracing.apm.agent.enabled", "true");
+                        node.setting("tracing.apm.enabled", "true");
                         node.setting("tracing.apm.agent.transaction_sample_rate", "0.10");
                         node.setting("tracing.apm.agent.metrics_interval", "10s");
                         node.setting("tracing.apm.agent.server_url", "http://127.0.0.1:" + mockServer.getPort());
@@ -213,8 +213,8 @@ public void beforeStart() {
                 // if metrics were not enabled explicitly for gradlew run we should disable them
                 else if (node.getSettingKeys().contains("telemetry.metrics.enabled") == false) { // metrics
                     node.setting("telemetry.metrics.enabled", "false");
-                } else if (node.getSettingKeys().contains("tracing.apm.agent.enabled") == false) { // tracing
-                    node.setting("tracing.apm.agent.enable", "false");
+                } else if (node.getSettingKeys().contains("tracing.apm.enabled") == false) { // tracing
+                    node.setting("tracing.apm.enable", "false");
                 }
 
             }

build-tools/src/main/java/org/elasticsearch/gradle/util/GradleUtils.java

@@ -48,7 +48,7 @@ public static SourceSetContainer getJavaSourceSets(Project project) {
     }
 
     public static void maybeConfigure(TaskContainer tasks, String name, Action<? super Task> config) {
-        tasks.matching(t -> t.getName().equals(name)).configureEach(t -> config.execute(t));
+        tasks.matching(t -> t.getName().equals(name)).configureEach(config);
     }
 
     public static <T extends Task> void maybeConfigure(
@@ -57,7 +57,7 @@ public static <T extends Task> void maybeConfigure(
         Class<? extends T> type,
         Action<? super T> config
     ) {
-        tasks.withType(type).matching((Spec<T>) t -> t.getName().equals(name)).configureEach(task -> { config.execute(task); });
+        tasks.withType(type).matching((Spec<T>) t -> t.getName().equals(name)).configureEach(config);
     }
 
     public static TaskProvider<?> findByName(TaskContainer tasks, String name) {

client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/NoopPlugin.java

@@ -14,6 +14,7 @@
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Settings;
@@ -47,6 +48,7 @@ public class NoopPlugin extends Plugin implements ActionPlugin {
     @Override
     public List<RestHandler> getRestHandlers(
         Settings settings,
+        NamedWriteableRegistry namedWriteableRegistry,
         RestController restController,
         ClusterSettings clusterSettings,
         IndexScopedSettings indexScopedSettings,

client/rest/src/main/java/org/elasticsearch/client/RestClientBuilder.java

@@ -37,6 +37,8 @@
 import java.util.Locale;
 import java.util.Objects;
 import java.util.Properties;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.atomic.AtomicLong;
 
 import javax.net.ssl.SSLContext;
 
@@ -51,6 +53,9 @@ public final class RestClientBuilder {
     public static final int DEFAULT_MAX_CONN_PER_ROUTE = 10;
     public static final int DEFAULT_MAX_CONN_TOTAL = 30;
 
+    static final String THREAD_NAME_PREFIX = "elasticsearch-rest-client-";
+    private static final String THREAD_NAME_FORMAT = THREAD_NAME_PREFIX + "%d-thread-%d";
+
     public static final String VERSION;
     static final String META_HEADER_NAME = "X-Elastic-Client-Meta";
     static final String META_HEADER_VALUE;
@@ -298,6 +303,24 @@ public RestClient build() {
         return restClient;
     }
 
+    /**
+     * Similar to {@code org.apache.http.impl.nio.reactor.AbstractMultiworkerIOReactor.DefaultThreadFactory} but with better thread names.
+     */
+    private static class RestClientThreadFactory implements ThreadFactory {
+        private static final AtomicLong CLIENT_THREAD_POOL_ID_GENERATOR = new AtomicLong();
+
+        private final long clientThreadPoolId = CLIENT_THREAD_POOL_ID_GENERATOR.getAndIncrement(); // 0-based
+        private final AtomicLong clientThreadId = new AtomicLong();
+
+        @Override
+        public Thread newThread(Runnable runnable) {
+            return new Thread(
+                runnable,
+                String.format(Locale.ROOT, THREAD_NAME_FORMAT, clientThreadPoolId, clientThreadId.incrementAndGet()) // 1-based
+            );
+        }
+    }
+
     private CloseableHttpAsyncClient createHttpClient() {
         // default timeouts are all infinite
         RequestConfig.Builder requestConfigBuilder = RequestConfig.custom()
@@ -315,7 +338,8 @@ private CloseableHttpAsyncClient createHttpClient() {
                 .setMaxConnTotal(DEFAULT_MAX_CONN_TOTAL)
                 .setSSLContext(SSLContext.getDefault())
                 .setUserAgent(USER_AGENT_HEADER_VALUE)
-                .setTargetAuthenticationStrategy(new PersistentCredentialsAuthenticationStrategy());
+                .setTargetAuthenticationStrategy(new PersistentCredentialsAuthenticationStrategy())
+                .setThreadFactory(new RestClientThreadFactory());
             if (httpClientConfigCallback != null) {
                 httpClientBuilder = httpClientConfigCallback.customizeHttpClient(httpClientBuilder);
             }