Merge branch 'main' into esql-binary-operators-part-2

 Conflicts:
	docs/reference/esql/functions/types/to_lower.asciidoc
	docs/reference/esql/functions/types/to_upper.asciidoc
	x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/SubTests.java

.buildkite/pipelines/intake.yml

@@ -48,7 +48,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["7.17.17", "8.11.4", "8.12.0", "8.13.0"]
+            BWC_VERSION: ["7.17.18", "8.12.1", "8.13.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.buildkite/pipelines/periodic-packaging.yml

@@ -1105,6 +1105,22 @@ steps:
         env:
           BWC_VERSION: 7.17.17
 
+      - label: "{{matrix.image}} / 7.17.18 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v7.17.18
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            image:
+              - rocky-8
+              - ubuntu-2004
+        agents:
+          provider: gcp
+          image: family/elasticsearch-{{matrix.image}}
+          machineType: custom-16-32768
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 7.17.18
+
       - label: "{{matrix.image}} / 8.0.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.0.0
         timeout_in_minutes: 300
@@ -1809,6 +1825,22 @@ steps:
         env:
           BWC_VERSION: 8.12.0
 
+      - label: "{{matrix.image}} / 8.12.1 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.12.1
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            image:
+              - rocky-8
+              - ubuntu-2004
+        agents:
+          provider: gcp
+          image: family/elasticsearch-{{matrix.image}}
+          machineType: custom-16-32768
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 8.12.1
+
       - label: "{{matrix.image}} / 8.13.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.13.0
         timeout_in_minutes: 300

.buildkite/pipelines/periodic-platform-support.yml

@@ -89,6 +89,7 @@ steps:
           setup:
             image:
               - amazonlinux-2023
+              - amazonlinux-2
         agents:
           provider: aws
           imagePrefix: elasticsearch-{{matrix.image}}

.buildkite/pipelines/periodic.template.yml

@@ -2,7 +2,7 @@ steps:
   - group: bwc
     steps: $BWC_STEPS
   - label: concurrent-search-tests
-    command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dtests.jvm.argline=-Des.concurrent_search=true -Des.concurrent_search=true check
+    command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dtests.jvm.argline=-Des.concurrent_search=true -Des.concurrent_search=true functionalTests
     timeout_in_minutes: 420
     agents:
       provider: gcp
@@ -97,7 +97,7 @@ steps:
       diskSizeGb: 350
       machineType: custom-32-98304
   - label: single-processor-node-tests
-    command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dtests.configure_test_clusters_with_one_processor=true check
+    command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dtests.configure_test_clusters_with_one_processor=true functionalTests
     timeout_in_minutes: 420
     agents:
       provider: gcp

.buildkite/pipelines/periodic.yml

@@ -672,6 +672,16 @@ steps:
           buildDirectory: /dev/shm/bk
         env:
           BWC_VERSION: 7.17.17
+      - label: 7.17.18 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v7.17.18#bwcTest
+        timeout_in_minutes: 300
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2004
+          machineType: n1-standard-32
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 7.17.18
       - label: 8.0.0 / bwc
         command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.0.0#bwcTest
         timeout_in_minutes: 300
@@ -1112,6 +1122,16 @@ steps:
           buildDirectory: /dev/shm/bk
         env:
           BWC_VERSION: 8.12.0
+      - label: 8.12.1 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.12.1#bwcTest
+        timeout_in_minutes: 300
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2004
+          machineType: n1-standard-32
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 8.12.1
       - label: 8.13.0 / bwc
         command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.13.0#bwcTest
         timeout_in_minutes: 300
@@ -1123,7 +1143,7 @@ steps:
         env:
           BWC_VERSION: 8.13.0
   - label: concurrent-search-tests
-    command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dtests.jvm.argline=-Des.concurrent_search=true -Des.concurrent_search=true check
+    command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dtests.jvm.argline=-Des.concurrent_search=true -Des.concurrent_search=true functionalTests
     timeout_in_minutes: 420
     agents:
       provider: gcp
@@ -1218,7 +1238,7 @@ steps:
       diskSizeGb: 350
       machineType: custom-32-98304
   - label: single-processor-node-tests
-    command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dtests.configure_test_clusters_with_one_processor=true check
+    command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dtests.configure_test_clusters_with_one_processor=true functionalTests
     timeout_in_minutes: 420
     agents:
       provider: gcp

.buildkite/scripts/encryption-at-rest.sh

@@ -22,4 +22,4 @@ touch .output.log
 rm -Rf "$WORKSPACE"
 ln -s "$PWD" "$WORKSPACE"
 
-.ci/scripts/run-gradle.sh -Dbwc.checkout.align=true check
+.ci/scripts/run-gradle.sh -Dbwc.checkout.align=true functionalTests

.buildkite/scripts/fixture-deploy.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+set -euo pipefail
+
+echo "$DOCKER_REGISTRY_PASSWORD" | docker login -u "$DOCKER_REGISTRY_USERNAME" --password-stdin docker.elastic.co
+unset DOCKER_REGISTRY_USERNAME DOCKER_REGISTRY_PASSWORD
+
+docker buildx create --use
+.ci/scripts/run-gradle.sh deployFixtureDockerImages

.buildkite/scripts/release-tests.sh

@@ -20,4 +20,4 @@ curl --fail -o "${ML_IVY_REPO}/maven/org/elasticsearch/ml/ml-cpp/${ES_VERSION}/m
 curl --fail -o "${ML_IVY_REPO}/maven/org/elasticsearch/ml/ml-cpp/${ES_VERSION}/ml-cpp-${ES_VERSION}.zip" https://artifacts-snapshot.elastic.co/ml-cpp/${ES_VERSION}-SNAPSHOT/downloads/ml-cpp/ml-cpp-${ES_VERSION}-SNAPSHOT.zip
 
 .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dbuild.snapshot=false -Dbuild.ml_cpp.repo=file://${ML_IVY_REPO} \
-  -Dtests.jvm.argline=-Dbuild.snapshot=false -Dlicense.key=${WORKSPACE}/x-pack/license-tools/src/test/resources/public.key -Dbuild.id=deadbeef build
+  -Dtests.jvm.argline=-Dbuild.snapshot=false -Dlicense.key=${WORKSPACE}/x-pack/license-tools/src/test/resources/public.key -Dbuild.id=deadbeef assemble functionalTests

.ci/bwcVersions

@@ -66,6 +66,7 @@ BWC_VERSION:
   - "7.17.15"
   - "7.17.16"
   - "7.17.17"
+  - "7.17.18"
   - "8.0.0"
   - "8.0.1"
   - "8.1.0"
@@ -110,4 +111,5 @@ BWC_VERSION:
   - "8.11.3"
   - "8.11.4"
   - "8.12.0"
+  - "8.12.1"
   - "8.13.0"

.ci/snapshotBwcVersions

@@ -1,5 +1,4 @@
 BWC_VERSION:
-  - "7.17.17"
-  - "8.11.4"
-  - "8.12.0"
+  - "7.17.18"
+  - "8.12.1"
   - "8.13.0"

.github/CODEOWNERS

@@ -3,18 +3,18 @@
 # For more info, see https://help.github.com/articles/about-codeowners/
 
 # Stack Monitoring index templates
-x-pack/plugin/core/template-resources/src/main/resources/monitoring-alerts-7.json @elastic/infra-monitoring-ui
-x-pack/plugin/core/template-resources/src/main/resources/monitoring-beats-mb.json @elastic/infra-monitoring-ui
-x-pack/plugin/core/template-resources/src/main/resources/monitoring-beats.json @elastic/infra-monitoring-ui
-x-pack/plugin/core/template-resources/src/main/resources/monitoring-ent-search-mb.json @elastic/infra-monitoring-ui
-x-pack/plugin/core/template-resources/src/main/resources/monitoring-es-mb.json @elastic/infra-monitoring-ui
-x-pack/plugin/core/template-resources/src/main/resources/monitoring-es.json @elastic/infra-monitoring-ui
-x-pack/plugin/core/template-resources/src/main/resources/monitoring-kibana-mb.json @elastic/infra-monitoring-ui
-x-pack/plugin/core/template-resources/src/main/resources/monitoring-kibana.json @elastic/infra-monitoring-ui
-x-pack/plugin/core/template-resources/src/main/resources/monitoring-logstash-mb.json @elastic/infra-monitoring-ui
-x-pack/plugin/core/template-resources/src/main/resources/monitoring-logstash.json @elastic/infra-monitoring-ui
-x-pack/plugin/core/template-resources/src/main/resources/monitoring-mb-ilm-policy.json @elastic/infra-monitoring-ui
-x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/MonitoringTemplateRegistry.java @elastic/infra-monitoring-ui
+x-pack/plugin/core/template-resources/src/main/resources/monitoring-alerts-7.json @elastic/stack-monitoring
+x-pack/plugin/core/template-resources/src/main/resources/monitoring-beats-mb.json @elastic/stack-monitoring
+x-pack/plugin/core/template-resources/src/main/resources/monitoring-beats.json @elastic/stack-monitoring
+x-pack/plugin/core/template-resources/src/main/resources/monitoring-ent-search-mb.json @elastic/stack-monitoring
+x-pack/plugin/core/template-resources/src/main/resources/monitoring-es-mb.json @elastic/stack-monitoring
+x-pack/plugin/core/template-resources/src/main/resources/monitoring-es.json @elastic/stack-monitoring
+x-pack/plugin/core/template-resources/src/main/resources/monitoring-kibana-mb.json @elastic/stack-monitoring
+x-pack/plugin/core/template-resources/src/main/resources/monitoring-kibana.json @elastic/stack-monitoring
+x-pack/plugin/core/template-resources/src/main/resources/monitoring-logstash-mb.json @elastic/stack-monitoring
+x-pack/plugin/core/template-resources/src/main/resources/monitoring-logstash.json @elastic/stack-monitoring
+x-pack/plugin/core/template-resources/src/main/resources/monitoring-mb-ilm-policy.json @elastic/stack-monitoring
+x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/MonitoringTemplateRegistry.java @elastic/stack-monitoring
 
 # Fleet
 x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet @elastic/fleet

TRACING.md

@@ -24,7 +24,7 @@ In your `elasticsearch.yml` add the following configuration:
 
 ```
 tracing.apm.enabled: true
-tracing.apm.agent.server_url: https://<your-apm-server>:443
+telemetry.agent.server_url: https://<your-apm-server>:443
 ```
 
 When using a secret token to authenticate with the APM server, you must add it to the Elasticsearch keystore under `tracing.apm.secret_token`. For example, execute:
@@ -34,7 +34,7 @@ When using a secret token to authenticate with the APM server, you must add it t
 then enter the token when prompted. If you are using API keys, change the keystore key name to `tracing.apm.api_key`.
 
 All APM settings live under `tracing.apm`. All settings related to the Java agent
-go under `tracing.apm.agent`. Anything you set under there will be propagated to
+go under `telemetry.agent`. Anything you set under there will be propagated to
 the agent.
 
 For agent settings that can be changed dynamically, you can use the cluster
@@ -43,7 +43,7 @@ settings REST API. For example, to change the sampling rate:
     curl -XPUT \
       -H "Content-type: application/json" \
       -u "$USERNAME:$PASSWORD" \
-      -d '{ "persistent": { "tracing.apm.agent.transaction_sample_rate": "0.75" } }' \
+      -d '{ "persistent": { "telemetry.agent.transaction_sample_rate": "0.75" } }' \
       https://localhost:9200/_cluster/settings
 
 

benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/BlockBenchmark.java

@@ -720,7 +720,7 @@ private static long computeBytesRefCheckSum(BytesRefBlock block, int[] traversal
     }
 
     private static long computeDoubleCheckSum(DoubleBlock block, int[] traversalOrder) {
-        double sum = 0;
+        long sum = 0;
 
         for (int position : traversalOrder) {
             if (block.isNull(position)) {
@@ -729,11 +729,12 @@ private static long computeDoubleCheckSum(DoubleBlock block, int[] traversalOrde
             int start = block.getFirstValueIndex(position);
             int end = start + block.getValueCount(position);
             for (int i = start; i < end; i++) {
-                sum += block.getDouble(i);
+                // Use an operation that is not affected by rounding errors. Otherwise, the result may depend on the traversalOrder.
+                sum += (long) block.getDouble(i);
             }
         }
 
-        return (long) sum;
+        return sum;
     }
 
     private static long computeIntCheckSum(IntBlock block, int[] traversalOrder) {

benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/ValuesSourceReaderBenchmark.java

@@ -189,6 +189,11 @@ public String indexName() {
                     return "benchmark";
                 }
 
+                @Override
+                public MappedFieldType.FieldExtractPreference fieldExtractPreference() {
+                    return MappedFieldType.FieldExtractPreference.NONE;
+                }
+
                 @Override
                 public SearchLookup lookup() {
                     throw new UnsupportedOperationException();

benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/ShardsAvailabilityHealthIndicatorBenchmark.java

@@ -31,6 +31,7 @@
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.SystemIndices;
 import org.elasticsearch.tasks.TaskManager;
+import org.elasticsearch.telemetry.metric.MeterRegistry;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.BenchmarkMode;
@@ -166,7 +167,7 @@ public void setUp() throws Exception {
             .build();
 
         Settings settings = Settings.builder().put("node.name", ShardsAvailabilityHealthIndicatorBenchmark.class.getSimpleName()).build();
-        ThreadPool threadPool = new ThreadPool(settings);
+        ThreadPool threadPool = new ThreadPool(settings, MeterRegistry.NOOP);
 
         ClusterService clusterService = new ClusterService(
             Settings.EMPTY,

branches.json

@@ -7,9 +7,6 @@
     {
       "branch": "8.12"
     },
-    {
-      "branch": "8.11"
-    },
     {
       "branch": "7.17"
     }

build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/precommit/PomValidationPrecommitPlugin.java

@@ -30,11 +30,11 @@ public TaskProvider<? extends Task> createTask(Project project) {
                 .register("validate" + publicationName + "Pom", PomValidationTask.class);
             validatePom.configure(t -> t.dependsOn(validateTask));
             validateTask.configure(task -> {
-                GenerateMavenPom generateMavenPom = project.getTasks()
+                TaskProvider<GenerateMavenPom> generateMavenPom = project.getTasks()
                     .withType(GenerateMavenPom.class)
-                    .getByName("generatePomFileFor" + publicationName + "Publication");
+                    .named("generatePomFileFor" + publicationName + "Publication");
                 task.dependsOn(generateMavenPom);
-                task.getPomFile().fileValue(generateMavenPom.getDestination());
+                task.getPomFile().fileProvider(generateMavenPom.map(GenerateMavenPom::getDestination));
             });
         });
 

build-tools-internal/build.gradle

@@ -139,6 +139,10 @@ gradlePlugin {
       id = 'elasticsearch.java-module'
       implementationClass = 'org.elasticsearch.gradle.internal.ElasticsearchJavaModulePathPlugin'
     }
+    mrjar {
+      id = 'elasticsearch.mrjar'
+      implementationClass = 'org.elasticsearch.gradle.internal.MrjarPlugin'
+    }
     releaseTools {
       id = 'elasticsearch.release-tools'
       implementationClass = 'org.elasticsearch.gradle.internal.release.ReleaseToolsPlugin'

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/SymbolicLinkPreservingTarFuncTest.groovy

@@ -8,6 +8,8 @@
 
 package org.elasticsearch.gradle.internal
 
+import spock.lang.Ignore
+
 import org.apache.commons.compress.archivers.tar.TarArchiveEntry
 import org.apache.commons.compress.archivers.tar.TarArchiveInputStream
 import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream
@@ -36,6 +38,11 @@ class SymbolicLinkPreservingTarFuncTest extends AbstractGradleFuncTest {
         final Path linkToRealFolder = archiveSourceRoot.resolve("link-to-real-folder");
         Files.createSymbolicLink(linkToRealFolder, Paths.get("./real-folder"));
 
+        final Path realFolder2 = testProjectDir.getRoot().toPath().resolve("real-folder2")
+        final Path realFolderSub = realFolder2.resolve("sub")
+        Files.createDirectory(realFolder2);
+        Files.createDirectory(realFolderSub);
+
         buildFile << """
 import org.elasticsearch.gradle.internal.SymbolicLinkPreservingTar
 
@@ -56,6 +63,12 @@ tasks.register("buildBZip2Tar", SymbolicLinkPreservingTar) { SymbolicLinkPreserv
   tar.compression = Compression.BZIP2
   tar.preserveFileTimestamps = ${preserverTimestamp}
   from fileTree("archiveRoot")
+
+  into('config') {
+    dirMode 0750
+    fileMode 0660
+    from "real-folder2"
+  }
 }
 """
         when:
@@ -118,14 +131,20 @@ tasks.register("buildTar", SymbolicLinkPreservingTar) { SymbolicLinkPreservingTa
                 if (entry.getName().equals("real-folder/")) {
                     assert entry.isDirectory()
                     realFolderEntry = true
-                } else if (entry.getName().equals("real-folder/file")) {
+                }  else if (entry.getName().equals("real-folder/file")) {
                     assert entry.isFile()
                     fileEntry = true
                 } else if (entry.getName().equals("real-folder/link-to-file")) {
                     assert entry.isSymbolicLink()
                     assert normalized(entry.getLinkName()) == "./file"
                     linkToFileEntry = true
-                } else if (entry.getName().equals("link-in-folder/")) {
+                } else if (entry.getName().equals("config/")) {
+                    assert entry.isDirectory()
+                    assert entry.getMode() == 16877
+                } else if (entry.getName().equals("config/sub/")) {
+                    assert entry.isDirectory()
+                    assert entry.getMode() == 16872
+                }else if (entry.getName().equals("link-in-folder/")) {
                     assert entry.isDirectory()
                     linkInFolderEntry = true
                 } else if (entry.getName().equals("link-in-folder/link-to-file")) {

build-tools-internal/src/main/groovy/elasticsearch.fips.gradle

@@ -79,7 +79,6 @@ if (BuildParams.inFipsJvm) {
           // with no x-pack. Tests having security explicitly enabled/disabled will override this setting
           setting 'xpack.security.enabled', 'false'
           setting 'xpack.security.fips_mode.enabled', 'true'
-          setting 'xpack.security.fips_mode.required_providers', '["BCFIPS", "BCJSSE"]'
           setting 'xpack.license.self_generated.type', 'trial'
           setting 'xpack.security.authc.password_hashing.algorithm', 'pbkdf2_stretch'
           keystorePassword 'keystore-password'