Merge branch 'main' into Enable_OTel_data_by_default

.buildkite/hooks/pre-command

@@ -78,11 +78,15 @@ if [[ "${USE_SNYK_CREDENTIALS:-}" == "true" ]]; then
 fi
 
 if [[ "${USE_PROD_DOCKER_CREDENTIALS:-}" == "true" ]]; then
-  DOCKER_REGISTRY_USERNAME="$(vault read -field=username secret/ci/elastic-elasticsearch/migrated/prod_docker_registry_credentials)"
-  export DOCKER_REGISTRY_USERNAME
+  if which docker > /dev/null 2>&1; then
+    DOCKER_REGISTRY_USERNAME="$(vault read -field=username secret/ci/elastic-elasticsearch/migrated/prod_docker_registry_credentials)"
+    export DOCKER_REGISTRY_USERNAME
 
-  DOCKER_REGISTRY_PASSWORD="$(vault read -field=password secret/ci/elastic-elasticsearch/migrated/prod_docker_registry_credentials)"
-  export DOCKER_REGISTRY_PASSWORD
+    DOCKER_REGISTRY_PASSWORD="$(vault read -field=password secret/ci/elastic-elasticsearch/migrated/prod_docker_registry_credentials)"
+    export DOCKER_REGISTRY_PASSWORD
+
+    docker login --username "$DOCKER_REGISTRY_USERNAME" --password "$DOCKER_REGISTRY_PASSWORD" docker.elastic.co
+  fi
 fi
 
 if [[ "$BUILDKITE_AGENT_META_DATA_PROVIDER" != *"k8s"* ]]; then

.buildkite/pipelines/intake.yml

@@ -62,7 +62,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["8.15.2", "8.16.0", "9.0.0"]
+            BWC_VERSION: ["8.15.3", "8.16.0", "9.0.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.buildkite/pipelines/periodic-packaging.template.yml

@@ -8,7 +8,6 @@ steps:
           setup:
             image:
               - centos-7
-              - debian-10
               - debian-11
               - opensuse-leap-15
               - oraclelinux-7

.buildkite/pipelines/periodic-packaging.yml

@@ -9,7 +9,6 @@ steps:
           setup:
             image:
               - centos-7
-              - debian-10
               - debian-11
               - opensuse-leap-15
               - oraclelinux-7
@@ -288,8 +287,8 @@ steps:
         env:
           BWC_VERSION: 8.14.3
 
-      - label: "{{matrix.image}} / 8.15.2 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.15.2
+      - label: "{{matrix.image}} / 8.15.3 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.15.3
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -303,7 +302,7 @@ steps:
           buildDirectory: /dev/shm/bk
           diskSizeGb: 250
         env:
-          BWC_VERSION: 8.15.2
+          BWC_VERSION: 8.15.3
 
       - label: "{{matrix.image}} / 8.16.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.16.0

.buildkite/pipelines/periodic-platform-support.yml

@@ -8,7 +8,6 @@ steps:
           setup:
             image:
               - centos-7
-              - debian-10
               - debian-11
               - opensuse-leap-15
               - oraclelinux-7

.buildkite/pipelines/periodic.yml

@@ -302,8 +302,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 8.15.2 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.15.2#bwcTest
+      - label: 8.15.3 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.15.3#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -313,7 +313,7 @@ steps:
           preemptible: true
           diskSizeGb: 250
         env:
-          BWC_VERSION: 8.15.2
+          BWC_VERSION: 8.15.3
         retry:
           automatic:
             - exit_status: "-1"
@@ -431,7 +431,7 @@ steps:
           setup:
             ES_RUNTIME_JAVA:
               - openjdk17
-            BWC_VERSION: ["8.15.2", "8.16.0", "9.0.0"]
+            BWC_VERSION: ["8.15.3", "8.16.0", "9.0.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
@@ -475,7 +475,7 @@ steps:
             ES_RUNTIME_JAVA:
               - openjdk21
               - openjdk23
-            BWC_VERSION: ["8.15.2", "8.16.0", "9.0.0"]
+            BWC_VERSION: ["8.15.3", "8.16.0", "9.0.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
@@ -584,7 +584,7 @@ steps:
       machineType: n2-standard-8
       buildDirectory: /dev/shm/bk
       diskSizeGb: 250
-    if: build.branch == "main" || build.branch == "8.x" || build.branch == "7.17"
+    if: build.branch == "main" || build.branch == "7.17"
   - label: check-branch-consistency
     command: .ci/scripts/run-gradle.sh branchConsistency
     timeout_in_minutes: 15

.buildkite/pipelines/pull-request/packaging-tests-unix.yml

@@ -11,7 +11,6 @@ steps:
           setup:
             image:
               - centos-7
-              - debian-10
               - debian-11
               - opensuse-leap-15
               - oraclelinux-7
@@ -40,7 +39,6 @@ steps:
           setup:
             image:
               - centos-7
-              - debian-10
               - debian-11
               - opensuse-leap-15
               - oraclelinux-7
@@ -69,7 +67,6 @@ steps:
           setup:
             image:
               - centos-7
-              - debian-10
               - debian-11
               - opensuse-leap-15
               - oraclelinux-7

.buildkite/pull-requests.json

@@ -14,21 +14,6 @@
       "trigger_comment_regex": "(run\\W+elasticsearch-ci.+)|(^\\s*((buildkite|@elastic(search)?machine)\\s*)?test\\s+this(\\s+please)?)",
       "cancel_intermediate_builds": true,
       "cancel_intermediate_builds_on_comment": false
-    },
-    {
-      "enabled": true,
-      "pipeline_slug": "elasticsearch-pull-request-check-serverless-submodule",
-      "allow_org_users": true,
-      "allowed_repo_permissions": [
-        "admin",
-        "write"
-      ],
-      "set_commit_status": false,
-      "build_on_commit": true,
-      "build_on_comment": false,
-      "labels": [
-        "test-update-serverless"
-      ]
     }
   ]
 }

.ci/bwcVersions

@@ -14,6 +14,6 @@ BWC_VERSION:
   - "8.12.2"
   - "8.13.4"
   - "8.14.3"
-  - "8.15.2"
+  - "8.15.3"
   - "8.16.0"
   - "9.0.0"

.ci/scripts/packaging-test.sh

@@ -77,5 +77,6 @@ sudo -E env \
   --unset=ES_JAVA_HOME \
   --unset=JAVA_HOME \
   SYSTEM_JAVA_HOME=`readlink -f -n $BUILD_JAVA_HOME` \
+  DOCKER_CONFIG="${HOME}/.docker" \
   ./gradlew -g $HOME/.gradle --scan --parallel --build-cache -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/ --continue $@
 

.ci/snapshotBwcVersions

@@ -1,4 +1,4 @@
 BWC_VERSION:
-  - "8.15.2"
+  - "8.15.3"
   - "8.16.0"
   - "9.0.0"

CONTRIBUTING.md

@@ -111,9 +111,9 @@ Contributing to the Elasticsearch codebase
 
 **Repository:** [https://github.com/elastic/elasticsearch](https://github.com/elastic/elasticsearch)
 
-JDK 17 is required to build Elasticsearch. You must have a JDK 17 installation
+JDK 21 is required to build Elasticsearch. You must have a JDK 21 installation
 with the environment variable `JAVA_HOME` referencing the path to Java home for
-your JDK 17 installation.
+your JDK 21 installation.
 
 Elasticsearch uses the Gradle wrapper for its build. You can execute Gradle
 using the wrapper via the `gradlew` script on Unix systems or `gradlew.bat`
@@ -152,9 +152,9 @@ The definition of this Elasticsearch cluster can be found [here](build-tools-int
 ### Importing the project into IntelliJ IDEA
 
 The minimum IntelliJ IDEA version required to import the Elasticsearch project is 2020.1.
-Elasticsearch builds using Java 17. When importing into IntelliJ you will need
+Elasticsearch builds using Java 21. When importing into IntelliJ you will need
 to define an appropriate SDK. The convention is that **this SDK should be named
-"17"** so that the project import will detect it automatically. For more details
+"21"** so that the project import will detect it automatically. For more details
 on defining an SDK in IntelliJ please refer to [their documentation](https://www.jetbrains.com/help/idea/sdk.html#define-sdk).
 SDK definitions are global, so you can add the JDK from any project, or after
 project import. Importing with a missing JDK will still work, IntelliJ will

benchmarks/src/main/java/org/elasticsearch/benchmark/tdigest/SortBench.java

@@ -21,10 +21,12 @@
 
 package org.elasticsearch.benchmark.tdigest;
 
+import org.elasticsearch.common.breaker.NoopCircuitBreaker;
+import org.elasticsearch.search.aggregations.metrics.MemoryTrackingTDigestArrays;
 import org.elasticsearch.tdigest.Sort;
+import org.elasticsearch.tdigest.arrays.TDigestArrays;
 import org.elasticsearch.tdigest.arrays.TDigestDoubleArray;
 import org.elasticsearch.tdigest.arrays.TDigestIntArray;
-import org.elasticsearch.tdigest.arrays.WrapperTDigestArrays;
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.BenchmarkMode;
 import org.openjdk.jmh.annotations.Fork;
@@ -51,7 +53,8 @@
 @State(Scope.Thread)
 public class SortBench {
     private final int size = 100000;
-    private final TDigestDoubleArray values = WrapperTDigestArrays.INSTANCE.newDoubleArray(size);
+    private final TDigestArrays arrays = new MemoryTrackingTDigestArrays(new NoopCircuitBreaker("default-wrapper-tdigest-arrays"));
+    private final TDigestDoubleArray values = arrays.newDoubleArray(size);
 
     @Param({ "0", "1", "-1" })
     public int sortDirection;
@@ -72,7 +75,7 @@ public void setup() {
 
     @Benchmark
     public void stableSort() {
-        TDigestIntArray order = WrapperTDigestArrays.INSTANCE.newIntArray(size);
+        TDigestIntArray order = arrays.newIntArray(size);
         for (int i = 0; i < size; i++) {
             order.set(i, i);
         }

benchmarks/src/main/java/org/elasticsearch/benchmark/tdigest/TDigestBench.java

@@ -21,9 +21,11 @@
 
 package org.elasticsearch.benchmark.tdigest;
 
+import org.elasticsearch.common.breaker.NoopCircuitBreaker;
+import org.elasticsearch.search.aggregations.metrics.MemoryTrackingTDigestArrays;
 import org.elasticsearch.tdigest.MergingDigest;
 import org.elasticsearch.tdigest.TDigest;
-import org.elasticsearch.tdigest.arrays.WrapperTDigestArrays;
+import org.elasticsearch.tdigest.arrays.TDigestArrays;
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.BenchmarkMode;
 import org.openjdk.jmh.annotations.Fork;
@@ -56,24 +58,25 @@
 @Threads(1)
 @State(Scope.Thread)
 public class TDigestBench {
+    private static final TDigestArrays arrays = new MemoryTrackingTDigestArrays(new NoopCircuitBreaker("default-wrapper-tdigest-arrays"));
 
     public enum TDigestFactory {
         MERGE {
             @Override
             TDigest create(double compression) {
-                return new MergingDigest(WrapperTDigestArrays.INSTANCE, compression, (int) (10 * compression));
+                return new MergingDigest(arrays, compression, (int) (10 * compression));
             }
         },
         AVL_TREE {
             @Override
             TDigest create(double compression) {
-                return TDigest.createAvlTreeDigest(WrapperTDigestArrays.INSTANCE, compression);
+                return TDigest.createAvlTreeDigest(arrays, compression);
             }
         },
         HYBRID {
             @Override
             TDigest create(double compression) {
-                return TDigest.createHybridDigest(WrapperTDigestArrays.INSTANCE, compression);
+                return TDigest.createHybridDigest(arrays, compression);
             }
         };
 

build-tools-internal/src/main/groovy/elasticsearch.ide.gradle

@@ -169,6 +169,7 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') {
               '-ea',
               '-Djava.security.manager=allow',
               '-Djava.locale.providers=CLDR',
+              '-Dtests.testfeatures.enabled=true',
               '-Des.nativelibs.path="' + testLibraryPath + '"',
               // TODO: only open these for mockito when it is modularized
               '--add-opens=java.base/java.security.cert=ALL-UNNAMED',

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java

@@ -16,7 +16,7 @@ public enum DockerBase {
     DEFAULT("ubuntu:20.04", "", "apt-get"),
 
     // "latest" here is intentional, since the image name specifies "8"
-    UBI("docker.elastic.co/ubi8/ubi-minimal:latest", "-ubi8", "microdnf"),
+    UBI("docker.elastic.co/ubi8/ubi-minimal:latest", "-ubi", "microdnf"),
 
     // The Iron Bank base image is UBI (albeit hardened), but we are required to parameterize the Docker build
     IRON_BANK("${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG}", "-ironbank", "yum"),

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java

@@ -108,6 +108,7 @@ public void execute(Task t) {
                 "-Xmx" + System.getProperty("tests.heap.size", "512m"),
                 "-Xms" + System.getProperty("tests.heap.size", "512m"),
                 "-Djava.security.manager=allow",
+                "-Dtests.testfeatures.enabled=true",
                 "--add-opens=java.base/java.util=ALL-UNNAMED",
                 // TODO: only open these for mockito when it is modularized
                 "--add-opens=java.base/java.security.cert=ALL-UNNAMED",

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docker/DockerBuildTask.java

@@ -30,6 +30,7 @@
 import org.gradle.api.tasks.PathSensitivity;
 import org.gradle.api.tasks.TaskAction;
 import org.gradle.process.ExecOperations;
+import org.gradle.process.ExecSpec;
 import org.gradle.workers.WorkAction;
 import org.gradle.workers.WorkParameters;
 import org.gradle.workers.WorkerExecutor;
@@ -166,6 +167,7 @@ private void pullBaseImage(String baseImage) {
             for (int attempt = 1; attempt <= maxAttempts; attempt++) {
                 try {
                     LoggedExec.exec(execOperations, spec -> {
+                        maybeConfigureDockerConfig(spec);
                         spec.executable("docker");
                         spec.args("pull");
                         spec.args(baseImage);
@@ -181,6 +183,13 @@ private void pullBaseImage(String baseImage) {
             throw new GradleException("Failed to pull Docker base image [" + baseImage + "], all attempts failed");
         }
 
+        private void maybeConfigureDockerConfig(ExecSpec spec) {
+            String dockerConfig = System.getenv("DOCKER_CONFIG");
+            if (dockerConfig != null) {
+                spec.environment("DOCKER_CONFIG", dockerConfig);
+            }
+        }
+
         @Override
         public void execute() {
             final Parameters parameters = getParameters();
@@ -193,6 +202,8 @@ public void execute() {
             final boolean isCrossPlatform = isCrossPlatform();
 
             LoggedExec.exec(execOperations, spec -> {
+                maybeConfigureDockerConfig(spec);
+
                 spec.executable("docker");
 
                 if (isCrossPlatform) {

build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/header/InjectHeaderTests.java

@@ -26,9 +26,9 @@ public class InjectHeaderTests extends InjectFeatureTests {
 
     private static final Map<String, String> headers = Map.of(
         "Content-Type",
-        "application/vnd.elasticsearch+json;compatible-with=7",
+        "application/vnd.elasticsearch+json;compatible-with=8",
         "Accept",
-        "application/vnd.elasticsearch+json;compatible-with=7"
+        "application/vnd.elasticsearch+json;compatible-with=8"
     );
 
     /**

distribution/docker/src/docker/Dockerfile

@@ -163,9 +163,16 @@ RUN <%= retry.loop(package_manager,
           "      ${package_manager} update && \n" +
           "      ${package_manager} upgrade && \n" +
           "      ${package_manager} add --no-cache \n" +
-          "        bash ca-certificates curl libsystemd netcat-openbsd p11-kit p11-kit-trust shadow tini unzip zip zstd && \n" +
+          "        bash java-cacerts curl libstdc++ libsystemd netcat-openbsd p11-kit p11-kit-trust posix-libc-utils shadow tini unzip zip zstd && \n" +
           "      rm -rf /var/cache/apk/* "
      ) %>
+
+# Set Bash as the default shell for future commands
+SHELL ["/bin/bash", "-c"]
+
+# Optionally set Bash as the default shell in the container at runtime
+CMD ["/bin/bash"]
+
 <% } else if (docker_base == "default" || docker_base == "cloud") { %>
 
 # Change default shell to bash, then install required packages with retries.
@@ -224,7 +231,7 @@ COPY --from=builder --chown=0:0 /opt /opt
 <% } %>
 
 ENV PATH /usr/share/elasticsearch/bin:\$PATH
-
+ENV SHELL /bin/bash
 COPY ${bin_dir}/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
 
 # 1. Sync the user and group permissions of /etc/passwd
@@ -249,6 +256,8 @@ RUN chmod g=u /etc/passwd && \\
 # stays up-to-date with changes to Ubuntu's store)
 COPY bin/docker-openjdk /etc/ca-certificates/update.d/docker-openjdk
 RUN /etc/ca-certificates/update.d/docker-openjdk
+<% } else if (docker_base == 'wolfi') { %>
+RUN ln -sf /etc/ssl/certs/java/cacerts /usr/share/elasticsearch/jdk/lib/security/cacerts
 <% } else { %>
 RUN ln -sf /etc/pki/ca-trust/extracted/java/cacerts /usr/share/elasticsearch/jdk/lib/security/cacerts
 <% } %>