Merge branch 'main' into blockFactory

ChrisHegarty · Sep 22, 2023 · 6da7f4c · 6da7f4c
2 parents 61d3196 + e70d774
commit 6da7f4c
Show file tree

Hide file tree

Showing 271 changed files with 2,103 additions and 1,493 deletions.
diff --git a/.buildkite/pipelines/intake.yml b/.buildkite/pipelines/intake.yml
@@ -40,7 +40,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["7.17.14", "8.10.2", "8.11.0"]
+            BWC_VERSION: ["7.17.14", "8.10.3", "8.11.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

diff --git a/.buildkite/pipelines/periodic-packaging.yml b/.buildkite/pipelines/periodic-packaging.yml
@@ -1632,6 +1632,22 @@ steps:
         env:
           BWC_VERSION: 8.10.2
 
+      - label: "{{matrix.image}} / 8.10.3 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh --build-cache -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/ -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.10.3
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            image:
+              - rocky-8
+              - ubuntu-2004
+        agents:
+          provider: gcp
+          image: family/elasticsearch-{{matrix.image}}
+          machineType: custom-16-32768
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 8.10.3
+
       - label: "{{matrix.image}} / 8.11.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh --build-cache -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/ -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.11.0
         timeout_in_minutes: 300

diff --git a/.buildkite/pipelines/periodic.yml b/.buildkite/pipelines/periodic.yml
@@ -1002,6 +1002,16 @@ steps:
           buildDirectory: /dev/shm/bk
         env:
           BWC_VERSION: 8.10.2
+      - label: 8.10.3 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.10.3#bwcTest
+        timeout_in_minutes: 300
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2004
+          machineType: custom-32-98304
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 8.10.3
       - label: 8.11.0 / bwc
         command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.11.0#bwcTest
         timeout_in_minutes: 300

diff --git a/.ci/bwcVersions b/.ci/bwcVersions
@@ -99,4 +99,5 @@ BWC_VERSION:
   - "8.10.0"
   - "8.10.1"
   - "8.10.2"
+  - "8.10.3"
   - "8.11.0"
diff --git a/.ci/snapshotBwcVersions b/.ci/snapshotBwcVersions
@@ -1,4 +1,4 @@
 BWC_VERSION:
   - "7.17.14"
-  - "8.10.2"
+  - "8.10.3"
   - "8.11.0"
diff --git a/...s-internal/src/main/java/org/elasticsearch/gradle/internal/RestrictedBuildApiService.java b/...s-internal/src/main/java/org/elasticsearch/gradle/internal/RestrictedBuildApiService.java
@@ -67,7 +67,6 @@ private static ListMultimap<Class<?>, String> createLegacyRestTestBasePluginUsag
         map.put(LegacyRestTestBasePlugin.class, ":qa:remote-clusters");
         map.put(LegacyRestTestBasePlugin.class, ":qa:repository-multi-version");
         map.put(LegacyRestTestBasePlugin.class, ":qa:rolling-upgrade");
-        map.put(LegacyRestTestBasePlugin.class, ":qa:rolling-upgrade-legacy");
         map.put(LegacyRestTestBasePlugin.class, ":qa:smoke-test-http");
         map.put(LegacyRestTestBasePlugin.class, ":qa:smoke-test-ingest-disabled");
         map.put(LegacyRestTestBasePlugin.class, ":qa:smoke-test-ingest-with-all-dependencies");

diff --git a/build-tools-internal/src/main/resources/checkstyle_suppressions.xml b/build-tools-internal/src/main/resources/checkstyle_suppressions.xml
@@ -32,7 +32,7 @@
 
   <!-- Intentionally have multi line string for a bulk request, otherwise this needs to fallback to string concatenation  -->
   <suppress files="modules[/\\]data-streams[/\\]src[/\\]javaRestTest[/\\]java[/\\]org[/\\]elasticsearch[/\\]datastreams[/\\]TsdbDataStreamRestIT.java" checks="LineLength" />
-  <suppress files="qa[/\\]rolling-upgrade[/\\]src[/\\]javaRestTest[/\\]java[/\\]org[/\\]elasticsearch[/\\]upgrades[/\\]TsdbIT.java" checks="LineLength" />
+  <suppress files="qa[/\\]rolling-upgrade[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]upgrades[/\\]TsdbIT.java" checks="LineLength" />
 
   <!-- Gradle requires inputs to be seriablizable -->
   <suppress files="build-tools-internal[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]gradle[/\\]internal[/\\]precommit[/\\]TestingConventionRule.java" checks="RegexpSinglelineJava" />

diff --git a/build-tools-internal/version.properties b/build-tools-internal/version.properties
@@ -2,7 +2,7 @@ elasticsearch     = 8.11.0
 lucene            = 9.7.0
 
 bundled_jdk_vendor = openjdk
-bundled_jdk = 20.0.2+9@6e380f22cbe7469fa75fb448bd903d8e
+bundled_jdk = 21+35@fd2272bbf8e04c3dbaee13770090416c
 
 # optional dependencies
 spatial4j         = 0.7

diff --git a/docs/changelog/99631.yaml b/docs/changelog/99631.yaml
@@ -0,0 +1,5 @@
+pr: 99631
+summary: Add component info versions to node info in a pluggable way
+area: Infra/REST API
+type: enhancement
+issues: []
diff --git a/docs/changelog/99673.yaml b/docs/changelog/99673.yaml
@@ -0,0 +1,5 @@
+pr: 99673
+summary: Adding retry logic for start model deployment API
+area: Machine Learning
+type: bug
+issues: [ ]
diff --git a/docs/changelog/99724.yaml b/docs/changelog/99724.yaml
@@ -0,0 +1,5 @@
+pr: 99724
+summary: Upgrade bundled JDK to Java 21
+area: Packaging
+type: upgrade
+issues: []
diff --git a/docs/changelog/99726.yaml b/docs/changelog/99726.yaml
@@ -0,0 +1,6 @@
+pr: 99726
+summary: "ESQL: Account for an exception being thrown when building a `BytesRefArrayBlock`"
+area: ES|QL
+type: bug
+issues:
+ - 99472
diff --git a/docs/changelog/99738.yaml b/docs/changelog/99738.yaml
@@ -0,0 +1,6 @@
+pr: 99738
+summary: Ignore "index not found" error when `delete_dest_index` flag is set but the
+  dest index doesn't exist
+area: Transform
+type: bug
+issues: []
diff --git a/docs/reference/cluster/nodes-info.asciidoc b/docs/reference/cluster/nodes-info.asciidoc
@@ -139,6 +139,9 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=node-id]
 `index_version`::
 		The most recent index version that this node can read.
 
+`component_versions`::
+		The version numbers of individual components loaded in this node.
+
 The `os` flag can be set to retrieve information that concern the operating
 system:
 
@@ -236,6 +239,10 @@ The API returns the following response:
       "version": "{version}",
       "transport_version": 100000298,
       "index_version": 100000074,
+      "component_versions": {
+        "ml_config_version": 100000162,
+        "transform_config_version": 100000096
+      },
       "build_flavor": "default",
       "build_type": "{build_type}",
       "build_hash": "587409e",
@@ -276,6 +283,7 @@ The API returns the following response:
 // TESTRESPONSE[s/"ip": "192.168.17"/"ip": $body.$_path/]
 // TESTRESPONSE[s/"transport_version": 100000298/"transport_version": $body.$_path/]
 // TESTRESPONSE[s/"index_version": 100000074/"index_version": $body.$_path/]
+// TESTRESPONSE[s/"component_versions": \{[^\}]*\}/"component_versions": $body.$_path/]
 // TESTRESPONSE[s/"build_hash": "587409e"/"build_hash": $body.$_path/]
 // TESTRESPONSE[s/"roles": \[[^\]]*\]/"roles": $body.$_path/]
 // TESTRESPONSE[s/"attributes": \{[^\}]*\}/"attributes": $body.$_path/]
@@ -311,6 +319,10 @@ The API returns the following response:
       "version": "{version}",
       "transport_version": 100000298,
       "index_version": 100000074,
+      "component_versions": {
+        "ml_config_version": 100000162,
+        "transform_config_version": 100000096
+      },
       "build_flavor": "default",
       "build_type": "{build_type}",
       "build_hash": "587409e",
@@ -375,6 +387,7 @@ The API returns the following response:
 // TESTRESPONSE[s/"ip": "192.168.17"/"ip": $body.$_path/]
 // TESTRESPONSE[s/"transport_version": 100000298/"transport_version": $body.$_path/]
 // TESTRESPONSE[s/"index_version": 100000074/"index_version": $body.$_path/]
+// TESTRESPONSE[s/"component_versions": \{[^\}]*\}/"component_versions": $body.$_path/]
 // TESTRESPONSE[s/"build_hash": "587409e"/"build_hash": $body.$_path/]
 // TESTRESPONSE[s/"roles": \[[^\]]*\]/"roles": $body.$_path/]
 // TESTRESPONSE[s/"attributes": \{[^\}]*\}/"attributes": $body.$_path/]

diff --git a/docs/reference/setup/install.asciidoc b/docs/reference/setup/install.asciidoc
@@ -90,6 +90,14 @@ the bundled JVM are treated as if they were within {es} itself.
 The bundled JVM is located within the `jdk` subdirectory of the {es} home
 directory. You may remove this directory if using your own JVM.
 
+[discrete]
+[[jvm-agents]]
+=== JVM and Java agents
+
+Don't use third-party Java agents that attach to the JVM. These agents
+can reduce {es} performance, including freezing or crashing nodes.
+
+
 include::install/targz.asciidoc[]
 
 include::install/zip-windows.asciidoc[]

diff --git a/docs/reference/troubleshooting/network-timeouts.asciidoc b/docs/reference/troubleshooting/network-timeouts.asciidoc
@@ -1,11 +1,17 @@
 tag::troubleshooting-network-timeouts-gc-vm[]
 * GC pauses are recorded in the GC logs that {es} emits by default, and also
 usually by the `JvmMonitorService` in the main node logs. Use these logs to
-confirm whether or not GC is resulting in delays.
+confirm whether or not the node is experiencing high heap usage with long GC
+pauses. If so, <<high-jvm-memory-pressure,the troubleshooting guide for high
+heap usage>> has some suggestions for further investigation but typically you
+will need to capture a heap dump during a time of high heap usage to fully
+understand the problem.
 
 * VM pauses also affect other processes on the same host. A VM pause also
 typically causes a discontinuity in the system clock, which {es} will report in
-its logs.
+its logs. If you see evidence of other processes pausing at the same time, or
+unexpected clock discontinuities, investigate the infrastructure on which you
+are running {es}.
 end::troubleshooting-network-timeouts-gc-vm[]
 
 tag::troubleshooting-network-timeouts-packet-capture-elections[]

diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml
@@ -1600,6 +1600,27 @@
             <sha256 value="7e5870fd2e19b87cbd1981c4ff7203897384c2eb104977f40ce4951b40ab433e" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="openjdk_21" name="linux" version="21">
+         <artifact name="linux-21-aarch64.tar.gz">
+            <sha256 value="e8f4ed1a69815ddf56d7da365116eefc1e5a1159396dffee3dd21616a86d5d28" origin="Generated by Gradle"/>
+         </artifact>
+         <artifact name="linux-21-x64.tar.gz">
+            <sha256 value="a30c454a9bef8f46d5f1bf3122830014a8fbe7ac03b5f8729bc3add4b92a1d0a" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="openjdk_21" name="macos" version="21">
+         <artifact name="macos-21-aarch64.tar.gz">
+            <sha256 value="f12e1e0a2dffc847951598f597c8ee60fb0913932f24b2b09c62cfd2f0f4dfb9" origin="Generated by Gradle"/>
+         </artifact>
+         <artifact name="macos-21-x64.tar.gz">
+            <sha256 value="af32e84c11009f72f783fdcdc9917efc277893988f097e198e2576875d1e88c1" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="openjdk_21" name="windows" version="21">
+         <artifact name="windows-21-x64.zip">
+            <sha256 value="5434faaf029e66e7ce6e75770ca384de476750984a7d2881ef7686894c4b4944" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.abego.treelayout" name="org.abego.treelayout.core" version="1.0.3">
          <artifact name="org.abego.treelayout.core-1.0.3.jar">
             <sha256 value="fa5e31395c39c2e7d46aca0f81f72060931607b2fa41bd36038eb2cb6fb93326" origin="Generated by Gradle"/>

diff --git a/...s/src/main/java/org/elasticsearch/datastreams/action/DataStreamsStatsTransportAction.java b/...s/src/main/java/org/elasticsearch/datastreams/action/DataStreamsStatsTransportAction.java
@@ -70,7 +70,7 @@ public DataStreamsStatsTransportAction(
             actionFilters,
             indexNameExpressionResolver,
             DataStreamsStatsAction.Request::new,
-            ThreadPool.Names.MANAGEMENT
+            transportService.getThreadPool().executor(ThreadPool.Names.MANAGEMENT)
         );
         this.clusterService = clusterService;
         this.indicesService = indicesService;

diff --git a/...c/main/java/org/elasticsearch/ingest/geoip/stats/GeoIpDownloaderStatsTransportAction.java b/...c/main/java/org/elasticsearch/ingest/geoip/stats/GeoIpDownloaderStatsTransportAction.java
@@ -52,7 +52,7 @@ public GeoIpDownloaderStatsTransportAction(
             actionFilters,
             Request::new,
             NodeRequest::new,
-            ThreadPool.Names.MANAGEMENT
+            threadPool.executor(ThreadPool.Names.MANAGEMENT)
         );
         this.transportService = transportService;
         this.registry = registry;

diff --git a/.../lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java b/.../lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java
@@ -515,7 +515,7 @@ public TransportAction(
                 // Creating a in-memory index is not light weight
                 // TODO: is MANAGEMENT TP the right TP? Right now this is an admin api (see action name).
                 Request::new,
-                ThreadPool.Names.MANAGEMENT
+                threadPool.executor(ThreadPool.Names.MANAGEMENT)
             );
             this.scriptService = scriptService;
             this.indicesServices = indicesServices;

diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportRethrottleAction.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportRethrottleAction.java
@@ -46,7 +46,7 @@ public TransportRethrottleAction(
             RethrottleRequest::new,
             ListTasksResponse::new,
             TaskInfo::from,
-            ThreadPool.Names.MANAGEMENT
+            transportService.getThreadPool().executor(ThreadPool.Names.MANAGEMENT)
         );
         this.client = client;
     }

diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/UpdateByQueryWithScriptTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/UpdateByQueryWithScriptTests.java
@@ -52,8 +52,6 @@ protected UpdateByQueryRequest request() {
     @Override
     protected TransportUpdateByQueryAction.AsyncIndexBySearchAction action(ScriptService scriptService, UpdateByQueryRequest request) {
         TransportService transportService = mock(TransportService.class);
-
-        // TODO: temporary, remove in #97879
         when(transportService.getThreadPool()).thenReturn(threadPool);
 
         TransportUpdateByQueryAction transportAction = new TransportUpdateByQueryAction(

diff --git a/qa/full-cluster-restart/build.gradle b/qa/full-cluster-restart/build.gradle
@@ -13,11 +13,6 @@ apply plugin: 'elasticsearch.internal-java-rest-test'
 apply plugin: 'elasticsearch.internal-test-artifact'
 apply plugin: 'elasticsearch.bwc-test'
 
-test {
-  // CI doesn't like it when there's multiple clusters running at once
-  maxParallelForks = 1
-}
-
 BuildParams.bwcVersions.withIndexCompatible { bwcVersion, baseName ->
   tasks.register(bwcTaskName(bwcVersion), StandaloneRestIntegTestTask) {
     usesBwcDistribution(bwcVersion)