Merge remote-tracking branch 'refs/remotes/origin/main' into carlosde…

…lest/semantic-text-hide-inference

# Conflicts:
#	x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextNonDynamicFieldMapperTests.java

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/toolchain/OracleOpenJdkToolchainResolver.java

@@ -26,7 +26,53 @@
 
 public abstract class OracleOpenJdkToolchainResolver extends AbstractCustomJavaToolchainResolver {
 
-    record JdkBuild(JavaLanguageVersion languageVersion, String version, String buildNumber, String hash) {}
+    interface JdkBuild {
+        JavaLanguageVersion languageVersion();
+
+        String url(String os, String arch, String extension);
+    }
+
+    record ReleasedJdkBuild(JavaLanguageVersion languageVersion, String version, String buildNumber, String hash) implements JdkBuild {
+
+        @Override
+        public String url(String os, String arch, String extension) {
+            return "https://download.oracle.com/java/GA/jdk"
+                + version
+                + "/"
+                + hash
+                + "/"
+                + buildNumber
+                + "/GPL/openjdk-"
+                + version
+                + "_"
+                + os
+                + "-"
+                + arch
+                + "_bin."
+                + extension;
+        }
+    }
+
+    record EarlyAccessJdkBuild(JavaLanguageVersion languageVersion, String version, String buildNumber) implements JdkBuild {
+
+        @Override
+        public String url(String os, String arch, String extension) {
+            return "https://download.java.net/java/early_access/jdk"
+                + version
+                + "/"
+                + version
+                + "/GPL/openjdk-"
+                + version
+                + "-ea+"
+                + buildNumber
+                + "_"
+                + os
+                + "-"
+                + arch
+                + "_bin."
+                + extension;
+        }
+    }
 
     private static final Pattern VERSION_PATTERN = Pattern.compile(
         "(\\d+)(\\.\\d+\\.\\d+(?:\\.\\d+)?)?\\+(\\d+(?:\\.\\d+)?)(@([a-f0-9]{32}))?"
@@ -39,7 +85,11 @@ record JdkBuild(JavaLanguageVersion languageVersion, String version, String buil
     );
 
     // package private so it can be replaced by tests
-    List<JdkBuild> builds = List.of(getBundledJdkBuild());
+    List<JdkBuild> builds = List.of(
+        getBundledJdkBuild(),
+        // 23 early access
+        new EarlyAccessJdkBuild(JavaLanguageVersion.of(23), "23", "23")
+    );
 
     private JdkBuild getBundledJdkBuild() {
         String bundledJdkVersion = VersionProperties.getBundledJdkVersion();
@@ -51,7 +101,7 @@ private JdkBuild getBundledJdkBuild() {
         String baseVersion = jdkVersionMatcher.group(1) + (jdkVersionMatcher.group(2) != null ? (jdkVersionMatcher.group(2)) : "");
         String build = jdkVersionMatcher.group(3);
         String hash = jdkVersionMatcher.group(5);
-        return new JdkBuild(bundledJdkMajorVersion, baseVersion, build, hash);
+        return new ReleasedJdkBuild(bundledJdkMajorVersion, baseVersion, build, hash);
     }
 
     /**
@@ -68,24 +118,7 @@ public Optional<JavaToolchainDownload> resolve(JavaToolchainRequest request) {
         String extension = operatingSystem.equals(OperatingSystem.WINDOWS) ? "zip" : "tar.gz";
         String arch = toArchString(request.getBuildPlatform().getArchitecture());
         String os = toOsString(operatingSystem);
-        return Optional.of(
-            () -> URI.create(
-                "https://download.oracle.com/java/GA/jdk"
-                    + build.version
-                    + "/"
-                    + build.hash
-                    + "/"
-                    + build.buildNumber
-                    + "/GPL/openjdk-"
-                    + build.version
-                    + "_"
-                    + os
-                    + "-"
-                    + arch
-                    + "_bin."
-                    + extension
-            )
-        );
+        return Optional.of(() -> URI.create(build.url(os, arch, extension)));
     }
 
     /**
@@ -113,7 +146,7 @@ private JdkBuild findSupportedBuild(JavaToolchainRequest request) {
 
         JavaLanguageVersion languageVersion = javaToolchainSpec.getLanguageVersion().get();
         for (JdkBuild build : builds) {
-            if (build.languageVersion.equals(languageVersion)) {
+            if (build.languageVersion().equals(languageVersion)) {
                 return build;
             }
         }

build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/toolchain/OracleOpenJdkToolchainResolverSpec.groovy

@@ -25,7 +25,8 @@ class OracleOpenJdkToolchainResolverSpec extends AbstractToolchainResolverSpec {
             }
         }
         toolChain.builds = [
-            new OracleOpenJdkToolchainResolver.JdkBuild(JavaLanguageVersion.of(20), "20", "36", "bdc68b4b9cbc4ebcb30745c85038d91d")
+            new OracleOpenJdkToolchainResolver.ReleasedJdkBuild(JavaLanguageVersion.of(20), "20", "36", "bdc68b4b9cbc4ebcb30745c85038d91d"),
+            new OracleOpenJdkToolchainResolver.EarlyAccessJdkBuild(JavaLanguageVersion.of(21), "21", "6")
         ]
         toolChain
     }
@@ -40,7 +41,18 @@ class OracleOpenJdkToolchainResolverSpec extends AbstractToolchainResolverSpec {
          [20, anyVendor(), MAC_OS, AARCH64, "https://download.oracle.com/java/GA/jdk20/bdc68b4b9cbc4ebcb30745c85038d91d/36/GPL/openjdk-20_macos-aarch64_bin.tar.gz"],
          [20, anyVendor(), LINUX, X86_64, "https://download.oracle.com/java/GA/jdk20/bdc68b4b9cbc4ebcb30745c85038d91d/36/GPL/openjdk-20_linux-x64_bin.tar.gz"],
          [20, anyVendor(), LINUX, AARCH64, "https://download.oracle.com/java/GA/jdk20/bdc68b4b9cbc4ebcb30745c85038d91d/36/GPL/openjdk-20_linux-aarch64_bin.tar.gz"],
-         [20, anyVendor(), WINDOWS, X86_64, "https://download.oracle.com/java/GA/jdk20/bdc68b4b9cbc4ebcb30745c85038d91d/36/GPL/openjdk-20_windows-x64_bin.zip"]
+         [20, anyVendor(), WINDOWS, X86_64, "https://download.oracle.com/java/GA/jdk20/bdc68b4b9cbc4ebcb30745c85038d91d/36/GPL/openjdk-20_windows-x64_bin.zip"],
+         // https://download.java.net/java/early_access/jdk23/23/GPL/openjdk-23-ea+23_macos-aarch64_bin.tar.gz
+         [21, ORACLE, MAC_OS, X86_64, "https://download.java.net/java/early_access/jdk21/21/GPL/openjdk-21-ea+6_macos-x64_bin.tar.gz"],
+         [21, ORACLE, MAC_OS, AARCH64, "https://download.java.net/java/early_access/jdk21/21/GPL/openjdk-21-ea+6_macos-aarch64_bin.tar.gz"],
+         [21, ORACLE, LINUX, X86_64, "https://download.java.net/java/early_access/jdk21/21/GPL/openjdk-21-ea+6_linux-x64_bin.tar.gz"],
+         [21, ORACLE, LINUX, AARCH64, "https://download.java.net/java/early_access/jdk21/21/GPL/openjdk-21-ea+6_linux-aarch64_bin.tar.gz"],
+         [21, ORACLE, WINDOWS, X86_64, "https://download.java.net/java/early_access/jdk21/21/GPL/openjdk-21-ea+6_windows-x64_bin.zip"],
+         [21, anyVendor(), MAC_OS, X86_64, "https://download.java.net/java/early_access/jdk21/21/GPL/openjdk-21-ea+6_macos-x64_bin.tar.gz"],
+         [21, anyVendor(), MAC_OS, AARCH64, "https://download.java.net/java/early_access/jdk21/21/GPL/openjdk-21-ea+6_macos-aarch64_bin.tar.gz"],
+         [21, anyVendor(), LINUX, X86_64, "https://download.java.net/java/early_access/jdk21/21/GPL/openjdk-21-ea+6_linux-x64_bin.tar.gz"],
+         [21, anyVendor(), LINUX, AARCH64, "https://download.java.net/java/early_access/jdk21/21/GPL/openjdk-21-ea+6_linux-aarch64_bin.tar.gz"],
+         [21, anyVendor(), WINDOWS, X86_64, "https://download.java.net/java/early_access/jdk21/21/GPL/openjdk-21-ea+6_windows-x64_bin.zip"]
         ]
     }
 

distribution/tools/plugin-cli/build.gradle

@@ -23,8 +23,9 @@ dependencies {
   compileOnly project(":libs:elasticsearch-cli")
   implementation project(":libs:elasticsearch-plugin-api")
   implementation project(":libs:elasticsearch-plugin-scanner")
-  implementation 'org.ow2.asm:asm:9.6'
-  implementation 'org.ow2.asm:asm-tree:9.6'
+  // TODO: asm is picked up from the plugin scanner, we should consolidate so it is not defined twice
+  implementation 'org.ow2.asm:asm:9.7'
+  implementation 'org.ow2.asm:asm-tree:9.7'
 
   api "org.bouncycastle:bcpg-fips:1.0.7.1"
   api "org.bouncycastle:bc-fips:1.0.2.4"

docs/changelog/108306.yaml

@@ -0,0 +1,5 @@
+pr: 108306
+summary: Enable inter-segment concurrency for low cardinality numeric terms aggs
+area: Aggregations
+type: enhancement
+issues: []

docs/changelog/108679.yaml

@@ -0,0 +1,6 @@
+pr: 108679
+summary: Suppress deprecation warnings from ingest pipelines when deleting trained model
+area: Machine Learning
+type: bug
+issues:
+ - 105004

docs/changelog/108693.yaml

@@ -0,0 +1,5 @@
+pr: 108693
+summary: Test pipeline run after reroute
+area: Ingest Node
+type: enhancement
+issues: []

docs/changelog/108736.yaml

@@ -0,0 +1,5 @@
+pr: 108736
+summary: Harden field-caps request dispatcher
+area: Search
+type: bug
+issues: []

docs/changelog/108780.yaml

@@ -0,0 +1,6 @@
+pr: 108780
+summary: Add `continent_code` support to the geoip processor
+area: Ingest Node
+type: enhancement
+issues:
+ - 85820

docs/changelog/108786.yaml

@@ -0,0 +1,5 @@
+pr: 108786
+summary: Make ingest byte stat names more descriptive
+area: Ingest Node
+type: enhancement
+issues: []

docs/changelog/108796.yaml

@@ -0,0 +1,5 @@
+pr: 108796
+summary: Return ingest byte stats even when 0-valued
+area: Ingest Node
+type: enhancement
+issues: []

docs/changelog/108802.yaml

@@ -0,0 +1,5 @@
+pr: 108802
+summary: Fix multithreading copies in lib vec
+area: Vector Search
+type: bug
+issues: []

docs/changelog/108820.yaml

@@ -0,0 +1,5 @@
+pr: 108820
+summary: Allow `LuceneSourceOperator` to early terminate
+area: ES|QL
+type: enhancement
+issues: []

docs/changelog/108822.yaml

@@ -0,0 +1,6 @@
+pr: 108822
+summary: Update ASM to 9.7 for plugin scanner
+area: Infra/Plugins
+type: upgrade
+issues:
+ - 108776

docs/reference/cluster/cluster-info.asciidoc

@@ -207,21 +207,33 @@ pipeline.
 (integer)
 Total number of failed operations for the ingest pipeline.
 
-`ingested_in_bytes`::
-(Optional, integer)
-Total number of bytes of all documents ingested by the pipeline.
-This field is only present on pipelines which are the first to process a document.
-Thus, it is not present on pipelines which only serve as a final pipeline after a default pipeline, a pipeline run after
-a reroute processor, or pipelines in pipeline processors.
-
-`produced_in_bytes`::
-(Optional, integer)
-Total number of bytes of all documents produced by the pipeline.
-This field is only present on pipelines which are the first to process a document.
-Thus, it is not present on pipelines which only serve as a final pipeline after a default pipeline, a pipeline run after
-a reroute processor, or pipelines in pipeline processors.
-In situations where there are subsequent pipelines, the value represents the size of the document after all pipelines
-have run.
+`ingested_as_first_pipeline`::
+(<<byte-units,byte value>>)
+Total ingested size of all documents which were first processed by this pipeline.
+A document is not added to the stat value for this pipeline if it is a final pipeline after a default pipeline, a pipeline
+run after a reroute processor, or is within a pipeline processor.
+Instead, the document size is added to the stat value of the pipeline which initially ingested the document.
+
+`ingested_as_first_pipeline_in_bytes`::
+(integer)
+Total ingested size, in bytes, of all documents which were first processed by this pipeline.
+A document is not added to the stat value for this pipeline if it is a final pipeline after a default pipeline, a pipeline
+run after a reroute processor, or is within a pipeline processor.
+Instead, the document size is added to the stat value of the pipeline which initially ingested the document.
+
+`produced_as_first_pipeline`::
+(<<byte-units,byte value>>)
+Total produced size of all documents which were first processed by this pipeline.
+A document is not added to the stat value for this pipeline if it is a final pipeline after a default pipeline, a pipeline
+run after a reroute processor, or is within a pipeline processor.
+Instead, the document size is added to the stat value of the pipeline which initially ingested the document.
+
+`produced_as_first_pipeline_in_bytes`::
+(integer)
+Total produced size, in bytes, of all documents which were first processed by this pipeline.
+A document is not added to the stat value for this pipeline if it is a final pipeline after a default pipeline, a pipeline
+run after a reroute processor, or is within a pipeline processor.
+Instead, the document size is added to the stat value of the pipeline which initially ingested the document.
 
 `processors`::
 (array of objects)

docs/reference/cluster/nodes-stats.asciidoc

@@ -2643,21 +2643,33 @@ pipeline.
 (integer)
 Total number of failed operations for the ingest pipeline.
 
-`ingested_in_bytes`::
-(Optional, integer)
-Total number of bytes of all documents ingested by the pipeline.
-This field is only present on pipelines which are the first to process a document.
-Thus, it is not present on pipelines which only serve as a final pipeline after a default pipeline, a pipeline run after
-a reroute processor, or pipelines in pipeline processors.
-
-`produced_in_bytes`::
-(Optional, integer)
-Total number of bytes of all documents produced by the pipeline.
-This field is only present on pipelines which are the first to process a document.
-Thus, it is not present on pipelines which only serve as a final pipeline after a default pipeline, a pipeline run after
-a reroute processor, or pipelines in pipeline processors.
-In situations where there are subsequent pipelines, the value represents the size of the document after all pipelines
-have run.
+`ingested_as_first_pipeline`::
+(<<byte-units,byte value>>)
+Total ingested size of all documents which were first processed by this pipeline.
+A document is not added to the stat value for this pipeline if it is a final pipeline after a default pipeline, a pipeline
+run after a reroute processor, or is within a pipeline processor.
+Instead, the document size is added to the stat value of the pipeline which initially ingested the document.
+
+`ingested_as_first_pipeline_in_bytes`::
+(integer)
+Total ingested size, in bytes, of all documents which were first processed by this pipeline.
+A document is not added to the stat value for this pipeline if it is a final pipeline after a default pipeline, a pipeline
+run after a reroute processor, or is within a pipeline processor.
+Instead, the document size is added to the stat value of the pipeline which initially ingested the document.
+
+`produced_as_first_pipeline`::
+(<<byte-units,byte value>>)
+Total produced size of all documents which were first processed by this pipeline.
+A document is not added to the stat value for this pipeline if it is a final pipeline after a default pipeline, a pipeline
+run after a reroute processor, or is within a pipeline processor.
+Instead, the document size is added to the stat value of the pipeline which initially ingested the document.
+
+`produced_as_first_pipeline_in_bytes`::
+(integer)
+Total produced size, in bytes, of all documents which were first processed by this pipeline.
+A document is not added to the stat value for this pipeline if it is a final pipeline after a default pipeline, a pipeline
+run after a reroute processor, or is within a pipeline processor.
+Instead, the document size is added to the stat value of the pipeline which initially ingested the document.
 
 `processors`::
 (array of objects)

docs/reference/ilm/actions/ilm-rollover.asciidoc

@@ -7,6 +7,13 @@ Phases allowed: hot.
 Rolls over a target to a new index when the existing index satisfies
 the specified rollover conditions.
 
+[NOTE]
+====
+When an index is rolled over, the previous index's age is updated to reflect the rollover time. 
+This date, rather than the index's `creation_date`, is used in {ilm} 
+`min_age` phase calculations. <<min-age-calculation,Learn more>>.
+====
+
 IMPORTANT: If the rollover action is used on a <<ccr-put-follow,follower index>>,
 policy execution waits until the leader index rolls over (or is
 <<skipping-rollover, otherwise marked complete>>),
@@ -46,11 +53,11 @@ PUT my-index-000001
 [[ilm-rollover-options]]
 ==== Options
 
-A rollover action must specify at least one max_* condition, it may include zero
-or more min_* conditions. An empty rollover action is invalid.
+A rollover action must specify at least one `max_*` condition, it may include zero
+or more `min_*` conditions. An empty rollover action is invalid.
 
-The index will rollover once any max_* condition is satisfied and all
-min_* conditions are satisfied. Note, however, that empty indices are not rolled
+The index will roll over once any `max_*` condition is satisfied and all
+`min_*` conditions are satisfied. Note, however, that empty indices are not rolled
 over by default.
 
 // tag::rollover-conditions[]
@@ -256,7 +263,7 @@ PUT _ilm/policy/my_policy
 ===== Roll over using multiple conditions
 
 When you specify multiple rollover conditions,
-the index is rolled over when _any_ of the max_* and _all_ of the min_* conditions are met.
+the index is rolled over when _any_ of the `max_*` and _all_ of the `min_*` conditions are met.
 This example rolls the index over if it is at least 7 days old or at least 100 gigabytes,
 but only as long as the index contains at least 1000 documents.
 

docs/reference/ilm/error-handling.asciidoc

@@ -154,11 +154,12 @@ You can use the <<ilm-explain-lifecycle,{ilm-init} Explain API>> to monitor the
 === Common {ilm-init} setting issues
 
 [discrete]
+[[min-age-calculation]]
 ==== How `min_age` is calculated
 
 When setting up an <<set-up-lifecycle-policy,{ilm-init} policy>> or <<getting-started-index-lifecycle-management,automating rollover with {ilm-init}>>, be aware that `min_age` can be relative to either the rollover time or the index creation time.
 
-If you use <<ilm-rollover,{ilm-init} rollover>>, `min_age` is calculated relative to the time the index was rolled over. This is because the <<indices-rollover-index,rollover API>> generates a new index. The `creation_date` of the new index (retrievable via <<indices-get-settings>>) is used in the calculation. If you do not use rollover in the {ilm-init} policy, `min_age` is calculated relative to the `creation_date` of the original index.
+If you use <<ilm-rollover,{ilm-init} rollover>>, `min_age` is calculated relative to the time the index was rolled over. This is because the <<indices-rollover-index,rollover API>> generates a new index and updates the `age` of the previous index to reflect the rollover time. If the index hasn't been rolled over, then the `age` is the same as the `creation_date` for the index.
 
 You can override how `min_age` is calculated using the `index.lifecycle.origination_date` and `index.lifecycle.parse_origination_date` <<ilm-settings,{ilm-init} settings>>.
 

docs/reference/ilm/ilm-index-lifecycle.asciidoc

@@ -43,6 +43,12 @@ a "cold" phase with a minimum age either unset, or >= 10 days.
 The minimum age defaults to zero, which causes {ilm-init} to move indices to the next phase
 as soon as all actions in the current phase complete.
 
+[NOTE]
+====
+If an index has been <<ilm-rollover,rolled over>>, then the `min_age` value is relative to the time 
+the index was rolled over, not the index creation time. <<min-age-calculation,Learn more>>.
+====
+
 If an index has unallocated shards and the <<cluster-health,cluster health status>> is yellow,
 the index can still transition to the next phase according to its {ilm} policy.
 However, because {es} can only perform certain clean up tasks on a green

docs/reference/ilm/ilm-tutorial.asciidoc

@@ -57,7 +57,7 @@ reaches either a `max_primary_shard_size` of 50 gigabytes or a `max_age` of 30 d
 
 [NOTE]
 ====
-The `min_age` value is relative to the rollover time, not the index creation time.
+The `min_age` value is relative to the rollover time, not the index creation time. <<min-age-calculation,Learn more>>.
 ====
 
 You can create the policy through {kib} or with the