Merge branch 'main' into add-alibabacloud-inference

.buildkite/scripts/lucene-snapshot/update-branch.sh

@@ -2,17 +2,17 @@
 
 set -euo pipefail
 
-if [[ "$BUILDKITE_BRANCH" != "lucene_snapshot" ]]; then
-  echo "Error: This script should only be run on the lucene_snapshot branch"
+if [[ "$BUILDKITE_BRANCH" != "lucene_snapshot"* ]]; then
+  echo "Error: This script should only be run on lucene_snapshot branches"
   exit 1
 fi
 
-echo --- Updating lucene_snapshot branch with main
+echo --- Updating "$BUILDKITE_BRANCH" branch with main
 
 git config --global user.name elasticsearchmachine
 git config --global user.email '[email protected]'
 
-git checkout lucene_snapshot
+git checkout "$BUILDKITE_BRANCH"
 git fetch origin main
 git merge --no-edit origin/main
-git push origin lucene_snapshot
+git push origin "$BUILDKITE_BRANCH"

.buildkite/scripts/lucene-snapshot/update-es-snapshot.sh

@@ -2,8 +2,8 @@
 
 set -euo pipefail
 
-if [[ "$BUILDKITE_BRANCH" != "lucene_snapshot" ]]; then
-  echo "Error: This script should only be run on the lucene_snapshot branch"
+if [[ "$BUILDKITE_BRANCH" != "lucene_snapshot"* ]]; then
+  echo "Error: This script should only be run on the lucene_snapshot branches"
   exit 1
 fi
 

benchmarks/build.gradle

@@ -77,7 +77,7 @@ tasks.named("run").configure {
   executable = "${BuildParams.runtimeJavaHome}/bin/java"
   args << "-Dplugins.dir=${buildDir}/plugins" << "-Dtests.index=${buildDir}/index"
   dependsOn "copyExpression", "copyPainless"
-  systemProperty 'java.library.path', file("../libs/native/libraries/build/platform/${platformName()}-${os.arch}")
+  systemProperty 'es.nativelibs.path', file("../libs/native/libraries/build/platform/${platformName()}-${os.arch}")
 }
 
 String platformName() {

build-tools-internal/src/main/groovy/elasticsearch.ide.gradle

@@ -168,8 +168,7 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') {
               '-ea',
               '-Djava.security.manager=allow',
               '-Djava.locale.providers=SPI,COMPAT',
-              '-Djava.library.path=' + testLibraryPath,
-              '-Djna.library.path=' + testLibraryPath,
+              '-Des.nativelibs.path=' + testLibraryPath,
               // TODO: only open these for mockito when it is modularized
               '--add-opens=java.base/java.security.cert=ALL-UNNAMED',
               '--add-opens=java.base/java.nio.channels=ALL-UNNAMED',

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaBasePlugin.java

@@ -189,9 +189,7 @@ private static void configureNativeLibraryPath(Project project) {
             var libraryPath = (Supplier<String>) () -> TestUtil.getTestLibraryPath(nativeConfigFiles.getAsPath());
 
             test.dependsOn(nativeConfigFiles);
-            // we may use JNA or the JDK's foreign function api to load libraries, so we set both sysprops
-            systemProperties.systemProperty("java.library.path", libraryPath);
-            systemProperties.systemProperty("jna.library.path", libraryPath);
+            systemProperties.systemProperty("es.nativelibs.path", libraryPath);
         });
     }
 

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/TestUtil.java

@@ -11,16 +11,14 @@
 import org.elasticsearch.gradle.Architecture;
 import org.elasticsearch.gradle.ElasticsearchDistribution;
 
-import java.io.File;
 import java.util.Locale;
 
 public class TestUtil {
 
     public static String getTestLibraryPath(String nativeLibsDir) {
         String arch = Architecture.current().toString().toLowerCase(Locale.ROOT);
         String platform = String.format(Locale.ROOT, "%s-%s", ElasticsearchDistribution.CURRENT_PLATFORM, arch);
-        String existingLibraryPath = System.getProperty("java.library.path");
 
-        return String.format(Locale.ROOT, "%s/%s%c%s", nativeLibsDir, platform, File.pathSeparatorChar, existingLibraryPath);
+        return String.format(Locale.ROOT, "%s/%s", nativeLibsDir, platform);
     }
 }

catalog-info.yaml

@@ -125,7 +125,7 @@ spec:
         ELASTIC_SLACK_NOTIFICATIONS_ENABLED: "true"
         SLACK_NOTIFICATIONS_CHANNEL: "#lucene"
         SLACK_NOTIFICATIONS_ALL_BRANCHES: "true"
-      branch_configuration: lucene_snapshot
+      branch_configuration: lucene_snapshot lucene_snapshot_10
       default_branch: lucene_snapshot
       teams:
         elasticsearch-team: {}
@@ -142,6 +142,10 @@ spec:
           branch: lucene_snapshot
           cronline: "0 2 * * * America/New_York"
           message: "Builds a new lucene snapshot 1x per day"
+        Periodically on lucene_snapshot_10:
+          branch: lucene_snapshot_10
+          cronline: "0 2 * * * America/New_York"
+          message: "Builds a new lucene snapshot 1x per day"
 ---
 # yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json
 apiVersion: backstage.io/v1alpha1
@@ -169,7 +173,7 @@ spec:
         ELASTIC_SLACK_NOTIFICATIONS_ENABLED: "true"
         SLACK_NOTIFICATIONS_CHANNEL: "#lucene"
         SLACK_NOTIFICATIONS_ALL_BRANCHES: "true"
-      branch_configuration: lucene_snapshot
+      branch_configuration: lucene_snapshot lucene_snapshot_10
       default_branch: lucene_snapshot
       teams:
         elasticsearch-team: {}
@@ -186,6 +190,10 @@ spec:
           branch: lucene_snapshot
           cronline: "0 6 * * * America/New_York"
           message: "Merges main into lucene_snapshot branch 1x per day"
+        Periodically on lucene_snapshot_10:
+          branch: lucene_snapshot_10
+          cronline: "0 6 * * * America/New_York"
+          message: "Merges main into lucene_snapshot_10 branch 1x per day"
 ---
 # yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json
 apiVersion: backstage.io/v1alpha1
@@ -213,7 +221,7 @@ spec:
         ELASTIC_SLACK_NOTIFICATIONS_ENABLED: "true"
         SLACK_NOTIFICATIONS_CHANNEL: "#lucene"
         SLACK_NOTIFICATIONS_ALL_BRANCHES: "true"
-      branch_configuration: lucene_snapshot
+      branch_configuration: lucene_snapshot lucene_snapshot_10
       default_branch: lucene_snapshot
       teams:
         elasticsearch-team: {}
@@ -230,6 +238,10 @@ spec:
           branch: lucene_snapshot
           cronline: "0 9,12,15,18 * * * America/New_York"
           message: "Runs tests against lucene_snapshot branch several times per day"
+        Periodically on lucene_snapshot_10:
+          branch: lucene_snapshot_10
+          cronline: "0 9,12,15,18 * * * America/New_York"
+          message: "Runs tests against lucene_snapshot_10 branch several times per day"
 ---
 # yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json
 apiVersion: backstage.io/v1alpha1

distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java

@@ -10,11 +10,7 @@
 
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
-import org.elasticsearch.core.SuppressForbidden;
 
-import java.io.File;
-import java.nio.file.Path;
-import java.nio.file.Paths;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
@@ -25,7 +21,6 @@ final class SystemJvmOptions {
     static List<String> systemJvmOptions(Settings nodeSettings, final Map<String, String> sysprops) {
         String distroType = sysprops.get("es.distribution.type");
         boolean isHotspot = sysprops.getOrDefault("sun.management.compiler", "").contains("HotSpot");
-        String libraryPath = findLibraryPath(sysprops);
 
         return Stream.concat(
             Stream.of(
@@ -73,8 +68,6 @@ static List<String> systemJvmOptions(Settings nodeSettings, final Map<String, St
                 maybeOverrideDockerCgroup(distroType),
                 maybeSetActiveProcessorCount(nodeSettings),
                 setReplayFile(distroType, isHotspot),
-                "-Djava.library.path=" + libraryPath,
-                "-Djna.library.path=" + libraryPath,
                 // Pass through distribution type
                 "-Des.distribution.type=" + distroType
             ),
@@ -144,38 +137,4 @@ private static Stream<String> maybeWorkaroundG1Bug() {
         }
         return Stream.of();
     }
-
-    private static String findLibraryPath(Map<String, String> sysprops) {
-        // working dir is ES installation, so we use relative path here
-        Path platformDir = Paths.get("lib", "platform");
-        String existingPath = sysprops.get("java.library.path");
-        assert existingPath != null;
-
-        String osname = sysprops.get("os.name");
-        String os;
-        if (osname.startsWith("Windows")) {
-            os = "windows";
-        } else if (osname.startsWith("Linux")) {
-            os = "linux";
-        } else if (osname.startsWith("Mac OS")) {
-            os = "darwin";
-        } else {
-            os = "unsupported_os[" + osname + "]";
-        }
-        String archname = sysprops.get("os.arch");
-        String arch;
-        if (archname.equals("amd64") || archname.equals("x86_64")) {
-            arch = "x64";
-        } else if (archname.equals("aarch64")) {
-            arch = archname;
-        } else {
-            arch = "unsupported_arch[" + archname + "]";
-        }
-        return platformDir.resolve(os + "-" + arch).toAbsolutePath() + getPathSeparator() + existingPath;
-    }
-
-    @SuppressForbidden(reason = "no way to get path separator with nio")
-    private static String getPathSeparator() {
-        return File.pathSeparator;
-    }
 }

distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/JvmOptionsParserTests.java

@@ -17,7 +17,6 @@
 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.StringReader;
-import java.nio.file.FileSystems;
 import java.nio.file.Files;
 import java.nio.file.NoSuchFileException;
 import java.nio.file.Path;
@@ -30,12 +29,10 @@
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicBoolean;
 
-import static org.hamcrest.Matchers.allOf;
 import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.hasEntry;
 import static org.hamcrest.Matchers.hasItem;
 import static org.hamcrest.Matchers.hasKey;
 import static org.hamcrest.Matchers.hasSize;
@@ -44,14 +41,7 @@
 @WithoutSecurityManager
 public class JvmOptionsParserTests extends ESTestCase {
 
-    private static final Map<String, String> TEST_SYSPROPS = Map.of(
-        "os.name",
-        "Linux",
-        "os.arch",
-        "aarch64",
-        "java.library.path",
-        "/usr/lib"
-    );
+    private static final Map<String, String> TEST_SYSPROPS = Map.of("os.name", "Linux", "os.arch", "aarch64");
 
     public void testSubstitution() {
         final List<String> jvmOptions = JvmOptionsParser.substitutePlaceholders(
@@ -390,40 +380,4 @@ public void testCommandLineDistributionType() {
         final List<String> jvmOptions = SystemJvmOptions.systemJvmOptions(Settings.EMPTY, sysprops);
         assertThat(jvmOptions, hasItem("-Des.distribution.type=testdistro"));
     }
-
-    public void testLibraryPath() {
-        assertLibraryPath("Mac OS", "aarch64", "darwin-aarch64");
-        assertLibraryPath("Mac OS", "amd64", "darwin-x64");
-        assertLibraryPath("Mac OS", "x86_64", "darwin-x64");
-        assertLibraryPath("Linux", "aarch64", "linux-aarch64");
-        assertLibraryPath("Linux", "amd64", "linux-x64");
-        assertLibraryPath("Linux", "x86_64", "linux-x64");
-        assertLibraryPath("Windows", "amd64", "windows-x64");
-        assertLibraryPath("Windows", "x86_64", "windows-x64");
-        assertLibraryPath("Unknown", "aarch64", "unsupported_os[Unknown]-aarch64");
-        assertLibraryPath("Mac OS", "Unknown", "darwin-unsupported_arch[Unknown]");
-    }
-
-    private void assertLibraryPath(String os, String arch, String expected) {
-        String existingPath = "/usr/lib";
-        var sysprops = Map.of("os.name", os, "os.arch", arch, "java.library.path", existingPath);
-        final List<String> jvmOptions = SystemJvmOptions.systemJvmOptions(Settings.EMPTY, sysprops);
-        Map<String, String> options = new HashMap<>();
-        for (var jvmOption : jvmOptions) {
-            if (jvmOption.startsWith("-D")) {
-                String[] parts = jvmOption.substring(2).split("=");
-                assert parts.length == 2;
-                options.put(parts[0], parts[1]);
-            }
-        }
-        String separator = FileSystems.getDefault().getSeparator();
-        assertThat(
-            options,
-            hasEntry(equalTo("java.library.path"), allOf(containsString("platform" + separator + expected), containsString(existingPath)))
-        );
-        assertThat(
-            options,
-            hasEntry(equalTo("jna.library.path"), allOf(containsString("platform" + separator + expected), containsString(existingPath)))
-        );
-    }
 }

docs/changelog/110633.yaml

@@ -0,0 +1,5 @@
+pr: 110633
+summary: Add manage roles privilege
+area: Authorization
+type: enhancement
+issues: []

docs/changelog/111516.yaml

@@ -0,0 +1,5 @@
+pr: 111516
+summary: Adding support for `allow_partial_search_results` in PIT
+area: Search
+type: enhancement
+issues: []

docs/changelog/111523.yaml

@@ -0,0 +1,5 @@
+pr: 111523
+summary: Search coordinator uses `event.ingested` in cluster state to do rewrites
+area: Search
+type: enhancement
+issues: []

docs/changelog/111947.yaml

@@ -0,0 +1,5 @@
+pr: 111947
+summary: Improve performance of grok pattern cycle detection
+area: Ingest Node
+type: bug
+issues: []

docs/changelog/111950.yaml

@@ -0,0 +1,6 @@
+pr: 111950
+summary: "[ES|QL] Name parameter with leading underscore"
+area: ES|QL
+type: enhancement
+issues:
+ - 111821

docs/changelog/112139.yaml

@@ -0,0 +1,6 @@
+pr: 112139
+summary: Fix NPE when executing doc value queries over shape geometries with empty
+  segments
+area: Geo
+type: bug
+issues: []

docs/changelog/112199.yaml

@@ -0,0 +1,5 @@
+pr: 112199
+summary: Support docvalues only query in shape field
+area: Geo
+type: enhancement
+issues: []

docs/changelog/112200.yaml

@@ -0,0 +1,6 @@
+pr: 112200
+summary: "ES|QL: better validation of GROK patterns"
+area: ES|QL
+type: bug
+issues:
+ - 112111

docs/changelog/112214.yaml

@@ -0,0 +1,5 @@
+pr: 112214
+summary: '`ByteArrayStreamInput:` Return -1 when there are no more bytes to read'
+area: Infra/Core
+type: bug
+issues: []

docs/changelog/112218.yaml

@@ -0,0 +1,9 @@
+pr: 112218
+summary: "ESQL: Fix a bug in `MV_PERCENTILE`"
+area: ES|QL
+type: bug
+issues:
+ - 112193
+ - 112180
+ - 112187
+ - 112188

docs/reference/aggregations/metrics/scripted-metric-aggregation.asciidoc

@@ -6,6 +6,8 @@
 
 A metric aggregation that executes using scripts to provide a metric output.
 
+WARNING: `scripted_metric` is not available in {serverless-full}.
+
 WARNING: Using scripts can result in slower search speeds. See
 <<scripts-and-search-speed>>.
 
@@ -127,7 +129,7 @@ init_script::       Executed prior to any collection of documents. Allows the ag
 +
 In the above example, the `init_script` creates an array `transactions` in the `state` object.
 
-map_script::        Executed once per document collected. This is a required script. 
+map_script::        Executed once per document collected. This is a required script.
 +
 In the above example, the `map_script` checks the value of the type field. If the value is 'sale' the value of the amount field
 is added to the transactions array. If the value of the type field is not 'sale' the negated value of the amount field is added
@@ -282,4 +284,4 @@ params::           Optional. An object whose contents will be passed as variable
 
 If a parent bucket of the scripted metric aggregation does not collect any documents an empty aggregation response will be returned from the
 shard with a `null` value. In this case the `reduce_script`'s `states` variable will contain `null` as a response from that shard.
-`reduce_script`'s should therefore expect and deal with `null` responses from shards.  
+`reduce_script`'s should therefore expect and deal with `null` responses from shards.

docs/reference/ingest/processors/community-id.asciidoc

@@ -23,11 +23,12 @@ configuration is required.
 | `source_port`      | no       | `source.port` | Field containing the source port.
 | `destination_ip`   | no       | `destination.ip` | Field containing the destination IP address.
 | `destination_port` | no       | `destination.port` | Field containing the destination port.
-| `iana_number` | no | `network.iana_number` | Field containing the IANA number. The following protocol numbers are currently supported: `1` ICMP, `2` IGMP, `6` TCP, `17` UDP, `47` GRE, `58` ICMP IPv6, `88` EIGRP, `89` OSPF, `103` PIM, and `132` SCTP.
+| `iana_number` | no | `network.iana_number` | Field containing the IANA number.
 | `icmp_type`        | no       | `icmp.type`   | Field containing the ICMP type.
 | `icmp_code`        | no       | `icmp.code`   | Field containing the ICMP code.
-| `transport`        | no       | `network.transport` | Field containing the transport protocol.
-Used only when the `iana_number` field is not present.
+| `transport`        | no       | `network.transport` | Field containing the transport protocol name or number.
+Used only when the `iana_number` field is not present. The following protocol names are currently supported:
+`ICMP`, `IGMP`, `TCP`, `UDP`, `GRE`, `ICMP IPv6`, `EIGRP`, `OSPF`, `PIM`, and `SCTP`.
 | `target_field`     | no       | `network.community_id` | Output field for the community ID.
 | `seed`             | no       | `0`           | Seed for the community ID hash. Must be between
 0 and 65535 (inclusive). The seed can prevent hash collisions between network domains, such as