Merge branch 'main' into batch-the-chunks

# Conflicts:
#	x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceTests.java

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionModuleCheckTaskProvider.java

@@ -48,10 +48,11 @@ public class InternalDistributionModuleCheckTaskProvider {
     /** ES jars in the lib directory that are not modularized. For now, es-log4j is the only one. */
     private static final List<String> ES_JAR_EXCLUDES = List.of("elasticsearch-log4j");
 
-    /** List of the current Elasticsearch Java Modules, by name. */
+    /** List of the current Elasticsearch Java Modules, alphabetically by name. */
     private static final List<String> EXPECTED_ES_SERVER_MODULES = List.of(
         "org.elasticsearch.base",
         "org.elasticsearch.cli",
+        "org.elasticsearch.entitlement",
         "org.elasticsearch.geo",
         "org.elasticsearch.grok",
         "org.elasticsearch.logging",

build-tools/src/main/java/org/elasticsearch/gradle/testclusters/RunTask.java

@@ -42,6 +42,7 @@ public abstract class RunTask extends DefaultTestClustersTask {
 
     private Boolean debug = false;
     private Boolean cliDebug = false;
+    private Boolean entitlementsEnabled = false;
     private Boolean apmServerEnabled = false;
 
     private Boolean preserveData = false;
@@ -69,6 +70,14 @@ public void setCliDebug(boolean enabled) {
         this.cliDebug = enabled;
     }
 
+    @Option(
+        option = "entitlements",
+        description = "Use the Entitlements agent system in place of SecurityManager to enforce sandbox policies."
+    )
+    public void setEntitlementsEnabled(boolean enabled) {
+        this.entitlementsEnabled = enabled;
+    }
+
     @Input
     public Boolean getDebug() {
         return debug;
@@ -79,6 +88,11 @@ public Boolean getCliDebug() {
         return cliDebug;
     }
 
+    @Input
+    public Boolean getEntitlementsEnabled() {
+        return entitlementsEnabled;
+    }
+
     @Input
     public Boolean getApmServerEnabled() {
         return apmServerEnabled;
@@ -226,6 +240,9 @@ else if (node.getSettingKeys().contains("telemetry.metrics.enabled") == false) {
         if (cliDebug) {
             enableCliDebug();
         }
+        if (entitlementsEnabled) {
+            enableEntitlements();
+        }
     }
 
     @TaskAction

build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersAware.java

@@ -74,4 +74,12 @@ default void enableCliDebug() {
             }
         }
     }
+
+    default void enableEntitlements() {
+        for (ElasticsearchCluster cluster : getClusters()) {
+            for (ElasticsearchNode node : cluster.getNodes()) {
+                node.cliJvmArgs("-Des.entitlements.enabled=true");
+            }
+        }
+    }
 }

distribution/build.gradle

@@ -262,7 +262,7 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
    *             Properties to expand when copying packaging files             *
    *****************************************************************************/
   configurations {
-    ['libs', 'libsVersionChecker', 'libsCliLauncher', 'libsServerCli', 'libsWindowsServiceCli', 'libsPluginCli', 'libsKeystoreCli', 'libsSecurityCli', 'libsGeoIpCli', 'libsAnsiConsole', 'libsNative'].each {
+    ['libs', 'libsVersionChecker', 'libsCliLauncher', 'libsServerCli', 'libsWindowsServiceCli', 'libsPluginCli', 'libsKeystoreCli', 'libsSecurityCli', 'libsGeoIpCli', 'libsAnsiConsole', 'libsNative', 'libsEntitlementAgent', 'libsEntitlementBridge'].each {
       create(it) {
         canBeConsumed = false
         canBeResolved = true
@@ -292,6 +292,8 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
     libsSecurityCli project(':x-pack:plugin:security:cli')
     libsGeoIpCli project(':distribution:tools:geoip-cli')
     libsNative project(':libs:native:native-libraries')
+    libsEntitlementAgent project(':libs:entitlement:agent')
+    libsEntitlementBridge project(':libs:entitlement:bridge')
   }
 
   project.ext {
@@ -336,6 +338,12 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
             include (os + '-' + architecture + '/*')
           }
         }
+        into('entitlement-agent') {
+          from(configurations.libsEntitlementAgent)
+        }
+        into('entitlement-bridge') {
+          from(configurations.libsEntitlementBridge)
+        }
       }
     }
 

distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java

@@ -12,18 +12,20 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
 import java.util.List;
 import java.util.Map;
-import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
 final class SystemJvmOptions {
 
     static List<String> systemJvmOptions(Settings nodeSettings, final Map<String, String> sysprops) {
         String distroType = sysprops.get("es.distribution.type");
         boolean isHotspot = sysprops.getOrDefault("sun.management.compiler", "").contains("HotSpot");
-
-        return Stream.concat(
+        boolean useEntitlements = Boolean.parseBoolean(sysprops.getOrDefault("es.entitlements.enabled", "false"));
+        return Stream.of(
             Stream.of(
                 /*
                  * Cache ttl in seconds for positive DNS lookups noting that this overrides the JDK security property
@@ -35,8 +37,6 @@ static List<String> systemJvmOptions(Settings nodeSettings, final Map<String, St
                  * networkaddress.cache.negative ttl; set to -1 to cache forever.
                  */
                 "-Des.networkaddress.cache.negative.ttl=10",
-                // Allow to set the security manager.
-                "-Djava.security.manager=allow",
                 // pre-touch JVM emory pages during initialization
                 "-XX:+AlwaysPreTouch",
                 // explicitly set the stack size
@@ -61,15 +61,17 @@ static List<String> systemJvmOptions(Settings nodeSettings, final Map<String, St
                 "-Dlog4j2.disable.jmx=true",
                 "-Dlog4j2.formatMsgNoLookups=true",
                 "-Djava.locale.providers=CLDR",
-                maybeEnableNativeAccess(),
-                maybeOverrideDockerCgroup(distroType),
-                maybeSetActiveProcessorCount(nodeSettings),
-                setReplayFile(distroType, isHotspot),
                 // Pass through distribution type
                 "-Des.distribution.type=" + distroType
             ),
-            maybeWorkaroundG1Bug()
-        ).filter(e -> e.isEmpty() == false).collect(Collectors.toList());
+            maybeEnableNativeAccess(),
+            maybeOverrideDockerCgroup(distroType),
+            maybeSetActiveProcessorCount(nodeSettings),
+            maybeSetReplayFile(distroType, isHotspot),
+            maybeWorkaroundG1Bug(),
+            maybeAllowSecurityManager(),
+            maybeAttachEntitlementAgent(useEntitlements)
+        ).flatMap(s -> s).toList();
     }
 
     /*
@@ -86,42 +88,42 @@ static List<String> systemJvmOptions(Settings nodeSettings, final Map<String, St
      * that cgroup statistics are available for the container this process
      * will run in.
      */
-    private static String maybeOverrideDockerCgroup(String distroType) {
+    private static Stream<String> maybeOverrideDockerCgroup(String distroType) {
         if ("docker".equals(distroType)) {
-            return "-Des.cgroups.hierarchy.override=/";
+            return Stream.of("-Des.cgroups.hierarchy.override=/");
         }
-        return "";
+        return Stream.empty();
     }
 
-    private static String setReplayFile(String distroType, boolean isHotspot) {
+    private static Stream<String> maybeSetReplayFile(String distroType, boolean isHotspot) {
         if (isHotspot == false) {
             // the replay file option is only guaranteed for hotspot vms
-            return "";
+            return Stream.empty();
         }
         String replayDir = "logs";
         if ("rpm".equals(distroType) || "deb".equals(distroType)) {
             replayDir = "/var/log/elasticsearch";
         }
-        return "-XX:ReplayDataFile=" + replayDir + "/replay_pid%p.log";
+        return Stream.of("-XX:ReplayDataFile=" + replayDir + "/replay_pid%p.log");
     }
 
     /*
      * node.processors determines thread pool sizes for Elasticsearch. When it
      * is set, we need to also tell the JVM to respect a different value
      */
-    private static String maybeSetActiveProcessorCount(Settings nodeSettings) {
+    private static Stream<String> maybeSetActiveProcessorCount(Settings nodeSettings) {
         if (EsExecutors.NODE_PROCESSORS_SETTING.exists(nodeSettings)) {
             int allocated = EsExecutors.allocatedProcessors(nodeSettings);
-            return "-XX:ActiveProcessorCount=" + allocated;
+            return Stream.of("-XX:ActiveProcessorCount=" + allocated);
         }
-        return "";
+        return Stream.empty();
     }
 
-    private static String maybeEnableNativeAccess() {
+    private static Stream<String> maybeEnableNativeAccess() {
         if (Runtime.version().feature() >= 21) {
-            return "--enable-native-access=org.elasticsearch.nativeaccess,org.apache.lucene.core";
+            return Stream.of("--enable-native-access=org.elasticsearch.nativeaccess,org.apache.lucene.core");
         }
-        return "";
+        return Stream.empty();
     }
 
     /*
@@ -134,4 +136,37 @@ private static Stream<String> maybeWorkaroundG1Bug() {
         }
         return Stream.of();
     }
+
+    private static Stream<String> maybeAllowSecurityManager() {
+        // Will become conditional on useEntitlements once entitlements can run without SM
+        return Stream.of("-Djava.security.manager=allow");
+    }
+
+    private static Stream<String> maybeAttachEntitlementAgent(boolean useEntitlements) {
+        if (useEntitlements == false) {
+            return Stream.empty();
+        }
+
+        Path dir = Path.of("lib", "entitlement-bridge");
+        if (Files.exists(dir) == false) {
+            throw new IllegalStateException("Directory for entitlement bridge jar does not exist: " + dir);
+        }
+        String bridgeJar;
+        try (var s = Files.list(dir)) {
+            var candidates = s.limit(2).toList();
+            if (candidates.size() != 1) {
+                throw new IllegalStateException("Expected one jar in " + dir + "; found " + candidates.size());
+            }
+            bridgeJar = candidates.get(0).toString();
+        } catch (IOException e) {
+            throw new IllegalStateException("Failed to list entitlement jars in: " + dir, e);
+        }
+        return Stream.of(
+            "-Des.entitlements.enabled=true",
+            "-XX:+EnableDynamicAgentLoading",
+            "-Djdk.attach.allowAttachSelf=true",
+            "--patch-module=java.base=" + bridgeJar,
+            "--add-exports=java.base/org.elasticsearch.entitlement.bridge=org.elasticsearch.entitlement"
+        );
+    }
 }

docs/changelog/114879.yaml

@@ -0,0 +1,5 @@
+pr: 114879
+summary: Add refresh `.security` index call between security migrations
+area: Security
+type: enhancement
+issues: []

docs/changelog/115876.yaml

@@ -0,0 +1,5 @@
+pr: 115876
+summary: Inference duration and error metrics
+area: Machine Learning
+type: enhancement
+issues: []

docs/changelog/116082.yaml

@@ -0,0 +1,5 @@
+pr: 116082
+summary: Add support for bitwise inner-product in painless
+area: Vector Search
+type: enhancement
+issues: []

docs/changelog/116128.yaml

@@ -0,0 +1,5 @@
+pr: 116128
+summary: Add num docs and size to logsdb telemetry
+area: Logs
+type: enhancement
+issues: []

docs/changelog/116174.yaml

@@ -0,0 +1,7 @@
+pr: 116174
+summary: Handle with `illegalArgumentExceptions` negative values in HDR percentile
+  aggregations
+area: Aggregations
+type: bug
+issues:
+ - 115777

docs/changelog/116219.yaml

@@ -0,0 +1,6 @@
+pr: 116219
+summary: "[apm-data] Apply lazy rollover on index template creation"
+area: Data streams
+type: bug
+issues:
+ - 116230

docs/changelog/116259.yaml

@@ -0,0 +1,14 @@
+pr: 116259
+summary: Fix `_type` deprecation on simulate pipeline API
+area: Ingest Node
+type: deprecation
+issues: []
+deprecation:
+  title: Document `_type` deprecated on simulate pipeline API
+  area: REST API
+  details: >-
+    Passing a document with a `_type` property is deprecated in the `/_ingest/pipeline/{id}/_simulate` and
+    `/_ingest/pipeline/_simulate` APIs.
+  impact: >-
+    Users should already have stopped using mapping types, which were deprecated in {es} 7. This deprecation warning
+    will fire if they specify mapping types on documents pass to the simulate pipeline API.

docs/changelog/116266.yaml

@@ -0,0 +1,5 @@
+pr: 116266
+summary: Align dot prefix validation with Serverless
+area: Indices APIs
+type: bug
+issues: []

docs/reference/aggregations/metrics/percentile-aggregation.asciidoc

@@ -77,8 +77,8 @@ percentiles: `[ 1, 5, 25, 50, 75, 95, 99 ]`. The response will look like this:
 
 As you can see, the aggregation will return a calculated value for each percentile
 in the default range. If we assume response times are in milliseconds, it is
-immediately obvious that the webpage normally loads in 10-725ms, but occasionally
-spikes to 945-985ms.
+immediately obvious that the webpage normally loads in 10-720ms, but occasionally
+spikes to 940-980ms.
 
 Often, administrators are only interested in outliers -- the extreme percentiles.
 We can specify just the percents we are interested in (requested percentiles

docs/reference/connector/docs/_connectors-create-client.asciidoc

@@ -3,7 +3,7 @@
 
 To create a new {service-name} connector:
 
-. Navigate to the *Search -> Connectors* page in the Kibana UI.
+. In the Kibana UI, navigate to the *Search -> Content -> Connectors* page from the main menu, or use the {kibana-ref}/kibana-concepts-analysts.html#_finding_your_apps_and_objects[global search field].
 . Follow the instructions to create a new  *{service-name}* self-managed connector.
 
 [discrete#es-connectors-{service-name-stub}-client-create-use-the-api]

docs/reference/connector/docs/_connectors-create-native.asciidoc

@@ -3,7 +3,7 @@
 
 To create a new {service-name} connector:
 
-. Navigate to the *Search -> Connectors* page in the Kibana UI.
+. In the Kibana UI, navigate to the *Search -> Content -> Connectors* page from the main menu, or use the {kibana-ref}/kibana-concepts-analysts.html#_finding_your_apps_and_objects[global search field].
 . Follow the instructions to create a new native  *{service-name}* connector.
 
 For additional operations, see <<es-connectors-usage>>.

docs/reference/connector/docs/connectors-hosted-tutorial-mongo.asciidoc

@@ -123,7 +123,7 @@ Once you're deployment is created, navigate to *Search*.
 The Elastic connector will sync your MongoDB data into a search-optimized Elasticsearch index.
 The first step is to create your index in the Kibana UI.
 
-In the main menu navigate to *Search > Content > Indices*.
+In the main menu, navigate to *Search > Content > Indices*, or use the {kibana-ref}/kibana-concepts-analysts.html#_finding_your_apps_and_objects[global search field].
 
 Follow these steps to create your index:
 
@@ -178,7 +178,7 @@ If all the configuration details are correct, the sync will begin and documents
 
 As soon as your first documents are synced, you can view the documents and inspect the mapping for the index:
 
-* In Kibana, navigate to *Search* > *Content* > *Indices*.
+* In Kibana, navigate to *Search* > *Content* > *Indices* from the main menu, or use the {kibana-ref}/kibana-concepts-analysts.html#_finding_your_apps_and_objects[global search field].
 * Select your index, for example `search-mongo-sample`.
 * Choose the *Documents* tab to view the synced documents.
 Expand a document to view its fields.

docs/reference/connector/docs/connectors-managed-service.asciidoc

@@ -80,7 +80,7 @@ Create a new index to be managed by the connector.
 
 Continue from above, or navigate to the following location within the {kib} UI:
 
-*Search > Content > Elasticsearch indices*
+*Search > Content > Elasticsearch indices* from the main menu, or use the {kibana-ref}/kibana-concepts-analysts.html#_finding_your_apps_and_objects[global search field].
 
 Choose the index to configure, and then choose the *Configuration* tab.