Merge remote-tracking branch 'refs/remotes/not-napoleon/esql-compare-…

…nanos-and-millis' into esql-compare-nanos-and-millis

distribution/src/config/log4j2.properties

@@ -63,7 +63,7 @@ appender.deprecation_rolling.name = deprecation_rolling
 appender.deprecation_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation.json
 appender.deprecation_rolling.layout.type = ECSJsonLayout
 # Intentionally follows a different pattern to above
-appender.deprecation_rolling.layout.dataset = deprecation.elasticsearch
+appender.deprecation_rolling.layout.dataset = elasticsearch.deprecation
 appender.deprecation_rolling.filter.rate_limit.type = RateLimitingFilter
 
 appender.deprecation_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation-%i.json.gz

docs/changelog/117792.yaml

@@ -0,0 +1,6 @@
+pr: 117792
+summary: Address mapping and compute engine runtime field issues
+area: Mapping
+type: bug
+issues:
+ - 117644

docs/changelog/117898.yaml

@@ -0,0 +1,5 @@
+pr: 117898
+summary: Limit size of query
+area: ES|QL
+type: bug
+issues: []

docs/changelog/117914.yaml

@@ -0,0 +1,5 @@
+pr: 117914
+summary: Fix for propagating filters from compound to inner retrievers
+area: Ranking
+type: bug
+issues: []

docs/changelog/117917.yaml

@@ -0,0 +1,5 @@
+pr: 117917
+summary: Add option to store `sparse_vector` outside `_source`
+area: Mapping
+type: feature
+issues: []

docs/changelog/117933.yaml

@@ -0,0 +1,18 @@
+pr: 117933
+summary: Change `deprecation.elasticsearch` keyword to `elasticsearch.deprecation`
+area: Infra/Logging
+type: bug
+issues:
+ - 83251
+breaking:
+  title: Deprecation logging value change for "data_stream.dataset" and "event.dataset"
+  area: Logging
+  details: |-
+    This change modifies the "data_stream.dataset" and "event.dataset" value for deprecation logging
+    to use the value `elasticsearch.deprecation` instead of `deprecation.elasticsearch`. This is now
+    consistent with other values where the name of the service is the first part of the key.
+  impact: |-
+    If you are directly consuming deprecation logs for "data_stream.dataset" and "event.dataset" and filtering on
+    this value, you will need to update your filters to use `elasticsearch.deprecation` instead of
+    `deprecation.elasticsearch`.
+  notable: false

docs/changelog/117953.yaml

@@ -0,0 +1,5 @@
+pr: 117953
+summary: Acquire stats searcher for data stream stats
+area: Data streams
+type: bug
+issues: []

docs/reference/inference/put-inference.asciidoc

@@ -10,7 +10,6 @@ Creates an {infer} endpoint to perform an {infer} task.
 * For built-in models and models uploaded through Eland, the {infer} APIs offer an alternative way to use and manage trained models. However, if you do not plan to use the {infer} APIs to use these models or if you want to use non-NLP models, use the <<ml-df-trained-models-apis>>.
 ====
 
-
 [discrete]
 [[put-inference-api-request]]
 ==== {api-request-title}
@@ -47,6 +46,14 @@ Refer to the service list in the <<put-inference-api-desc,API description sectio
 
 The create {infer} API enables you to create an {infer} endpoint and configure a {ml} model to perform a specific {infer} task.
 
+[IMPORTANT]
+====
+* When creating an inference endpoint, the associated machine learning model is automatically deployed if it is not already running.
+* After creating the endpoint, wait for the model deployment to complete before using it. You can verify the deployment status by using the <<get-trained-models-stats, Get trained model statistics>> API. In the response, look for `"state": "fully_allocated"` and ensure the `"allocation_count"` matches the `"target_allocation_count"`.
+* Avoid creating multiple endpoints for the same model unless required, as each endpoint consumes significant resources.
+====
+
+
 The following services are available through the {infer} API.
 You can find the available task types next to the service name. 
 Click the links to review the configuration details of the services:

docs/reference/mapping/types/sparse-vector.asciidoc

@@ -26,6 +26,23 @@ PUT my-index
 
 See <<semantic-search-elser, semantic search with ELSER>> for a complete example on adding documents to a `sparse_vector` mapped field using ELSER.
 
+[[sparse-vectors-params]]
+==== Parameters for `sparse_vector` fields
+
+The following parameters are accepted by `sparse_vector` fields:
+
+[horizontal]
+
+<<mapping-store,store>>::
+
+Indicates whether the field value should be stored and retrievable independently of the <<mapping-source-field,_source>> field.
+Accepted values: true or false (default).
+The field's data is stored using term vectors, a disk-efficient structure compared to the original JSON input.
+The input map can be retrieved during a search request via the <<search-fields-param,`fields` parameter>>.
+To benefit from reduced disk usage, you must either:
+  * Exclude the field from <<source-filtering, _source>>.
+  * Use <<synthetic-source,synthetic `_source`>>.
+
 [[index-multi-value-sparse-vectors]]
 ==== Multi-value sparse vectors
 

docs/reference/search/retriever.asciidoc

@@ -765,11 +765,11 @@ clauses in a <<query-dsl-bool-query, boolean query>>.
 [[retriever-restrictions]]
 ==== Restrictions on search parameters when specifying a retriever
 
-When a retriever is specified as part of a search, the following elements are not allowed at the top-level.
-Instead they are only allowed as elements of specific retrievers:
+When a retriever is specified as part of a search, the following elements are not allowed at the top-level:
 
 * <<request-body-search-query, `query`>>
 * <<search-api-knn, `knn`>>
 * <<search-after, `search_after`>>
 * <<request-body-search-terminate-after, `terminate_after`>>
 * <<search-sort-param, `sort`>>
+* <<rescore, `rescore`>>

libs/entitlement/src/main/java/module-info.java

@@ -17,6 +17,7 @@
     requires static org.elasticsearch.entitlement.bridge; // At runtime, this will be in java.base
 
     exports org.elasticsearch.entitlement.runtime.api;
+    exports org.elasticsearch.entitlement.runtime.policy;
     exports org.elasticsearch.entitlement.instrumentation;
     exports org.elasticsearch.entitlement.bootstrap to org.elasticsearch.server;
     exports org.elasticsearch.entitlement.initialization to java.base;

libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java

@@ -18,6 +18,8 @@
 import org.elasticsearch.entitlement.instrumentation.MethodKey;
 import org.elasticsearch.entitlement.instrumentation.Transformer;
 import org.elasticsearch.entitlement.runtime.api.ElasticsearchEntitlementChecker;
+import org.elasticsearch.entitlement.runtime.policy.CreateClassLoaderEntitlement;
+import org.elasticsearch.entitlement.runtime.policy.ExitVMEntitlement;
 import org.elasticsearch.entitlement.runtime.policy.Policy;
 import org.elasticsearch.entitlement.runtime.policy.PolicyManager;
 import org.elasticsearch.entitlement.runtime.policy.PolicyParser;
@@ -86,9 +88,11 @@ private static Class<?> internalNameToClass(String internalName) {
     private static PolicyManager createPolicyManager() throws IOException {
         Map<String, Policy> pluginPolicies = createPluginPolicies(EntitlementBootstrap.bootstrapArgs().pluginData());
 
-        // TODO: What should the name be?
         // TODO(ES-10031): Decide what goes in the elasticsearch default policy and extend it
-        var serverPolicy = new Policy("server", List.of());
+        var serverPolicy = new Policy(
+            "server",
+            List.of(new Scope("org.elasticsearch.server", List.of(new ExitVMEntitlement(), new CreateClassLoaderEntitlement())))
+        );
         return new PolicyManager(serverPolicy, pluginPolicies, EntitlementBootstrap.bootstrapArgs().pluginResolver());
     }
 

libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/api/ElasticsearchEntitlementChecker.java

@@ -10,7 +10,6 @@
 package org.elasticsearch.entitlement.runtime.api;
 
 import org.elasticsearch.entitlement.bridge.EntitlementChecker;
-import org.elasticsearch.entitlement.runtime.policy.FlagEntitlementType;
 import org.elasticsearch.entitlement.runtime.policy.PolicyManager;
 
 import java.net.URL;
@@ -30,27 +29,27 @@ public ElasticsearchEntitlementChecker(PolicyManager policyManager) {
 
     @Override
     public void check$java_lang_System$exit(Class<?> callerClass, int status) {
-        policyManager.checkFlagEntitlement(callerClass, FlagEntitlementType.SYSTEM_EXIT);
+        policyManager.checkExitVM(callerClass);
     }
 
     @Override
     public void check$java_net_URLClassLoader$(Class<?> callerClass, URL[] urls) {
-        policyManager.checkFlagEntitlement(callerClass, FlagEntitlementType.CREATE_CLASSLOADER);
+        policyManager.checkCreateClassLoader(callerClass);
     }
 
     @Override
     public void check$java_net_URLClassLoader$(Class<?> callerClass, URL[] urls, ClassLoader parent) {
-        policyManager.checkFlagEntitlement(callerClass, FlagEntitlementType.CREATE_CLASSLOADER);
+        policyManager.checkCreateClassLoader(callerClass);
     }
 
     @Override
     public void check$java_net_URLClassLoader$(Class<?> callerClass, URL[] urls, ClassLoader parent, URLStreamHandlerFactory factory) {
-        policyManager.checkFlagEntitlement(callerClass, FlagEntitlementType.CREATE_CLASSLOADER);
+        policyManager.checkCreateClassLoader(callerClass);
     }
 
     @Override
     public void check$java_net_URLClassLoader$(Class<?> callerClass, String name, URL[] urls, ClassLoader parent) {
-        policyManager.checkFlagEntitlement(callerClass, FlagEntitlementType.CREATE_CLASSLOADER);
+        policyManager.checkCreateClassLoader(callerClass);
     }
 
     @Override
@@ -61,6 +60,6 @@ public ElasticsearchEntitlementChecker(PolicyManager policyManager) {
         ClassLoader parent,
         URLStreamHandlerFactory factory
     ) {
-        policyManager.checkFlagEntitlement(callerClass, FlagEntitlementType.CREATE_CLASSLOADER);
+        policyManager.checkCreateClassLoader(callerClass);
     }
 }

libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/CreateClassLoaderEntitlement.java

@@ -12,5 +12,4 @@
 public class CreateClassLoaderEntitlement implements Entitlement {
     @ExternalEntitlement
     public CreateClassLoaderEntitlement() {}
-
 }

libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/FlagEntitlementType.java → libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/ExitVMEntitlement.java

@@ -9,7 +9,7 @@
 
 package org.elasticsearch.entitlement.runtime.policy;
 
-public enum FlagEntitlementType {
-    SYSTEM_EXIT,
-    CREATE_CLASSLOADER;
-}
+/**
+ * Internal policy type (not-parseable -- not available to plugins).
+ */
+public class ExitVMEntitlement implements Entitlement {}

libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/FileEntitlement.java

@@ -20,6 +20,9 @@ public class FileEntitlement implements Entitlement {
     public static final int READ_ACTION = 0x1;
     public static final int WRITE_ACTION = 0x2;
 
+    public static final String READ = "read";
+    public static final String WRITE = "write";
+
     private final String path;
     private final int actions;
 
@@ -29,12 +32,12 @@ public FileEntitlement(String path, List<String> actionsList) {
         int actionsInt = 0;
 
         for (String actionString : actionsList) {
-            if ("read".equals(actionString)) {
+            if (READ.equals(actionString)) {
                 if ((actionsInt & READ_ACTION) == READ_ACTION) {
                     throw new IllegalArgumentException("file action [read] specified multiple times");
                 }
                 actionsInt |= READ_ACTION;
-            } else if ("write".equals(actionString)) {
+            } else if (WRITE.equals(actionString)) {
                 if ((actionsInt & WRITE_ACTION) == WRITE_ACTION) {
                     throw new IllegalArgumentException("file action [write] specified multiple times");
                 }