Merge branch 'main' into ccs-telemetry-remotes

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionModuleCheckTaskProvider.java

@@ -59,7 +59,6 @@ public class InternalDistributionModuleCheckTaskProvider {
         "org.elasticsearch.nativeaccess",
         "org.elasticsearch.plugin",
         "org.elasticsearch.plugin.analysis",
-        "org.elasticsearch.pluginclassloader",
         "org.elasticsearch.securesm",
         "org.elasticsearch.server",
         "org.elasticsearch.simdvec",

distribution/tools/entitlement-agent/README.md

@@ -0,0 +1,10 @@
+### Entitlement Agent
+
+This is a java agent that instruments sensitive class library methods with calls into the `entitlement-runtime` module to check for permissions granted under the _entitlements_ system.
+
+The entitlements system provides an alternative to the legacy `SecurityManager` system, which is deprecated for removal.
+With this agent, the Elasticsearch server can retain some control over which class library methods can be invoked by which callers.
+
+This module is responsible for inserting the appropriate bytecode to achieve enforcement of the rules governed by the `entitlement-runtime` module.
+
+It is not responsible for permission granting or checking logic. That responsibility lies with `entitlement-runtime`.

distribution/tools/entitlement-agent/build.gradle

@@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+apply plugin: 'elasticsearch.build'
+
+configurations {
+  entitlementRuntime
+}
+
+dependencies {
+  entitlementRuntime project(":libs:elasticsearch-entitlement-runtime")
+  implementation project(":libs:elasticsearch-entitlement-runtime")
+  testImplementation project(":test:framework")
+}
+
+tasks.named('test').configure {
+  dependsOn('jar')
+  jvmArgs "-javaagent:${ tasks.named('jar').flatMap{ it.archiveFile }.get()}"
+}
+
+tasks.named('jar').configure {
+  manifest {
+    attributes(
+      'Premain-Class': 'org.elasticsearch.entitlement.agent.EntitlementAgent'
+      , 'Can-Retransform-Classes': 'true'
+    )
+  }
+}
+
+tasks.named('forbiddenApisMain').configure {
+  replaceSignatureFiles 'jdk-signatures'
+}
+

libs/plugin-classloader/src/main/java/module-info.java → distribution/tools/entitlement-agent/src/main/java/module-info.java

@@ -7,6 +7,7 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-module org.elasticsearch.pluginclassloader {
-    exports org.elasticsearch.plugins.loader;
+module org.elasticsearch.entitlement.agent {
+    requires java.instrument;
+    requires org.elasticsearch.entitlement.runtime;
 }

server/src/main/java/org/elasticsearch/cluster/ack/AckedRequest.java → distribution/tools/entitlement-agent/src/main/java/org/elasticsearch/entitlement/agent/EntitlementAgent.java

@@ -7,22 +7,15 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-package org.elasticsearch.cluster.ack;
+package org.elasticsearch.entitlement.agent;
 
-import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.entitlement.runtime.api.EntitlementChecks;
 
-/**
- * Identifies a cluster state update request with acknowledgement support
- */
-public interface AckedRequest {
+import java.lang.instrument.Instrumentation;
 
-    /**
-     * Returns the acknowledgement timeout
-     */
-    TimeValue ackTimeout();
+public class EntitlementAgent {
 
-    /**
-     * Returns the timeout for the request to be completed on the master node
-     */
-    TimeValue masterNodeTimeout();
+    public static void premain(String agentArgs, Instrumentation inst) throws Exception {
+        EntitlementChecks.setAgentBooted();
+    }
 }

distribution/tools/entitlement-agent/src/test/java/org/elasticsearch/entitlement/agent/EntitlementAgentTests.java

@@ -0,0 +1,29 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.entitlement.agent;
+
+import org.elasticsearch.entitlement.runtime.api.EntitlementChecks;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.ESTestCase.WithoutSecurityManager;
+
+/**
+ * This is an end-to-end test that runs with the javaagent installed.
+ * It should exhaustively test every instrumented method to make sure it passes with the entitlement
+ * and fails without it.
+ * See {@code build.gradle} for how we set the command line arguments for this test.
+ */
+@WithoutSecurityManager
+public class EntitlementAgentTests extends ESTestCase {
+
+    public void testAgentBooted() {
+        assertTrue(EntitlementChecks.isAgentBooted());
+    }
+
+}

docs/changelog/112768.yaml

@@ -0,0 +1,5 @@
+pr: 112768
+summary: Deduplicate Kuromoji User Dictionary
+area: Search
+type: enhancement
+issues: []

docs/changelog/113102.yaml

@@ -0,0 +1,5 @@
+pr: 113102
+summary: Trigger merges after recovery
+area: Recovery
+type: enhancement
+issues: []

docs/changelog/113103.yaml

@@ -0,0 +1,6 @@
+pr: 113103
+summary: "ESQL: Align year diffing to the rest of the units in DATE_DIFF: chronological"
+area: ES|QL
+type: bug
+issues:
+ - 112482

docs/changelog/113123.yaml

@@ -0,0 +1,6 @@
+pr: 113123
+summary: "ES|QL: Skip CASE function from `InferIsNotNull` rule checks"
+area: ES|QL
+type: bug
+issues:
+ - 112704

docs/plugins/analysis-kuromoji.asciidoc

@@ -133,6 +133,11 @@ unknown words. It can be set to:
 
     Whether punctuation should be discarded from the output. Defaults to `true`.
 
+`lenient`::
+
+    Whether the `user_dictionary` should be deduplicated on the provided `text`.
+    False by default causing duplicates to generate an error.
+
 `user_dictionary`::
 +
 --
@@ -221,7 +226,8 @@ PUT kuromoji_sample
             "type": "kuromoji_tokenizer",
             "mode": "extended",
             "discard_punctuation": "false",
-            "user_dictionary": "userdict_ja.txt"
+            "user_dictionary": "userdict_ja.txt",
+            "lenient": "true"
           }
         },
         "analyzer": {

docs/plugins/analysis-nori.asciidoc

@@ -58,6 +58,11 @@ It can be set to:
 
     Whether punctuation should be discarded from the output. Defaults to `true`.
 
+`lenient`::
+
+    Whether the `user_dictionary` should be deduplicated on the provided `text`.
+    False by default causing duplicates to generate an error.
+
 `user_dictionary`::
 +
 --
@@ -104,7 +109,8 @@ PUT nori_sample
             "type": "nori_tokenizer",
             "decompound_mode": "mixed",
             "discard_punctuation": "false",
-            "user_dictionary": "userdict_ko.txt"
+            "user_dictionary": "userdict_ko.txt",
+            "lenient": "true"
           }
         },
         "analyzer": {
@@ -299,7 +305,6 @@ Which responds with:
 }
 --------------------------------------------------
 
-
 [[analysis-nori-speech]]
 ==== `nori_part_of_speech` token filter
 

docs/reference/inference/service-elasticsearch.asciidoc

@@ -179,6 +179,7 @@ PUT _inference/text_embedding/my-e5-model
       "min_number_of_allocations": 3,
       "max_number_of_allocations": 10
     },
+    "num_threads": 1,
     "model_id": ".multilingual-e5-small"
   }
 }

docs/reference/inference/service-elser.asciidoc

@@ -147,7 +147,8 @@ PUT _inference/sparse_embedding/my-elser-model
       "enabled": true,
       "min_number_of_allocations": 3,
       "max_number_of_allocations": 10
-    }
+    },
+    "num_threads": 1
   }
 }
 ------------------------------------------------------------

docs/reference/query-dsl/sparse-vector-query.asciidoc

@@ -104,7 +104,7 @@ Default: `5`.
 `tokens_weight_threshold`::
 (Optional, float)
 preview:[]
-Tokens whose weight is less than `tokens_weight_threshold` are considered nonsignificant and pruned.
+Tokens whose weight is less than `tokens_weight_threshold` are considered insignificant and pruned.
 This value must be between 0 and 1.
 Default: `0.4`.
 

docs/reference/query-dsl/text-expansion-query.asciidoc

@@ -68,7 +68,7 @@ Default: `5`.
 `tokens_weight_threshold`::
 (Optional, float)
 preview:[]
-Tokens whose weight is less than `tokens_weight_threshold` are considered nonsignificant and pruned.
+Tokens whose weight is less than `tokens_weight_threshold` are considered insignificant and pruned.
 This value must be between 0 and 1.
 Default: `0.4`.
 

docs/reference/query-dsl/weighted-tokens-query.asciidoc

@@ -58,7 +58,7 @@ This value must between 1 and 100.
 Default: `5`.
 
 `tokens_weight_threshold`::
-(Optional, float) Tokens whose weight is less than `tokens_weight_threshold` are considered nonsignificant and pruned.
+(Optional, float) Tokens whose weight is less than `tokens_weight_threshold` are considered insignificant and pruned.
 This value must be between 0 and 1.
 Default: `0.4`.
 

docs/reference/release-notes/8.15.0.asciidoc

@@ -35,6 +35,7 @@ can be configured using the https://www.elastic.co/guide/en/elasticsearch/refere
 ** These indices have many conflicting field mappings
 ** Many of those fields are included in the request
 These issues deplete heap memory, increasing the likelihood of OOM errors. (issue: {es-issue}111964[#111964], {es-issue}111358[#111358]).
+In Kibana, you might indirectly execute these queries when using Discover, or adding a Field Statistics panel to a dashboard.
 +
 To work around this issue, you have a number of options:
 ** Downgrade to an earlier version

docs/reference/release-notes/8.15.1.asciidoc

@@ -15,6 +15,7 @@ can be configured using the https://www.elastic.co/guide/en/elasticsearch/refere
 ** These indices have many conflicting field mappings
 ** Many of those fields are included in the request
 These issues deplete heap memory, increasing the likelihood of OOM errors. (issue: {es-issue}111964[#111964], {es-issue}111358[#111358]).
+In Kibana, you might indirectly execute these queries when using Discover, or adding a Field Statistics panel to a dashboard.
 +
 To work around this issue, you have a number of options:
 ** Downgrade to an earlier version
@@ -23,6 +24,14 @@ To work around this issue, you have a number of options:
 <<esql-kibana-enable,disable ES|QL queries in {kib}>>
 ** Change the default data view in Discover to a smaller set of indices and/or one with fewer mapping conflicts.
 
+* Index Stats, Node Stats and Cluster Stats API can return a null pointer exception if an index contains a `dense_vector` field 
+but there is an index segment that does not contain any documents with a dense vector field ({es-pull}112720[#112720]). Workarounds:
+** If the affected index already contains documents with a dense vector field, force merge the index to a single segment.
+** If the affected index does not already contain documents with a dense vector field, index a document with a dense vector field
+and then force merge to a single segment.
+** If the affected index's `dense_vector` fields are unused, reindex without the `dense_vector` fields.
+
+
 [[bug-8.15.1]]
 [float]
 === Bug fixes

docs/reference/search/search-your-data/semantic-search-semantic-text.asciidoc

@@ -36,7 +36,11 @@ PUT _inference/sparse_embedding/my-elser-endpoint <1>
 {
   "service": "elser", <2>
   "service_settings": {
-    "num_allocations": 1,
+    "adaptive_allocations": { <3>
+      "enabled": true,
+      "min_number_of_allocations": 3,
+      "max_number_of_allocations": 10
+    },
     "num_threads": 1
   }
 }
@@ -46,6 +50,8 @@ PUT _inference/sparse_embedding/my-elser-endpoint <1>
 be used and ELSER creates sparse vectors. The `inference_id` is
 `my-elser-endpoint`.
 <2> The `elser` service is used in this example.
+<3> This setting enables and configures {ml-docs}/ml-nlp-elser.html#elser-adaptive-allocations[adaptive allocations].
+Adaptive allocations make it possible for ELSER to automatically scale up or down resources based on the current load on the process.
 
 [NOTE]
 ====

libs/entitlement-runtime/README.md

@@ -0,0 +1,14 @@
+### Entitlement runtime
+
+This module implements mechanisms to grant and check permissions under the _entitlements_ system.
+
+The entitlements system provides an alternative to the legacy `SecurityManager` system, which is deprecated for removal.
+The `entitlement-agent` tool instruments sensitive class library methods with calls to this module, in order to enforce the controls.
+
+This module is responsible for:
+- Defining which class library methods are sensitive
+- Defining what permissions should be checked for each sensitive method
+- Implementing the permission checks
+- Offering a "grant" API to grant permissions
+
+It is not responsible for anything to do with bytecode instrumentation; that responsibility lies with `entitlement-agent`.

libs/plugin-classloader/build.gradle → libs/entitlement-runtime/build.gradle

@@ -6,13 +6,19 @@
  * your election, the "Elastic License 2.0", the "GNU Affero General Public
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
-
-// This is only required because :server needs this at runtime.
-// We'll be removing this in 8.0 so for now just publish the JAR to make dependency resolution work.
+apply plugin: 'elasticsearch.build'
 apply plugin: 'elasticsearch.publish'
 
-tasks.named("test").configure { enabled = false }
+dependencies {
+  compileOnly project(':libs:elasticsearch-core')
+
+  testImplementation project(":test:framework")
+}
+
+tasks.named('forbiddenApisMain').configure {
+  replaceSignatureFiles 'jdk-signatures'
+}
 
-// test depend on ES core...
-tasks.named('forbiddenApisMain').configure { enabled = false}
-tasks.named("jarHell").configure { enabled = false }
+tasks.named('forbiddenApisMain').configure {
+  replaceSignatureFiles 'jdk-signatures'
+}