Merge branch 'main' into esql-translator-prototype-2

build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/VersionPropertiesLoader.java

@@ -21,11 +21,8 @@
 public class VersionPropertiesLoader {
     static Properties loadBuildSrcVersion(File input, ProviderFactory providerFactory) throws IOException {
         Properties props = new Properties();
-        InputStream is = new FileInputStream(input);
-        try {
+        try (InputStream is = new FileInputStream(input)) {
             props.load(is);
-        } finally {
-            is.close();
         }
         loadBuildSrcVersion(props, providerFactory);
         return props;

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/CopyRestApiTask.java

@@ -146,13 +146,16 @@ private boolean projectHasYamlRestTests() {
         try {
             // check source folder for tests
             if (sourceResourceDir != null && new File(sourceResourceDir, REST_TEST_PREFIX).exists()) {
-                return Files.walk(sourceResourceDir.toPath().resolve(REST_TEST_PREFIX))
-                    .anyMatch(p -> p.getFileName().toString().endsWith("yml"));
+                try (var files = Files.walk(sourceResourceDir.toPath().resolve(REST_TEST_PREFIX))) {
+                    return files.anyMatch(p -> p.getFileName().toString().endsWith("yml"));
+                }
             }
             // check output for cases where tests are copied programmatically
             File yamlTestOutputDir = new File(additionalYamlTestsDir.get().getAsFile(), REST_TEST_PREFIX);
             if (yamlTestOutputDir.exists()) {
-                return Files.walk(yamlTestOutputDir.toPath()).anyMatch(p -> p.getFileName().toString().endsWith("yml"));
+                try (var files = Files.walk(yamlTestOutputDir.toPath())) {
+                    return files.anyMatch(p -> p.getFileName().toString().endsWith("yml"));
+                }
             }
         } catch (IOException e) {
             throw new IllegalStateException(String.format("Error determining if this project [%s] has rest tests.", getProject()), e);

build-tools/src/main/java/org/elasticsearch/gradle/LoggedExec.java

@@ -118,7 +118,9 @@ public void run() {
                 try {
                     // the file may not exist if the command never output anything
                     if (Files.exists(spoolFile.toPath())) {
-                        Files.lines(spoolFile.toPath()).forEach(logger::error);
+                        try (var lines = Files.lines(spoolFile.toPath())) {
+                            lines.forEach(logger::error);
+                        }
                     }
                 } catch (IOException e) {
                     throw new RuntimeException("could not log", e);

build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java

@@ -458,7 +458,9 @@ public synchronized void start() {
                 // make sure we always start fresh
                 if (Files.exists(workingDir)) {
                     if (preserveDataDir) {
-                        Files.list(workingDir).filter(path -> path.equals(confPathData) == false).forEach(this::uncheckedDeleteWithRetry);
+                        try (var files = Files.list(workingDir)) {
+                            files.filter(path -> path.equals(confPathData) == false).forEach(this::uncheckedDeleteWithRetry);
+                        }
                     } else {
                         deleteWithRetry(workingDir);
                     }

docs/changelog/104711.yaml

@@ -0,0 +1,5 @@
+pr: 104711
+summary: "Fixing NPE when requesting [_none_] for `stored_fields`"
+area: Search
+type: bug
+issues: []

docs/changelog/105063.yaml

@@ -0,0 +1,5 @@
+pr: 105063
+summary: Infrastructure for metering the update requests
+area: Infra/Metrics
+type: enhancement
+issues: []

docs/changelog/105315.yaml

@@ -0,0 +1,6 @@
+pr: 105315
+summary: Always show `composed_of` field for composable index templates
+area: Indices APIs
+type: bug
+issues:
+ - 104627

docs/changelog/105360.yaml

@@ -0,0 +1,6 @@
+pr: 105360
+summary: Cross-cluster painless/execute actions should check permissions only on target
+  remote cluster
+area: Search
+type: bug
+issues: []

docs/changelog/105365.yaml

@@ -0,0 +1,6 @@
+pr: 105365
+summary: Fix bug in `rule_query` where `text_expansion` errored because it was not
+  rewritten
+area: Application
+type: bug
+issues: []

docs/changelog/105373.yaml

@@ -0,0 +1,5 @@
+pr: 105373
+summary: "Fix parsing of flattened fields within subobjects: false"
+area: Mapping
+type: bug
+issues: []

docs/changelog/105439.yaml

@@ -0,0 +1,6 @@
+pr: 105439
+summary: Support Profile Activate with JWTs with client authn
+area: Authentication
+type: enhancement
+issues:
+ - 105342

docs/changelog/105449.yaml

@@ -0,0 +1,6 @@
+pr: 105449
+summary: Don't stop checking if the `HealthNode` persistent task is present
+area: Health
+type: bug
+issues:
+ - 98926

docs/changelog/105486.yaml

@@ -0,0 +1,5 @@
+pr: 105486
+summary: Fix use-after-free at event-loop shutdown
+area: Network
+type: bug
+issues: []

docs/changelog/105546.yaml

@@ -0,0 +1,6 @@
+pr: 105546
+summary: '`GlobalOrdCardinalityAggregator` should use `HyperLogLogPlusPlus` instead
+  of `HyperLogLogPlusPlusSparse`'
+area: Aggregations
+type: bug
+issues: []

docs/changelog/105588.yaml

@@ -0,0 +1,5 @@
+pr: 105588
+summary: '`URLRepository` should not block shutdown'
+area: Snapshot/Restore
+type: bug
+issues: []

docs/changelog/105633.yaml

@@ -0,0 +1,6 @@
+pr: 105633
+summary: "[Connector API] Bugfix: support list type in filtering advenced snippet\
+  \ value"
+area: Application
+type: bug
+issues: []

docs/reference/health/health.asciidoc

@@ -73,7 +73,7 @@ for health status set `verbose` to `false` to disable the more expensive analysi
 
   `repository_integrity`::
       Tracks repository integrity and reports health issues
-      that arise if repositories become corrupted.
+      that arise if repositories become corrupted, unknown, or invalid.
 
   `slm`::
       Reports health issues related to
@@ -356,6 +356,14 @@ watermark threshold>>.
     (Optional, array of strings) If corrupted repositories have been detected in the system, the names of up to ten of
     them are displayed in this field. If no corrupted repositories are found, this detail is omitted.
 
+`unknown_repositories`::
+    (Optional, int) The number of repositories that have been determined to be unknown by at least one node.
+    If there are no unknown repositories detected, this detail is omitted.
+
+`invalid_repositories`::
+    (Optional, int) The number of repositories that have been determined to be invalid by at least one node.
+    If there are no invalid repositories detected, this detail is omitted.
+
 [[health-api-response-details-ilm]]
 ===== ilm
 

docs/reference/how-to/size-your-shards.asciidoc

@@ -221,6 +221,15 @@ GET _cat/shards?v=true
 ----
 // TEST[setup:my_index]
 
+[discrete]
+[[shard-count-per-node-recommendation]]
+==== Add enough nodes to stay within the cluster shard limits
+
+The <<cluster-shard-limit,cluster shard limits>> prevent creation of more than
+1000 non-frozen shards per node, and 3000 frozen shards per dedicated frozen
+node. Make sure you have enough nodes of each type in your cluster to handle
+the number of shards you need.
+
 [discrete]
 [[field-count-recommendation]]
 ==== Allow enough heap for field mappers and overheads

docs/reference/landing-page.asciidoc

@@ -79,6 +79,11 @@
 
 <h3 class="gtk">Get to know Elasticsearch</h3>
 
+<p>
+  <em>New webinar:</em>
+  <a href="https://www.elastic.co/virtual-events/architecting-search-apps-on-google-cloud">Architect search apps with Google Cloud</a>
+</p>
+
 <div class="my-5">
   <div class="d-flex align-items-center mb-3">
     <h4 class="mt-3">

docs/reference/modules/cluster/misc.asciidoc

@@ -24,35 +24,46 @@ API can make the cluster read-write again.
 
 [discrete]
 [[cluster-shard-limit]]
-==== Cluster shard limit
+==== Cluster shard limits
 
-There is a soft limit on the number of shards in a cluster, based on the number
-of nodes in the cluster. This is intended to prevent operations which may
-unintentionally destabilize the cluster.
+There is a limit on the number of shards in a cluster, based on the number of
+nodes in the cluster. This is intended to prevent a runaway process from
+creating too many shards which can harm performance and in extreme cases may
+destabilize your cluster.
 
-IMPORTANT: This limit is intended as a safety net, not a sizing recommendation. The
-exact number of shards your cluster can safely support depends on your hardware
-configuration and workload, but should remain well below this limit in almost
-all cases, as the default limit is set quite high.
+[IMPORTANT]
+====
 
-If an operation, such as creating a new index, restoring a snapshot of an index,
-or opening a closed index would lead to the number of shards in the cluster
-going over this limit, the operation will fail with an error indicating the
-shard limit.
+These limits are intended as a safety net to protect against runaway shard
+creation and are not a sizing recommendation. The exact number of shards your
+cluster can safely support depends on your hardware configuration and workload,
+and may be smaller than the default limits.
 
-If the cluster is already over the limit, due to changes in node membership or
-setting changes, all operations that create or open indices will fail until
-either the limit is increased as described below, or some indices are
-<<indices-open-close,closed>> or <<indices-delete-index,deleted>> to bring the
-number of shards below the limit.
+We do not recommend increasing these limits beyond the defaults. Clusters with
+more shards may appear to run well in normal operation, but may take a very
+long time to recover from temporary disruptions such as a network partition or
+an unexpected node restart, and may encounter problems when performing
+maintenance activities such as a rolling restart or upgrade.
 
-The cluster shard limit defaults to 1,000 shards per non-frozen data node for
+====
+
+If an operation, such as creating a new index, restoring a snapshot of an
+index, or opening a closed index would lead to the number of shards in the
+cluster going over this limit, the operation will fail with an error indicating
+the shard limit. To resolve this, either scale out your cluster by adding
+nodes, or <<indices-delete-index,delete some indices>> to bring the number of
+shards below the limit.
+
+If a cluster is already over the limit, perhaps due to changes in node
+membership or setting changes, all operations that create or open indices will
+fail.
+
+The cluster shard limit defaults to 1000 shards per non-frozen data node for
 normal (non-frozen) indices and 3000 shards per frozen data node for frozen
-indices.
-Both primary and replica shards of all open indices count toward the limit,
-including unassigned shards.
-For example, an open index with 5 primary shards and 2 replicas counts as 15 shards.
-Closed indices do not contribute to the shard count.
+indices. Both primary and replica shards of all open indices count toward the
+limit, including unassigned shards. For example, an open index with 5 primary
+shards and 2 replicas counts as 15 shards. Closed indices do not contribute to
+the shard count.
 
 You can dynamically adjust the cluster shard limit with the following setting:
 
@@ -99,12 +110,13 @@ For example, a cluster with a `cluster.max_shards_per_node.frozen` setting of
 `100` and three frozen data nodes has a frozen shard limit of 300. If the
 cluster already contains 296 shards, {es} rejects any request that adds five or
 more frozen shards to the cluster.
+--
 
-NOTE: These setting do not limit shards for individual nodes. To limit the
-number of shards for each node, use the
+NOTE: These limits only apply to actions which create shards and do not limit
+the number of shards assigned to each node. To limit the number of shards
+assigned to each node, use the
 <<cluster-total-shards-per-node,`cluster.routing.allocation.total_shards_per_node`>>
 setting.
---
 
 [discrete]
 [[user-defined-data]]

docs/reference/rest-api/security/activate-user-profile.asciidoc

@@ -28,6 +28,9 @@ Creates or updates a user profile on behalf of another user.
 The activate user profile API creates or updates a profile document for end
 users with information that is extracted from the user's authentication object,
 including `username`, `full_name`, `roles`, and the authentication realm.
+For example, in the JWT `access_token` case, the profile user's `username` is
+extracted from the JWT token claim pointed to by the `claims.principal`
+setting of the JWT realm that authenticated the token.
 
 When updating a profile document, the API enables the document if it was
 disabled. Any updates do not change existing content for either the `labels` or
@@ -46,8 +49,11 @@ is intended for.
 
 `access_token`::
 (Required*, string)
-The user's access token. If you specify the `access_token` grant type, this
-parameter is required. It is not valid with other grant types.
+The user's <<security-api-get-token, {es} access token>>, or JWT. Both <<jwt-realm-oauth2, access>> and
+<<jwt-realm-oidc, id>> JWT token types are supported, and they depend on the underlying JWT realm configuration.
+If you specify the `access_token` grant type, this parameter is required. It is not valid with other grant types.
+
+include::client-authentication.asciidoc[]
 
 `grant_type`::
 (Required, string)
@@ -57,24 +63,22 @@ The type of grant.
 [%collapsible%open]
 ====
 `access_token`::
-(Required*, string)
-In this type of grant, you must supply an access token that was created by the
-{es} token service. For more information, see
-<<security-api-get-token>> and <<token-service-settings>>.
+In this type of grant, you must supply either an access token, that was created by the
+{es} token service (see <<security-api-get-token>> and <<encrypt-http-communication>>),
+or a <<jwt-auth-realm, JWT>> (either a JWT `access_token` or a JWT `id_token`).
 
 `password`::
-(Required*, string)
 In this type of grant, you must supply the `username` and `password` for the
 user that you want to create the API key for.
 ====
 
 `password`::
-(Optional*, string)
+(Required*, string)
 The user's password. If you specify the `password` grant type, this parameter is
 required. It is not valid with other grant types.
 
 `username`::
-(Optional*, string)
+(Required*, string)
 The username that identifies the user. If you specify the `password` grant type,
 this parameter is required. It is not valid with other grant types.
 

docs/reference/rest-api/security/client-authentication.asciidoc

@@ -0,0 +1,16 @@
+`client_authentication`::
+(Optional, object) When using the `access_token` grant type, and when supplying a
+JWT, this specifies the client authentication for <<jwt-auth-realm, JWTs>> that
+need it (i.e. what's normally specified by the `ES-Client-Authentication` request header).
+
+`scheme`:::
+(Required, string) The scheme (case-sensitive) as it's supplied in the
+`ES-Client-Authentication` request header. Currently, the only supported
+value is <<jwt-auth-shared-secret-scheme-example, `SharedSecret`>>.
+
+`value`:::
+(Required, string) The value that follows the scheme for the client credentials
+as it's supplied in the `ES-Client-Authentication` request header. For example,
+if the request header would be `ES-Client-Authentication: SharedSecret myShar3dS3cret`
+if the client were to authenticate directly with a JWT, then `value` here should
+be `myShar3dS3cret`.

docs/reference/rest-api/security/grant-api-keys.asciidoc

@@ -89,29 +89,13 @@ It supports nested data structure.
 Within the `metadata` object, keys beginning with `_` are reserved for
 system usage.
 
-`client_authentication`::
-(Optional, object) When using the `access_token` grant type, and when supplying a
-JWT, this specifies the client authentication for <<jwt-auth-realm, JWTs>> that
-need it (i.e. what's normally specified by the `ES-Client-Authentication` request header).
-
-`scheme`:::
-(Required, string) The scheme (case-sensitive) as it's supplied in the
-`ES-Client-Authentication` request header. Currently, the only supported
-value is <<jwt-auth-shared-secret-scheme-example, `SharedSecret`>>.
-
-`value`:::
-(Required, string) The value that follows the scheme for the client credentials
-as it's supplied in the `ES-Client-Authentication` request header. For example,
-if the request header would be `ES-Client-Authentication: SharedSecret myShar3dS3cret`
-if the client were to authenticate directly with a JWT, then `value` here should
-be `myShar3dS3cret`.
+include::client-authentication.asciidoc[]
 
 `grant_type`::
 (Required, string)
 The type of grant. Supported grant types are: `access_token`,`password`.
 
 `access_token`:::
-(Required*, string)
 In this type of grant, you must supply either an access token, that was created by the
 {es} token service (see <<security-api-get-token>> and <<encrypt-http-communication>>),
 or a <<jwt-auth-realm, JWT>> (either a JWT `access_token` or a JWT `id_token`).
@@ -121,19 +105,21 @@ In this type of grant, you must supply the user ID and password for which you
 want to create the API key.
 
 `password`::
-(Optional*, string)
+(Required*, string)
 The user's password. If you specify the `password` grant type, this parameter is
 required. It is not valid with other grant types.
 
 `username`::
-(Optional*, string)
+(Required*, string)
 The user name that identifies the user. If you specify the `password` grant type,
 this parameter is required. It is not valid with other grant types.
 
 `run_as`::
 (Optional, string)
 The name of the user to be <<run-as-privilege,impersonated>>.
 
+*Indicates that the setting is required in some, but not all situations.
+
 [[security-api-grant-api-key-example]]
 ==== {api-examples-title}