Merge remote-tracking branch 'origin/main' into global_seq_no_translo…

…g_replicator

distribution/docker/src/docker/Dockerfile

@@ -22,7 +22,7 @@
 <% if (docker_base == 'iron_bank') { %>
 ARG BASE_REGISTRY=registry1.dso.mil
 ARG BASE_IMAGE=ironbank/redhat/ubi/ubi9
-ARG BASE_TAG=9.2
+ARG BASE_TAG=9.3
 <% } %>
 
 ################################################################################

distribution/docker/src/docker/iron_bank/hardening_manifest.yaml

@@ -14,7 +14,7 @@ tags:
 # Build args passed to Dockerfile ARGs
 args:
   BASE_IMAGE: "redhat/ubi/ubi9"
-  BASE_TAG: "9.2"
+  BASE_TAG: "9.3"
 
 # Docker image labels
 labels:

docs/changelog/102741.yaml

@@ -0,0 +1,6 @@
+pr: 102741
+summary: "[ILM] More resilient when a policy is added to searchable snapshot"
+area: ILM+SLM
+type: bug
+issues:
+ - 101958

docs/changelog/102843.yaml

@@ -0,0 +1,5 @@
+pr: 102843
+summary: Restore `SharedBytes.IO` refcounting on reads & writes
+area: Snapshot/Restore
+type: bug
+issues: []

docs/changelog/102902.yaml

@@ -0,0 +1,5 @@
+pr: 102902
+summary: Fast path for reading single doc with ordinals
+area: ES|QL
+type: enhancement
+issues: []

docs/changelog/102925.yaml

@@ -0,0 +1,5 @@
+pr: 102925
+summary: Add ldap user metadata mappings for full name and email
+area: Authentication
+type: enhancement
+issues: []

docs/changelog/103031.yaml

@@ -0,0 +1,9 @@
+pr: 103031
+summary: Collect warnings in compute service
+area: ES|QL
+type: bug
+issues:
+ - 100163
+ - 103028
+ - 102871
+ - 102982

docs/changelog/103047.yaml

@@ -0,0 +1,5 @@
+pr: 103047
+summary: Ensure `dynamicMapping` updates are handled in insertion order
+area: Mapping
+type: bug
+issues: []

docs/reference/ilm/actions/ilm-delete.asciidoc

@@ -16,6 +16,11 @@ Defaults to `true`.
 This option is applicable when the <<ilm-searchable-snapshot,searchable
 snapshot>> action is used in any previous phase.
 
+WARNING: If a policy with a searchable snapshot action is applied on an existing searchable snapshot index,
+the snapshot backing this index will NOT be deleted because it was not created by this policy. If you want
+to clean this snapshot, please delete it manually after the index is deleted using the <<delete-snapshot-api, delete snapshot API>>, you
+can find the repository and snapshot name using the <<indices-get-index, get index API>>.
+
 [[ilm-delete-action-ex]]
 ==== Example
 

docs/reference/query-dsl/rule-query.asciidoc

@@ -10,6 +10,7 @@ preview::[]
 Applies <<query-rules-apis,query rules>> to the query before returning results.
 This feature is used to promote documents in the manner of a <<query-dsl-pinned-query>> based on matching defined rules.
 If no matching query rules are defined, the "organic" matches for the query are returned.
+All matching rules are applied in the order in which they appear in the query ruleset.
 
 [NOTE]
 ====

docs/reference/query-rules/apis/put-query-ruleset.asciidoc

@@ -29,6 +29,9 @@ Requires the `manage_search_query_rules` privilege.
 `rules`::
 (Required, array of objects) The specific rules included in this query ruleset.
 
+There is a limit of 100 rules per ruleset.
+This can be increased up to 1000 using the `xpack.applications.rules.max_rules_per_ruleset` cluster setting.
+
 Each rule must have the following information:
 
 - `rule_id` (Required, string) A unique identifier for this rule.

docs/reference/search/search-your-data/search-using-query-rules.asciidoc

@@ -174,6 +174,9 @@ PUT /_query_rules/my-ruleset
 
 The API response returns a results of `created` or `updated` depending on whether this was a new or edited ruleset.
 
+NOTE: There is a limit of 100 rules per ruleset.
+This can be increased up to 1000 using the `xpack.applications.rules.max_rules_per_ruleset` cluster setting.
+
 [source,console-result]
 ----
 {
@@ -217,3 +220,8 @@ GET /my-index-000001/_search
 
 This rule query will match against `rule1` in the defined query ruleset, and will convert the organic query into a pinned query with `id1` and `id2` pinned as the top hits.
 Any other matches from the organic query will be returned below the pinned results.
+
+It's possible to have multiple rules in a ruleset match a single `rule_query`. In this case, the pinned documents are returned in the following order:
+
+- Where the matching rule appears in the ruleset
+- If multiple documents are specified in a single rule, in the order they are specified

docs/reference/settings/security-settings.asciidoc

@@ -431,6 +431,16 @@ Specifies the attribute to examine on the user for group membership.
 If any `group_search` settings are specified, this setting is ignored. Defaults
 to `memberOf`.
 
+`user_full_name_attribute`::
+(<<static-cluster-setting,Static>>)
+Specifies the attribute to examine on the user for the full name of the user.
+Defaults to `cn`.
+
+`user_email_attribute`::
+(<<static-cluster-setting,Static>>)
+Specifies the attribute to examine on the user for the email address of the user.
+Defaults to `mail`.
+
 `user_search.base_dn`::
 (<<static-cluster-setting,Static>>)
 Specifies a container DN to search for users. Required

docs/reference/setup/sysconfig/tcpretries.asciidoc

@@ -17,7 +17,7 @@ Most Linux distributions default to retransmitting any lost packets 15 times.
 Retransmissions back off exponentially, so these 15 retransmissions take over
 900 seconds to complete. This means it takes Linux many minutes to detect a
 network partition or a failed node with this method. Windows defaults to just 5
-retransmissions which corresponds with a timeout of around 6 seconds.
+retransmissions which corresponds with a timeout of around 13 seconds.
 
 The Linux default allows for communication over networks that may experience
 very long periods of packet loss, but this default is excessive and even harmful
@@ -32,7 +32,7 @@ therefore reduce the maximum number of TCP retransmissions.
 
 You can decrease the maximum number of TCP retransmissions to `5` by running the
 following command as `root`. Five retransmissions corresponds with a timeout of
-around six seconds.
+around 13 seconds.
 
 [source,sh]
 -------------------------------------

docs/reference/snapshot-restore/apis/restore-snapshot-api.asciidoc

@@ -159,6 +159,7 @@ The cluster state includes:
 * <<indices-templates-v1,Legacy index templates>>
 * <<ingest,Ingest pipelines>>
 * <<index-lifecycle-management,{ilm-init} policies>>
+* <<script-stored-scripts,Stored scripts>>
 * For snapshots taken after 7.12.0, <<feature-state,feature states>>
 // end::cluster-state-contents[]
 
@@ -260,12 +261,12 @@ You may want to restore an index in-place, for example when no alternative
 options surface after the <<cluster-allocation-explain>> API reports
 `no_valid_shard_copy`.
 
-The following request <<indices-close,closes>> `index_1` and then restores it 
+The following request <<indices-close,closes>> `index_1` and then restores it
 in-place from the `snapshot_2` snapshot in the `my_repository` repository.
 
 [source,console]
 ----
-POST index_1/_close 
+POST index_1/_close
 
 POST /_snapshot/my_repository/snapshot_2/_restore?wait_for_completion=true
 {

plugins/repository-hdfs/build.gradle

@@ -59,12 +59,6 @@ dependencies {
     runtimeOnly "org.slf4j:slf4j-nop:${versions.slf4j}"
     //  runtimeOnly("org.apache.logging.log4j:log4j-slf4j-impl:${versions.log4j}") https://github.com/elastic/elasticsearch/issues/93714
 
-    // Set the keytab files in the classpath so that we can access them from test code without the security manager
-    // freaking out.
-    if (isEclipse == false) {
-        javaRestTestRuntimeOnly project(path: ':test:fixtures:krb5kdc-fixture', configuration:'krb5KeytabsHdfsDir')
-    }
-
     krb5Keytabs project(path: ':test:fixtures:krb5kdc-fixture', configuration: 'krb5KeytabsHdfsDir')
     krb5Config project(path: ':test:fixtures:krb5kdc-fixture', configuration: 'krb5ConfHdfsFile')
 }
@@ -187,7 +181,11 @@ for (int hadoopVersion = minTestedHadoopVersion; hadoopVersion <= maxTestedHadoo
                 }
             }
             testClassesDirs = sourceSets.javaRestTest.output.classesDirs
-            classpath = sourceSets.javaRestTest.runtimeClasspath + files(portsFileDir)
+            // Set the keytab files in the classpath so that we can access them from test code without the security manager
+            // freaking out.
+            classpath = sourceSets.javaRestTest.runtimeClasspath +
+              configurations.krb5Keytabs +
+              files(portsFileDir)
         }
     }
 

server/src/main/java/org/elasticsearch/action/search/MultiSearchResponse.java

@@ -61,7 +61,12 @@ public static class Item implements Writeable, ChunkedToXContent {
         private final SearchResponse response;
         private final Exception exception;
 
-        public Item(SearchResponse response, Exception exception) {
+        /**
+         *
+         * @param response search response that is considered owned by this instance after this constructor returns or {@code null}
+         * @param exception exception in case of search failure
+         */
+        public Item(@Nullable SearchResponse response, @Nullable Exception exception) {
             this.response = response;
             this.exception = exception;
         }

server/src/main/java/org/elasticsearch/action/search/SearchResponseMerger.java

@@ -19,6 +19,7 @@
 import org.elasticsearch.action.search.SearchResponse.Clusters;
 import org.elasticsearch.action.search.TransportSearchAction.SearchTimeProvider;
 import org.elasticsearch.common.lucene.search.TopDocsAndMaxScore;
+import org.elasticsearch.core.Releasable;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.lucene.grouping.TopFieldGroups;
 import org.elasticsearch.search.SearchHit;
@@ -31,6 +32,7 @@
 import org.elasticsearch.search.profile.SearchProfileShardResult;
 import org.elasticsearch.search.suggest.Suggest;
 import org.elasticsearch.search.suggest.completion.CompletionSuggestion;
+import org.elasticsearch.transport.LeakTracker;
 
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -64,14 +66,20 @@
 // TODO it may make sense to integrate the remote clusters responses as a shard response in the initial search phase and ignore hits coming
 // from the remote clusters in the fetch phase. This would be identical to the removed QueryAndFetch strategy except that only the remote
 // cluster response would have the fetch results.
-final class SearchResponseMerger {
+final class SearchResponseMerger implements Releasable {
     final int from;
     final int size;
     final int trackTotalHitsUpTo;
     private final SearchTimeProvider searchTimeProvider;
     private final AggregationReduceContext.Builder aggReduceContextBuilder;
     private final List<SearchResponse> searchResponses = new CopyOnWriteArrayList<>();
 
+    private final Releasable releasable = LeakTracker.wrap(() -> {
+        for (SearchResponse searchResponse : searchResponses) {
+            searchResponse.decRef();
+        }
+    });
+
     SearchResponseMerger(
         int from,
         int size,
@@ -93,6 +101,7 @@ final class SearchResponseMerger {
      */
     void add(SearchResponse searchResponse) {
         assert searchResponse.getScrollId() == null : "merging scroll results is not supported";
+        searchResponse.mustIncRef();
         searchResponses.add(searchResponse);
     }
 
@@ -383,6 +392,11 @@ private static SearchHits topDocsToSearchHits(TopDocs topDocs, TopDocsStats topD
         return new SearchHits(searchHits, topDocsStats.getTotalHits(), topDocsStats.getMaxScore(), sortFields, groupField, groupValues);
     }
 
+    @Override
+    public void close() {
+        releasable.close();
+    }
+
     private static final class FieldDocAndSearchHit extends FieldDoc {
         private final SearchHit searchHit;
 

server/src/main/java/org/elasticsearch/action/search/TransportMultiSearchAction.java

@@ -154,9 +154,10 @@ void executeSearch(
          * when we handle the response rather than going recursive, we fork to another thread, otherwise we recurse.
          */
         final Thread thread = Thread.currentThread();
-        client.search(request.request, new ActionListener<SearchResponse>() {
+        client.search(request.request, new ActionListener<>() {
             @Override
             public void onResponse(final SearchResponse searchResponse) {
+                searchResponse.mustIncRef(); // acquire reference on behalf of MultiSearchResponse.Item below
                 handleResponse(request.responseSlot, new MultiSearchResponse.Item(searchResponse, null));
             }
 

server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java

@@ -765,7 +765,14 @@ private static ActionListener<SearchResponse> createCCSListener(
         SearchResponse.Clusters clusters,
         ActionListener<SearchResponse> originalListener
     ) {
-        return new CCSActionListener<>(clusterAlias, skipUnavailable, countDown, exceptions, clusters, originalListener) {
+        return new CCSActionListener<>(
+            clusterAlias,
+            skipUnavailable,
+            countDown,
+            exceptions,
+            clusters,
+            ActionListener.releaseAfter(originalListener, searchResponseMerger)
+        ) {
             @Override
             void innerOnResponse(SearchResponse searchResponse) {
                 // TODO: in CCS fail fast ticket we may need to fail the query if the cluster gets marked as FAILED
@@ -777,6 +784,11 @@ void innerOnResponse(SearchResponse searchResponse) {
             SearchResponse createFinalResponse() {
                 return searchResponseMerger.getMergedResponse(clusters);
             }
+
+            @Override
+            protected void releaseResponse(SearchResponse searchResponse) {
+                searchResponse.decRef();
+            }
         };
     }
 
@@ -1493,13 +1505,19 @@ private void maybeFinish() {
                         originalListener.onFailure(e);
                         return;
                     }
-                    originalListener.onResponse(response);
+                    try {
+                        originalListener.onResponse(response);
+                    } finally {
+                        releaseResponse(response);
+                    }
                 } else {
                     originalListener.onFailure(exceptions.get());
                 }
             }
         }
 
+        protected void releaseResponse(FinalResponse response) {}
+
         abstract FinalResponse createFinalResponse();
     }
 

server/src/main/java/org/elasticsearch/index/mapper/BlockDocValuesReader.java

@@ -555,8 +555,20 @@ private static class SingletonOrdinals extends BlockDocValuesReader {
             this.ordinals = ordinals;
         }
 
+        private BlockLoader.Block readSingleDoc(BlockFactory factory, int docId) throws IOException {
+            if (ordinals.advanceExact(docId)) {
+                BytesRef v = ordinals.lookupOrd(ordinals.ordValue());
+                return factory.constantBytes(v);
+            } else {
+                return factory.constantNulls();
+            }
+        }
+
         @Override
         public BlockLoader.Block read(BlockFactory factory, Docs docs) throws IOException {
+            if (docs.count() == 1) {
+                return readSingleDoc(factory, docs.get(0));
+            }
             try (BlockLoader.SingletonOrdinalsBuilder builder = factory.singletonOrdinalsBuilder(ordinals, docs.count())) {
                 for (int i = 0; i < docs.count(); i++) {
                     int doc = docs.get(i);

server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java

@@ -22,8 +22,8 @@
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
-import java.util.HashMap;
 import java.util.HashSet;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -166,9 +166,9 @@ protected DocumentParserContext(
             mappingParserContext,
             source,
             new HashSet<>(),
-            new HashMap<>(),
+            new LinkedHashMap<>(),
             new HashSet<>(),
-            new HashMap<>(),
+            new LinkedHashMap<>(),
             new ArrayList<>(),
             null,
             null,