Merge branch 'elastic:main' into ltr-min-window-size

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/SymbolicLinkPreservingTarFuncTest.groovy

@@ -8,6 +8,8 @@
 
 package org.elasticsearch.gradle.internal
 
+import spock.lang.Ignore
+
 import org.apache.commons.compress.archivers.tar.TarArchiveEntry
 import org.apache.commons.compress.archivers.tar.TarArchiveInputStream
 import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream
@@ -21,6 +23,7 @@ import java.nio.file.Path
 import java.nio.file.Paths
 import java.util.function.Function
 
+@Ignore("https://github.com/elastic/elasticsearch/issues/104428")
 class SymbolicLinkPreservingTarFuncTest extends AbstractGradleFuncTest {
 
     def setup() {

docs/changelog/104182.yaml

@@ -0,0 +1,5 @@
+pr: 104182
+summary: "Apm-data: fix `@custom` component templates"
+area: Data streams
+type: bug
+issues: []

docs/changelog/104355.yaml

@@ -0,0 +1,5 @@
+pr: 104355
+summary: Prepare enrich plan to support multi clusters
+area: ES|QL
+type: enhancement
+issues: []

docs/changelog/104396.yaml

@@ -0,0 +1,5 @@
+pr: 104396
+summary: Report current master in `PeerFinder`
+area: Cluster Coordination
+type: enhancement
+issues: []

docs/changelog/104418.yaml

@@ -0,0 +1,6 @@
+pr: 104418
+summary: Fix `routing_path` when template has multiple `path_match` and multi-fields
+area: TSDB
+type: bug
+issues:
+ - 104400

docs/reference/troubleshooting.asciidoc

@@ -58,9 +58,14 @@ fix problems that an {es} deployment might encounter.
 * <<troubleshooting-shards-capacity-issues,Troubleshooting shards capacity>>
 * <<remote-clusters-troubleshooting,Troubleshooting remote clusters>>
 
-If none of these solutions relate to your issue, you can still get help:
+[discrete]
+[[troubleshooting-contact-support]]
+=== Contact us
+
+If none of these guides relate to your issue, or you need further assistance,
+then you can contact us as follows:
 
-* For users with an active subscription, you can get help in several ways:
+* If you have an active subscription, you have several options:
 
 ** Go directly to the http://support.elastic.co[Support Portal]
 

modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java

@@ -31,7 +31,7 @@
 import java.io.UncheckedIOException;
 import java.time.Instant;
 import java.util.ArrayList;
-import java.util.HashMap;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Locale;
 
@@ -177,14 +177,18 @@ private List<String> findRoutingPaths(String indexName, Settings allSettings, Li
                 }
 
                 MappingParserContext parserContext = mapperService.parserContext();
-                for (String pathMatch : template.pathMatch()) {
+                for (Iterator<String> iterator = template.pathMatch().iterator(); iterator.hasNext();) {
                     var mapper = parserContext.typeParser(mappingSnippetType)
-                        // Since FieldMapper.parse modifies the Map passed in (removing entries for "type"), that means
-                        // that only the first pathMatch passed in gets recognized as a time_series_dimension. To counteract
-                        // that, we wrap the mappingSnippet in a new HashMap for each pathMatch instance.
-                        .parse(pathMatch, new HashMap<>(mappingSnippet), parserContext)
+                        .parse(iterator.next(), mappingSnippet, parserContext)
                         .build(MapperBuilderContext.root(false, false));
                     extractPath(routingPaths, mapper);
+                    if (iterator.hasNext()) {
+                        // Since FieldMapper.parse modifies the Map passed in (removing entries for "type"), that means
+                        // that only the first pathMatch passed in gets recognized as a time_series_dimension.
+                        // To avoid this, each parsing call uses a new mapping snippet.
+                        // Note that a shallow copy of the mappingSnippet map is not enough if there are multi-fields.
+                        mappingSnippet = template.mappingForName(templateName, KeywordFieldMapper.CONTENT_TYPE);
+                    }
                 }
             }
             return routingPaths;

modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java

@@ -493,6 +493,55 @@ public void testGenerateRoutingPathFromDynamicTemplateWithMultiplePathMatchEntri
         assertEquals(3, routingPathList.size());
     }
 
+    public void testGenerateRoutingPathFromDynamicTemplateWithMultiplePathMatchEntriesMultiFields() throws Exception {
+        Instant now = Instant.now().truncatedTo(ChronoUnit.SECONDS);
+        String mapping = """
+            {
+                "_doc": {
+                    "dynamic_templates": [
+                        {
+                            "labels": {
+                                "path_match": ["xprometheus.labels.*", "yprometheus.labels.*"],
+                                "mapping": {
+                                    "type": "keyword",
+                                    "time_series_dimension": true,
+                                    "fields": {
+                                      "text": {
+                                        "type": "text"
+                                      }
+                                    }
+                                }
+                            }
+                        }
+                    ],
+                    "properties": {
+                        "host": {
+                            "properties": {
+                                "id": {
+                                    "type": "keyword",
+                                    "time_series_dimension": true
+                                }
+                            }
+                        },
+                        "another_field": {
+                            "type": "keyword"
+                        }
+                    }
+                }
+            }
+            """;
+        Settings result = generateTsdbSettings(mapping, now);
+        assertThat(result.size(), equalTo(3));
+        assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis())));
+        assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis())));
+        assertThat(
+            IndexMetadata.INDEX_ROUTING_PATH.get(result),
+            containsInAnyOrder("host.id", "xprometheus.labels.*", "yprometheus.labels.*")
+        );
+        List<String> routingPathList = IndexMetadata.INDEX_ROUTING_PATH.get(result);
+        assertEquals(3, routingPathList.size());
+    }
+
     public void testGenerateRoutingPathFromDynamicTemplate_templateWithNoPathMatch() throws Exception {
         Instant now = Instant.now().truncatedTo(ChronoUnit.SECONDS);
         String mapping = """

modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobContainerRetriesTests.java

@@ -509,6 +509,7 @@ public void testWriteLargeBlobStreaming() throws Exception {
         assertEquals(blobSize, bytesReceived.get());
     }
 
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/104436")
     public void testReadRetriesAfterMeaningfulProgress() throws Exception {
         final int maxRetries = between(0, 5);
         final int bufferSizeBytes = scaledRandomIntBetween(

server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksIT.java

@@ -18,7 +18,6 @@
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.LatchedActionListener;
-import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksResponse;
 import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.GroupedActionListener;
@@ -166,7 +165,7 @@ public void testBanOnlyNodesWithOutstandingDescendantTasks() throws Exception {
         ActionFuture<TestResponse> rootTaskFuture = client().execute(TransportTestAction.ACTION, rootRequest);
         Set<TestRequest> pendingRequests = allowPartialRequest(rootRequest);
         TaskId rootTaskId = getRootTaskId(rootRequest);
-        ActionFuture<CancelTasksResponse> cancelFuture = clusterAdmin().prepareCancelTasks()
+        ActionFuture<ListTasksResponse> cancelFuture = clusterAdmin().prepareCancelTasks()
             .setTargetTaskId(rootTaskId)
             .waitForCompletion(true)
             .execute();
@@ -215,18 +214,18 @@ public void testCancelTaskMultipleTimes() throws Exception {
         ActionFuture<TestResponse> mainTaskFuture = client().execute(TransportTestAction.ACTION, rootRequest);
         TaskId taskId = getRootTaskId(rootRequest);
         allowPartialRequest(rootRequest);
-        CancelTasksResponse resp = clusterAdmin().prepareCancelTasks().setTargetTaskId(taskId).waitForCompletion(false).get();
+        ListTasksResponse resp = clusterAdmin().prepareCancelTasks().setTargetTaskId(taskId).waitForCompletion(false).get();
         assertThat(resp.getTaskFailures(), empty());
         assertThat(resp.getNodeFailures(), empty());
-        ActionFuture<CancelTasksResponse> cancelFuture = clusterAdmin().prepareCancelTasks()
+        ActionFuture<ListTasksResponse> cancelFuture = clusterAdmin().prepareCancelTasks()
             .setTargetTaskId(taskId)
             .waitForCompletion(true)
             .execute();
         assertFalse(cancelFuture.isDone());
         allowEntireRequest(rootRequest);
         assertThat(cancelFuture.actionGet().getTaskFailures(), empty());
         waitForRootTask(mainTaskFuture, false);
-        CancelTasksResponse cancelError = clusterAdmin().prepareCancelTasks()
+        ListTasksResponse cancelError = clusterAdmin().prepareCancelTasks()
             .setTargetTaskId(taskId)
             .waitForCompletion(randomBoolean())
             .get();
@@ -245,7 +244,7 @@ public void testDoNotWaitForCompletion() throws Exception {
             allowPartialRequest(rootRequest);
         }
         boolean waitForCompletion = randomBoolean();
-        ActionFuture<CancelTasksResponse> cancelFuture = clusterAdmin().prepareCancelTasks()
+        ActionFuture<ListTasksResponse> cancelFuture = clusterAdmin().prepareCancelTasks()
             .setTargetTaskId(taskId)
             .waitForCompletion(waitForCompletion)
             .execute();
@@ -311,7 +310,7 @@ public void testRemoveBanParentsOnDisconnect() throws Exception {
         client().execute(TransportTestAction.ACTION, rootRequest);
         Set<TestRequest> pendingRequests = allowPartialRequest(rootRequest);
         TaskId rootTaskId = getRootTaskId(rootRequest);
-        ActionFuture<CancelTasksResponse> cancelFuture = clusterAdmin().prepareCancelTasks()
+        ActionFuture<ListTasksResponse> cancelFuture = clusterAdmin().prepareCancelTasks()
             .setTargetTaskId(rootTaskId)
             .waitForCompletion(true)
             .execute();

server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java

@@ -16,7 +16,6 @@
 import org.elasticsearch.action.FailedNodeException;
 import org.elasticsearch.action.TaskOperationFailure;
 import org.elasticsearch.action.admin.cluster.health.TransportClusterHealthAction;
-import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksResponse;
 import org.elasticsearch.action.admin.cluster.node.tasks.get.GetTaskRequest;
 import org.elasticsearch.action.admin.cluster.node.tasks.get.GetTaskResponse;
 import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse;
@@ -503,7 +502,7 @@ public void testTasksCancellation() throws Exception {
         );
 
         logger.info("--> cancelling the main test task");
-        CancelTasksResponse cancelTasksResponse = clusterAdmin().prepareCancelTasks().setActions(TEST_TASK_ACTION.name()).get();
+        ListTasksResponse cancelTasksResponse = clusterAdmin().prepareCancelTasks().setActions(TEST_TASK_ACTION.name()).get();
         assertEquals(1, cancelTasksResponse.getTasks().size());
 
         expectThrows(TaskCancelledException.class, future);

server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CrossClusterIT.java

@@ -11,7 +11,7 @@
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.action.ActionFuture;
 import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest;
-import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksResponse;
+import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.SearchResponse;
@@ -268,7 +268,7 @@ public void testCancel() throws Exception {
 
         final CancelTasksRequest cancelRequest = new CancelTasksRequest().setTargetTaskId(rootTask.taskId());
         cancelRequest.setWaitForCompletion(randomBoolean());
-        final ActionFuture<CancelTasksResponse> cancelFuture = client().admin().cluster().cancelTasks(cancelRequest);
+        final ActionFuture<ListTasksResponse> cancelFuture = client().admin().cluster().cancelTasks(cancelRequest);
         assertBusy(() -> {
             final Iterable<TransportService> transportServices = cluster("cluster_a").getInstances(TransportService.class);
             for (TransportService transportService : transportServices) {

server/src/main/java/org/elasticsearch/TransportVersions.java

@@ -184,6 +184,8 @@ static TransportVersion def(int id) {
     public static final TransportVersion ML_INFERENCE_REQUEST_INPUT_TYPE_ADDED = def(8_572_00_0);
     public static final TransportVersion ESQL_ENRICH_POLICY_CCQ_MODE = def(8_573_00_0);
     public static final TransportVersion DATE_HISTOGRAM_SUPPORT_DOWNSAMPLED_TZ = def(8_574_00_0);
+    public static final TransportVersion PEERFINDER_REPORTS_PEERS_MASTERS = def(8_575_00_0);
+    public static final TransportVersion ESQL_MULTI_CLUSTERS_ENRICH = def(8_576_00_0);
 
     /*
      * STOP! READ THIS FIRST! No, really,

server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/cancel/CancelTasksAction.java

@@ -9,16 +9,17 @@
 package org.elasticsearch.action.admin.cluster.node.tasks.cancel;
 
 import org.elasticsearch.action.ActionType;
+import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse;
 
 /**
  * ActionType for cancelling running tasks
  */
-public class CancelTasksAction extends ActionType<CancelTasksResponse> {
+public class CancelTasksAction extends ActionType<ListTasksResponse> {
 
     public static final CancelTasksAction INSTANCE = new CancelTasksAction();
     public static final String NAME = "cluster:admin/tasks/cancel";
 
     private CancelTasksAction() {
-        super(NAME, CancelTasksResponse::new);
+        super(NAME, ListTasksResponse::new);
     }
 }

server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/cancel/CancelTasksRequestBuilder.java

@@ -8,13 +8,14 @@
 
 package org.elasticsearch.action.admin.cluster.node.tasks.cancel;
 
+import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse;
 import org.elasticsearch.action.support.tasks.TasksRequestBuilder;
 import org.elasticsearch.client.internal.ElasticsearchClient;
 
 /**
  * Builder for the request to cancel tasks running on the specified nodes
  */
-public class CancelTasksRequestBuilder extends TasksRequestBuilder<CancelTasksRequest, CancelTasksResponse, CancelTasksRequestBuilder> {
+public class CancelTasksRequestBuilder extends TasksRequestBuilder<CancelTasksRequest, ListTasksResponse, CancelTasksRequestBuilder> {
 
     public CancelTasksRequestBuilder(ElasticsearchClient client) {
         super(client, CancelTasksAction.INSTANCE, new CancelTasksRequest());

server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/cancel/TransportCancelTasksAction.java

@@ -12,6 +12,7 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.FailedNodeException;
 import org.elasticsearch.action.TaskOperationFailure;
+import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.tasks.TransportTasksAction;
 import org.elasticsearch.cluster.service.ClusterService;
@@ -31,7 +32,7 @@
  * For a task to be cancellable it has to return an instance of
  * {@link CancellableTask} from {@link TransportRequest#createTask}
  */
-public class TransportCancelTasksAction extends TransportTasksAction<CancellableTask, CancelTasksRequest, CancelTasksResponse, TaskInfo> {
+public class TransportCancelTasksAction extends TransportTasksAction<CancellableTask, CancelTasksRequest, ListTasksResponse, TaskInfo> {
 
     @Inject
     public TransportCancelTasksAction(ClusterService clusterService, TransportService transportService, ActionFilters actionFilters) {
@@ -41,7 +42,7 @@ public TransportCancelTasksAction(ClusterService clusterService, TransportServic
             transportService,
             actionFilters,
             CancelTasksRequest::new,
-            CancelTasksResponse::new,
+            ListTasksResponse::new,
             TaskInfo::from,
             // Cancellation is usually lightweight, and runs on the transport thread if the task didn't even start yet, but some
             // implementations of CancellableTask#onCancelled() are nontrivial so we use GENERIC here. TODO could it be SAME?
@@ -50,13 +51,13 @@ public TransportCancelTasksAction(ClusterService clusterService, TransportServic
     }
 
     @Override
-    protected CancelTasksResponse newResponse(
+    protected ListTasksResponse newResponse(
         CancelTasksRequest request,
         List<TaskInfo> tasks,
         List<TaskOperationFailure> taskOperationFailures,
         List<FailedNodeException> failedNodeExceptions
     ) {
-        return new CancelTasksResponse(tasks, taskOperationFailures, failedNodeExceptions);
+        return new ListTasksResponse(tasks, taskOperationFailures, failedNodeExceptions);
     }
 
     protected List<CancellableTask> processTasks(CancelTasksRequest request) {