[Profiling] Query in parallel on content nodes (elastic#104600)

In order to take advantage of inherent parallelism of modern SSDs, we
slice keys and issue multiple mgets concurrently. In elastic#103061 we have
introduced an additional heuristic to disable that behavior on the warm
and cold tier which usually use spinning disks. We have unintentionally
also disabled the behavior on content nodes, i.e. on any clusters which
do not use data tiers. With this commit we explicitly exclude content
nodes from the heuristic so they can benefit from speedups due to
concurrent mgets.

Relates elastic#103061

docs/changelog/104600.yaml

@@ -0,0 +1,5 @@
+pr: 104600
+summary: "[Profiling] Query in parallel on content nodes"
+area: Application
+type: bug
+issues: []

x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/IndexAllocation.java

@@ -55,6 +55,11 @@ static boolean isAnyAssignedToNode(ClusterState state, List<Index> indices, Pred
      * @return <code>true</code> iff at least one index is allocated to either a warm or cold data node.
      */
     static boolean isAnyOnWarmOrColdTier(ClusterState state, List<Index> indices) {
-        return isAnyAssignedToNode(state, indices, n -> DataTier.isWarmNode(n) || DataTier.isColdNode(n));
+        return isAnyAssignedToNode(
+            state,
+            indices,
+            // a content node is never considered a warm or cold node
+            n -> DataTier.isContentNode(n) == false && (DataTier.isWarmNode(n) || DataTier.isColdNode(n))
+        );
     }
 }

x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/IndexAllocationTests.java

@@ -36,6 +36,7 @@
 import java.util.UUID;
 
 public class IndexAllocationTests extends ESTestCase {
+    private final Index content = idx("content");
     private final Index hot = idx("hot");
     private final Index warm = idx("warm");
     private final Index cold = idx("cold");
@@ -49,6 +50,10 @@ public void testOtherIndicesNotOnWarmColdTier() {
         assertFalse(IndexAllocation.isAnyOnWarmOrColdTier(clusterState(), List.of(hot, frozen)));
     }
 
+    public void testIndicesOnContentNodeNotOnWarmColdTier() {
+        assertFalse(IndexAllocation.isAnyOnWarmOrColdTier(clusterState(), List.of(content)));
+    }
+
     public void testIndicesOnWarmColdTier() {
         assertTrue(IndexAllocation.isAnyOnWarmOrColdTier(clusterState(), List.of(warm)));
         assertTrue(IndexAllocation.isAnyOnWarmOrColdTier(clusterState(), List.of(cold)));
@@ -73,6 +78,20 @@ private ClusterState clusterState() {
         DiscoveryNode node = DiscoveryNodeUtils.create("node");
         DiscoveryNodes.Builder nodesBuilder = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node);
 
+        nodesBuilder.add(
+            DiscoveryNodeUtils.builder("n-" + content.getName())
+                .roles(
+                    Set.of(
+                        // content nodes have all roles
+                        DiscoveryNodeRole.DATA_CONTENT_NODE_ROLE,
+                        DiscoveryNodeRole.DATA_HOT_NODE_ROLE,
+                        DiscoveryNodeRole.DATA_WARM_NODE_ROLE,
+                        DiscoveryNodeRole.DATA_COLD_NODE_ROLE,
+                        DiscoveryNodeRole.DATA_FROZEN_NODE_ROLE
+                    )
+                )
+                .build()
+        );
         nodesBuilder.add(DiscoveryNodeUtils.builder("n-" + hot.getName()).roles(Set.of(DiscoveryNodeRole.DATA_HOT_NODE_ROLE)).build());
         nodesBuilder.add(DiscoveryNodeUtils.builder("n-" + warm.getName()).roles(Set.of(DiscoveryNodeRole.DATA_WARM_NODE_ROLE)).build());
         nodesBuilder.add(DiscoveryNodeUtils.builder("n-" + cold.getName()).roles(Set.of(DiscoveryNodeRole.DATA_COLD_NODE_ROLE)).build());
@@ -82,7 +101,7 @@ private ClusterState clusterState() {
 
         RoutingTable.Builder routingTableBuilder = RoutingTable.builder();
         Map<String, IndexMetadata> indices = new HashMap<>();
-        for (Index index : List.of(hot, warm, cold, frozen)) {
+        for (Index index : List.of(content, hot, warm, cold, frozen)) {
             indices.put(index.getName(), metadata(index));
             ShardRouting shardRouting = ShardRouting.newUnassigned(
                 new ShardId(index, 0),