Fix searching a filtered and unfiltered data stream alias (#95865) (#…

…96010)

This fixes the case when a `_search` request is targeting multiple
aliases that point to the _same data stream_, some being filtered and
some unfiltered.

Before this fix, such a scenario would apply the encountered filters as
`should` conditions on a `BoolQuery`, despite the request containing an
unfiltered (match_all) alias for the same data stream.

This fixes this scenario to disregard the filters when an unfiltered
alias is encountered in the resolved expressions, targeting the same
data stream.

This also fixes the same scenario for a request that targets the data
stream name and a filtered alias (the expected result is that the filter
should be disregarded)

Fixes #95786

docs/changelog/95865.yaml

@@ -0,0 +1,6 @@
+pr: 95865
+summary: Fix searching a filtered and unfiltered data stream alias
+area: Data streams
+type: bug
+issues:
+ - 95786

modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java

@@ -133,6 +133,7 @@
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.hasItemInArray;
+import static org.hamcrest.Matchers.hasKey;
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
@@ -801,6 +802,53 @@ public void testDataSteamAliasWithFilter() throws Exception {
         assertSearchHits(searchResponse, "1");
     }
 
+    public void testSearchFilteredAndUnfilteredAlias() throws Exception {
+        putComposableIndexTemplate("id1", List.of("logs-*"));
+        String dataStreamName = "logs-foobar";
+        client().prepareIndex(dataStreamName)
+            .setId("1")
+            .setSource("{\"@timestamp\": \"2022-12-12\", \"type\": \"x\"}", XContentType.JSON)
+            .setOpType(DocWriteRequest.OpType.CREATE)
+            .get();
+        client().prepareIndex(dataStreamName)
+            .setId("2")
+            .setSource("{\"@timestamp\": \"2022-12-12\", \"type\": \"y\"}", XContentType.JSON)
+            .setOpType(DocWriteRequest.OpType.CREATE)
+            .get();
+        refresh(dataStreamName);
+
+        AliasActions addFilteredAliasAction = new AliasActions(AliasActions.Type.ADD).index(dataStreamName)
+            .aliases("foo")
+            .filter(Map.of("term", Map.of("type", Map.of("value", "y"))));
+        AliasActions addUnfilteredAliasAction = new AliasActions(AliasActions.Type.ADD).index(dataStreamName).aliases("bar");
+
+        IndicesAliasesRequest aliasesAddRequest = new IndicesAliasesRequest();
+        aliasesAddRequest.addAliasAction(addFilteredAliasAction);
+        aliasesAddRequest.addAliasAction(addUnfilteredAliasAction);
+        assertAcked(client().admin().indices().aliases(aliasesAddRequest).actionGet());
+        GetAliasesResponse response = client().admin().indices().getAliases(new GetAliasesRequest()).actionGet();
+        assertThat(response.getDataStreamAliases(), hasKey("logs-foobar"));
+        assertThat(
+            response.getDataStreamAliases().get("logs-foobar"),
+            containsInAnyOrder(
+                new DataStreamAlias("bar", List.of("logs-foobar"), null, null),
+                new DataStreamAlias(
+                    "foo",
+                    List.of("logs-foobar"),
+                    null,
+                    Map.of("logs-foobar", Map.of("term", Map.of("type", Map.of("value", "y"))))
+                )
+            )
+        );
+
+        // Searching the filtered and unfiltered aliases should return all results (unfiltered):
+        SearchResponse searchResponse = client().prepareSearch("foo", "bar").get();
+        assertSearchHits(searchResponse, "1", "2");
+        // Searching the data stream name and the filtered alias should return all results (unfiltered):
+        searchResponse = client().prepareSearch("foo", dataStreamName).get();
+        assertSearchHits(searchResponse, "1", "2");
+    }
+
     public void testRandomDataSteamAliasesUpdate() throws Exception {
         putComposableIndexTemplate("id1", List.of("log-*"));
 

server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java

@@ -701,19 +701,41 @@ public String[] indexAliases(
         IndexAbstraction ia = state.metadata().getIndicesLookup().get(index);
         DataStream dataStream = ia.getParentDataStream();
         if (dataStream != null) {
+            if (skipIdentity == false && resolvedExpressions.contains(dataStream.getName())) {
+                // skip the filters when the request targets the data stream name
+                return null;
+            }
             Map<String, DataStreamAlias> dataStreamAliases = state.metadata().dataStreamAliases();
-            Stream<DataStreamAlias> stream;
+            List<DataStreamAlias> aliasesForDataStream;
             if (iterateIndexAliases(dataStreamAliases.size(), resolvedExpressions.size())) {
-                stream = dataStreamAliases.values()
+                aliasesForDataStream = dataStreamAliases.values()
                     .stream()
-                    .filter(dataStreamAlias -> resolvedExpressions.contains(dataStreamAlias.getName()));
+                    .filter(dataStreamAlias -> resolvedExpressions.contains(dataStreamAlias.getName()))
+                    .filter(dataStreamAlias -> dataStreamAlias.getDataStreams().contains(dataStream.getName()))
+                    .toList();
             } else {
-                stream = resolvedExpressions.stream().map(dataStreamAliases::get).filter(Objects::nonNull);
+                aliasesForDataStream = resolvedExpressions.stream()
+                    .map(dataStreamAliases::get)
+                    .filter(dataStreamAlias -> dataStreamAlias != null && dataStreamAlias.getDataStreams().contains(dataStream.getName()))
+                    .toList();
+            }
+
+            List<String> requiredAliases = null;
+            for (DataStreamAlias dataStreamAlias : aliasesForDataStream) {
+                if (requiredDataStreamAlias.test(dataStreamAlias)) {
+                    if (requiredAliases == null) {
+                        requiredAliases = new ArrayList<>(aliasesForDataStream.size());
+                    }
+                    requiredAliases.add(dataStreamAlias.getName());
+                } else {
+                    // we have a non-required alias for this data stream so no need to check further
+                    return null;
+                }
+            }
+            if (requiredAliases == null) {
+                return null;
             }
-            return stream.filter(dataStreamAlias -> dataStreamAlias.getDataStreams().contains(dataStream.getName()))
-                .filter(requiredDataStreamAlias)
-                .map(DataStreamAlias::getName)
-                .toArray(String[]::new);
+            return requiredAliases.toArray(Strings.EMPTY_ARRAY);
         } else {
             final Map<String, AliasMetadata> indexAliases = indexMetadata.getAliases();
             final AliasMetadata[] aliasCandidates;

server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java

@@ -1672,9 +1672,8 @@ public void testIndexAliasesDataStreamAliases() {
             String[] result = indexNameExpressionResolver.indexAliases(state, index, x -> true, x -> true, false, resolvedExpressions);
             assertThat(result, nullValue());
         }
-
         {
-            // Only resolve aliases that refer to dataStreamName1 where the filter is not null
+            // Null is returned, because the wildcard expands to a list of aliases containing an unfiltered alias for dataStreamName1
             Set<String> resolvedExpressions = indexNameExpressionResolver.resolveExpressions(state, "l*");
             String index = backingIndex1.getIndex().getName();
             String[] result = indexNameExpressionResolver.indexAliases(
@@ -1685,7 +1684,49 @@ public void testIndexAliasesDataStreamAliases() {
                 true,
                 resolvedExpressions
             );
-            assertThat(result, arrayContainingInAnyOrder("logs", "logs_foo"));
+            assertThat(result, nullValue());
+        }
+        {
+            // Null is returned, because an unfiltered alias is targeting the same data stream
+            Set<String> resolvedExpressions = indexNameExpressionResolver.resolveExpressions(state, "logs_bar", "logs");
+            String index = backingIndex1.getIndex().getName();
+            String[] result = indexNameExpressionResolver.indexAliases(
+                state,
+                index,
+                x -> true,
+                DataStreamAlias::filteringRequired,
+                true,
+                resolvedExpressions
+            );
+            assertThat(result, nullValue());
+        }
+        {
+            // The filtered alias is returned because although we target the data stream name, skipIdentity is true
+            Set<String> resolvedExpressions = indexNameExpressionResolver.resolveExpressions(state, dataStreamName1, "logs");
+            String index = backingIndex1.getIndex().getName();
+            String[] result = indexNameExpressionResolver.indexAliases(
+                state,
+                index,
+                x -> true,
+                DataStreamAlias::filteringRequired,
+                true,
+                resolvedExpressions
+            );
+            assertThat(result, arrayContainingInAnyOrder("logs"));
+        }
+        {
+            // Null is returned because we target the data stream name and skipIdentity is false
+            Set<String> resolvedExpressions = indexNameExpressionResolver.resolveExpressions(state, dataStreamName1, "logs");
+            String index = backingIndex1.getIndex().getName();
+            String[] result = indexNameExpressionResolver.indexAliases(
+                state,
+                index,
+                x -> true,
+                DataStreamAlias::filteringRequired,
+                false,
+                resolvedExpressions
+            );
+            assertThat(result, nullValue());
         }
     }
 

server/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java

@@ -88,6 +88,7 @@
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasToString;
 import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.nullValue;
 import static org.mockito.Mockito.mock;
@@ -733,14 +734,16 @@ public void testBuildAliasFilterDataStreamAliases() {
             assertThat(filter.should(), containsInAnyOrder(QueryBuilders.termQuery("foo", "baz"), QueryBuilders.termQuery("foo", "bax")));
         }
         {
+            // querying an unfiltered and a filtered alias for the same data stream should drop the filters
             String index = backingIndex1.getIndex().getName();
             AliasFilter result = indicesService.buildAliasFilter(state, index, Set.of("logs_foo", "logs", "logs_bar"));
-            assertThat(result.getAliases(), arrayContainingInAnyOrder("logs_foo", "logs"));
-            BoolQueryBuilder filter = (BoolQueryBuilder) result.getQueryBuilder();
-            assertThat(filter.filter(), empty());
-            assertThat(filter.must(), empty());
-            assertThat(filter.mustNot(), empty());
-            assertThat(filter.should(), containsInAnyOrder(QueryBuilders.termQuery("foo", "baz"), QueryBuilders.termQuery("foo", "bar")));
+            assertThat(result, is(AliasFilter.EMPTY));
+        }
+        {
+            // similarly, querying the data stream name and a filtered alias should drop the filter
+            String index = backingIndex1.getIndex().getName();
+            AliasFilter result = indicesService.buildAliasFilter(state, index, Set.of("logs", dataStreamName1));
+            assertThat(result, is(AliasFilter.EMPTY));
         }
     }
 }