[8.13] GET /_all should return hidden indices with visible aliases (e…

…lastic#106975) (elastic#107441)

GET /_all should return hidden indices if they are accessible through a
visible alias. This is currently the behavior when resolution occurs in the 
security layer. This changes adds this behavior to name resolution
when security is not used.

(cherry picked from da81510)

docs/changelog/106975.yaml

@@ -0,0 +1,5 @@
+pr: 106975
+summary: GET /_all should return hidden indices with visible aliases
+area: Indices APIs
+type: bug
+issues: []

server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java

@@ -1151,32 +1151,36 @@ private WildcardExpressionResolver() {
         }
 
         /**
-         * Returns all the indices and all the datastreams, considering the open/closed, system, and hidden context parameters.
+         * Returns all the indices, datastreams, and aliases, considering the open/closed, system, and hidden context parameters.
          * Depending on the context, returns the names of the datastreams themselves or their backing indices.
          */
         public static Collection<String> resolveAll(Context context) {
-            List<String> resolvedExpressions = resolveEmptyOrTrivialWildcard(context);
-            if (context.includeDataStreams() == false) {
-                return resolvedExpressions;
-            } else {
-                Stream<IndexAbstraction> dataStreamsAbstractions = context.getState()
-                    .metadata()
-                    .getIndicesLookup()
-                    .values()
-                    .stream()
-                    .filter(indexAbstraction -> indexAbstraction.getType() == Type.DATA_STREAM)
-                    .filter(
-                        indexAbstraction -> indexAbstraction.isSystem() == false
-                            || context.systemIndexAccessPredicate.test(indexAbstraction.getName())
-                    );
-                if (context.getOptions().expandWildcardsHidden() == false) {
-                    dataStreamsAbstractions = dataStreamsAbstractions.filter(indexAbstraction -> indexAbstraction.isHidden() == false);
-                }
-                // dedup backing indices if expand hidden indices option is true
-                Set<String> resolvedIncludingDataStreams = expandToOpenClosed(context, dataStreamsAbstractions).collect(Collectors.toSet());
-                resolvedIncludingDataStreams.addAll(resolvedExpressions);
-                return resolvedIncludingDataStreams;
+            List<String> concreteIndices = resolveEmptyOrTrivialWildcard(context);
+
+            if (context.includeDataStreams() == false && context.getOptions().ignoreAliases()) {
+                return concreteIndices;
             }
+
+            Stream<IndexAbstraction> ias = context.getState()
+                .metadata()
+                .getIndicesLookup()
+                .values()
+                .stream()
+                .filter(ia -> context.getOptions().expandWildcardsHidden() || ia.isHidden() == false)
+                .filter(ia -> shouldIncludeIfDataStream(ia, context) || shouldIncludeIfAlias(ia, context))
+                .filter(ia -> ia.isSystem() == false || context.systemIndexAccessPredicate.test(ia.getName()));
+
+            Set<String> resolved = expandToOpenClosed(context, ias).collect(Collectors.toSet());
+            resolved.addAll(concreteIndices);
+            return resolved;
+        }
+
+        private static boolean shouldIncludeIfDataStream(IndexAbstraction ia, IndexNameExpressionResolver.Context context) {
+            return context.includeDataStreams() && ia.getType() == Type.DATA_STREAM;
+        }
+
+        private static boolean shouldIncludeIfAlias(IndexAbstraction ia, IndexNameExpressionResolver.Context context) {
+            return context.getOptions().ignoreAliases() == false && ia.getType() == Type.ALIAS;
         }
 
         /**

server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java

@@ -1215,9 +1215,9 @@ public void testHiddenAliasAndHiddenIndexResolution() {
             indexNames = indexNameExpressionResolver.concreteIndexNames(state, includeHiddenOptions, visibleAlias);
             assertThat(Arrays.asList(indexNames), containsInAnyOrder(visibleIndex, hiddenIndex));
 
-            // A total wildcards does not resolve the hidden index in this case
+            // total wildcards should also resolve both visible and hidden indices if there is a visible alias
             indexNames = indexNameExpressionResolver.concreteIndexNames(state, excludeHiddenOptions, "*");
-            assertThat(Arrays.asList(indexNames), containsInAnyOrder(visibleIndex));
+            assertThat(Arrays.asList(indexNames), containsInAnyOrder(visibleIndex, hiddenIndex));
         }
 
         {

server/src/test/java/org/elasticsearch/cluster/metadata/WildcardExpressionResolverTests.java

@@ -280,6 +280,129 @@ public void testAll() {
         assertThat(IndexNameExpressionResolver.resolveExpressions(noExpandContext, "_all").size(), equalTo(0));
     }
 
+    public void testAllAliases() {
+        {
+            // hidden index with hidden alias should not be returned
+            Metadata.Builder mdBuilder = Metadata.builder()
+                .put(
+                    indexBuilder("index-hidden-alias", true) // index hidden
+                        .state(State.OPEN)
+                        .putAlias(AliasMetadata.builder("alias-hidden").isHidden(true)) // alias hidden
+                );
+
+            ClusterState state = ClusterState.builder(new ClusterName("_name")).metadata(mdBuilder).build();
+
+            IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(
+                state,
+                IndicesOptions.lenientExpandOpen(), // don't include hidden
+                SystemIndexAccessLevel.NONE
+            );
+            assertThat(newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolveAll(context)), equalTo(newHashSet()));
+        }
+
+        {
+            // hidden index with visible alias should be returned
+            Metadata.Builder mdBuilder = Metadata.builder()
+                .put(
+                    indexBuilder("index-visible-alias", true) // index hidden
+                        .state(State.OPEN)
+                        .putAlias(AliasMetadata.builder("alias-visible").isHidden(false)) // alias visible
+                );
+
+            ClusterState state = ClusterState.builder(new ClusterName("_name")).metadata(mdBuilder).build();
+
+            IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(
+                state,
+                IndicesOptions.lenientExpandOpen(), // don't include hidden
+                SystemIndexAccessLevel.NONE
+            );
+            assertThat(
+                newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolveAll(context)),
+                equalTo(newHashSet("index-visible-alias"))
+            );
+        }
+    }
+
+    public void testAllDataStreams() {
+
+        String dataStreamName = "foo_logs";
+        long epochMillis = randomLongBetween(1580536800000L, 1583042400000L);
+        IndexMetadata firstBackingIndexMetadata = createBackingIndex(dataStreamName, 1, epochMillis).build();
+
+        IndicesOptions indicesAndAliasesOptions = IndicesOptions.fromOptions(
+            randomBoolean(),
+            randomBoolean(),
+            true,
+            false,
+            true,
+            false,
+            false,
+            false
+        );
+
+        {
+            // visible data streams should be returned by _all even show backing indices are hidden
+            Metadata.Builder mdBuilder = Metadata.builder()
+                .put(firstBackingIndexMetadata, true)
+                .put(DataStreamTestHelper.newInstance(dataStreamName, List.of(firstBackingIndexMetadata.getIndex())));
+
+            ClusterState state = ClusterState.builder(new ClusterName("_name")).metadata(mdBuilder).build();
+
+            IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(
+                state,
+                indicesAndAliasesOptions,
+                false,
+                false,
+                true,
+                SystemIndexAccessLevel.NONE,
+                NONE,
+                NONE
+            );
+
+            assertThat(
+                newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolveAll(context)),
+                equalTo(newHashSet(DataStream.getDefaultBackingIndexName("foo_logs", 1, epochMillis)))
+            );
+        }
+
+        {
+            // if data stream itself is hidden, backing indices should not be returned
+            boolean hidden = true;
+            var dataStream = new DataStream(
+                dataStreamName,
+                List.of(firstBackingIndexMetadata.getIndex()),
+                1,
+                null,
+                hidden,
+                false,
+                false,
+                false,
+                null,
+                null,
+                false,
+                List.of(),
+                false
+            );
+
+            Metadata.Builder mdBuilder = Metadata.builder().put(firstBackingIndexMetadata, true).put(dataStream);
+
+            ClusterState state = ClusterState.builder(new ClusterName("_name")).metadata(mdBuilder).build();
+
+            IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(
+                state,
+                indicesAndAliasesOptions,
+                false,
+                false,
+                true,
+                SystemIndexAccessLevel.NONE,
+                NONE,
+                NONE
+            );
+
+            assertThat(newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolveAll(context)), equalTo(newHashSet()));
+        }
+    }
+
     public void testResolveEmpty() {
         Metadata.Builder mdBuilder = Metadata.builder()
             .put(