System indices treated as restricted indices

client/rest-high-level/qa/ssl-enabled/src/javaRestTest/java/org/elasticsearch/client/EnrollmentIT.java

@@ -59,6 +59,11 @@ protected Settings restClientSettings() {
             .build();
     }
 
+    @Override
+    protected Settings restAdminSettings() {
+        return restClientSettings();
+    }
+
     public void testEnrollNode() throws Exception {
         final NodeEnrollmentResponse nodeEnrollmentResponse =
             execute(highLevelClient().security()::enrollNode, highLevelClient().security()::enrollNodeAsync, RequestOptions.DEFAULT);

client/rest-high-level/qa/ssl-enabled/src/javaRestTest/java/org/elasticsearch/client/documentation/EnrollmentDocumentationIT.java

@@ -52,6 +52,11 @@ protected Settings restClientSettings() {
             .build();
     }
 
+    @Override
+    protected Settings restAdminSettings() {
+        return restClientSettings();
+    }
+
     public void testNodeEnrollment() throws Exception {
         RestHighLevelClient client = highLevelClient();
 

client/rest-high-level/src/test/java/org/elasticsearch/client/ESRestHighLevelClientTestCase.java

@@ -22,6 +22,7 @@
 import org.elasticsearch.client.cluster.RemoteInfoRequest;
 import org.elasticsearch.client.cluster.RemoteInfoResponse;
 import org.elasticsearch.client.indices.CreateIndexRequest;
+import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.core.Booleans;
 import org.elasticsearch.core.CheckedRunnable;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -66,6 +67,7 @@ public abstract class ESRestHighLevelClientTestCase extends ESRestTestCase {
     protected static final String CONFLICT_PIPELINE_ID = "conflict_pipeline";
 
     private static RestHighLevelClient restHighLevelClient;
+    private static RestHighLevelClient adminRestHighLevelClient;
     private static boolean async = Booleans.parseBoolean(System.getProperty("tests.rest.async", "false"));
 
     @Before
@@ -74,18 +76,35 @@ public void initHighLevelClient() throws IOException {
         if (restHighLevelClient == null) {
             restHighLevelClient = new HighLevelClient(client());
         }
+        if (adminRestHighLevelClient == null) {
+            adminRestHighLevelClient = new HighLevelClient(adminClient());
+        }
     }
 
     @AfterClass
     public static void cleanupClient() throws IOException {
         IOUtils.close(restHighLevelClient);
+        IOUtils.close(adminRestHighLevelClient);
         restHighLevelClient = null;
+        adminRestHighLevelClient = null;
     }
 
     protected static RestHighLevelClient highLevelClient() {
         return restHighLevelClient;
     }
 
+    @Override
+    protected Settings restAdminSettings() {
+        String token = basicAuthHeaderValue("admin_user", new SecureString("admin-password".toCharArray()));
+        return Settings.builder()
+            .put(ThreadContext.PREFIX + ".Authorization", token)
+            .build();
+    }
+
+    protected static RestHighLevelClient adminHighLevelClient() {
+        return adminRestHighLevelClient;
+    }
+
     /**
      * Executes the provided request using either the sync method or its async variant, both provided as functions
      */

client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningGetResultsIT.java

@@ -274,7 +274,7 @@ private void addModelSnapshotIndexRequests(BulkRequest bulkRequest) throws IOExc
 
     @After
     public void deleteJob() throws IOException {
-        new MlTestStateCleaner(logger, highLevelClient()).clearMlMetadata();
+        new MlTestStateCleaner(logger, adminHighLevelClient()).clearMlMetadata();
     }
 
     public void testGetModelSnapshots() throws IOException {

client/rest-high-level/src/test/java/org/elasticsearch/client/MlTestStateCleaner.java

@@ -12,6 +12,7 @@
 import org.elasticsearch.action.ingest.DeletePipelineRequest;
 import org.elasticsearch.client.core.PageParams;
 import org.elasticsearch.client.feature.ResetFeaturesRequest;
+import org.elasticsearch.client.feature.ResetFeaturesResponse;
 import org.elasticsearch.client.ml.GetTrainedModelsStatsRequest;
 
 import java.io.IOException;
@@ -36,7 +37,12 @@ public MlTestStateCleaner(Logger logger, RestHighLevelClient client) {
     public void clearMlMetadata() throws IOException {
         deleteAllTrainedModelIngestPipelines();
         // This resets all features, not just ML, but they should have been getting reset between tests anyway so it shouldn't matter
-        client.features().resetFeatures(new ResetFeaturesRequest(), RequestOptions.DEFAULT);
+        ResetFeaturesResponse response = client.features().resetFeatures(new ResetFeaturesRequest(), RequestOptions.DEFAULT);
+        for (ResetFeaturesResponse.ResetFeatureStateStatus status : response.getFeatureResetStatuses()) {
+            if (status.getStatus().equals("FAILURE")) {
+                logger.info("Reset state response: " + status.getException());
+            }
+        }
     }
 
     @SuppressWarnings("unchecked")

server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java

@@ -8,6 +8,7 @@
 
 package org.elasticsearch.cluster.metadata;
 
+import org.apache.lucene.util.automaton.Automaton;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.IndicesRequest;
@@ -786,6 +787,10 @@ public Predicate<String> getSystemIndexAccessPredicate() {
         return systemIndexAccessLevelPredicate;
     }
 
+    public Automaton getSystemNameAutomaton() {
+        return systemIndices.getSystemNameAutomaton();
+    }
+
     public Predicate<String> getNetNewSystemIndexPredicate() {
         return systemIndices::isNetNewSystemIndex;
     }

server/src/main/java/org/elasticsearch/indices/SystemIndices.java

@@ -42,7 +42,6 @@
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Optional;
-import java.util.Set;
 import java.util.function.Predicate;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
@@ -66,10 +65,12 @@ public class SystemIndices {
         TASKS_FEATURE_NAME, new Feature(TASKS_FEATURE_NAME, "Manages task results", List.of(TASKS_DESCRIPTOR))
     );
 
-    private final CharacterRunAutomaton systemIndexAutomaton;
-    private final CharacterRunAutomaton systemDataStreamIndicesAutomaton;
+    private final Automaton systemNameAutomaton;
     private final CharacterRunAutomaton netNewSystemIndexAutomaton;
-    private final Predicate<String> systemDataStreamAutomaton;
+    private final CharacterRunAutomaton systemNameRunAutomaton;
+    private final CharacterRunAutomaton systemIndexRunAutomaton;
+    private final CharacterRunAutomaton systemDataStreamIndicesRunAutomaton;
+    private final Predicate<String> systemDataStreamPredicate;
     private final Map<String, Feature> featureDescriptors;
     private final Map<String, CharacterRunAutomaton> productToSystemIndicesMatcher;
     private final ExecutorSelector executorSelector;
@@ -83,12 +84,19 @@ public SystemIndices(Map<String, Feature> pluginAndModulesDescriptors) {
         featureDescriptors = buildSystemIndexDescriptorMap(pluginAndModulesDescriptors);
         checkForOverlappingPatterns(featureDescriptors);
         checkForDuplicateAliases(this.getSystemIndexDescriptors());
-        this.systemIndexAutomaton = buildIndexCharacterRunAutomaton(featureDescriptors);
+        Automaton systemIndexAutomata = buildIndexAutomaton(featureDescriptors);
+        this.systemIndexRunAutomaton = new CharacterRunAutomaton(systemIndexAutomata);
+        Automaton systemDataStreamIndicesAutomata = buildDataStreamBackingIndicesAutomaton(featureDescriptors);
+        this.systemDataStreamIndicesRunAutomaton = new CharacterRunAutomaton(systemDataStreamIndicesAutomata);
+        this.systemDataStreamPredicate = buildDataStreamNamePredicate(featureDescriptors);
         this.netNewSystemIndexAutomaton = buildNetNewIndexCharacterRunAutomaton(featureDescriptors);
-        this.systemDataStreamIndicesAutomaton = buildDataStreamBackingIndicesAutomaton(featureDescriptors);
-        this.systemDataStreamAutomaton = buildDataStreamNamePredicate(featureDescriptors);
         this.productToSystemIndicesMatcher = getProductToSystemIndicesMap(featureDescriptors);
         this.executorSelector = new ExecutorSelector(this);
+        this.systemNameAutomaton = MinimizationOperations.minimize(
+            Operations.union(List.of(systemIndexAutomata, systemDataStreamIndicesAutomata, buildDataStreamAutomaton(featureDescriptors))),
+            Integer.MAX_VALUE
+        );
+        this.systemNameRunAutomaton = new CharacterRunAutomaton(systemNameAutomaton);
     }
 
     private static void checkForDuplicateAliases(Collection<SystemIndexDescriptor> descriptors) {
@@ -150,7 +158,7 @@ private static Map<String, CharacterRunAutomaton> getProductToSystemIndicesMap(M
      * is checked against index names, aliases, data stream names, and the names of indices that back a system data stream.
      */
     public boolean isSystemName(String name) {
-        return isSystemIndex(name) || isSystemDataStream(name) || isSystemIndexBackingDataStream(name);
+        return systemNameRunAutomaton.run(name);
     }
 
     /**
@@ -169,22 +177,30 @@ public boolean isSystemIndex(Index index) {
      * @return true if the index name matches a pattern from a {@link SystemIndexDescriptor}
      */
     public boolean isSystemIndex(String indexName) {
-        return systemIndexAutomaton.run(indexName);
+        return systemIndexRunAutomaton.run(indexName);
     }
 
     /**
      * Determines whether the provided name matches that of a system data stream that has been defined by a
      * {@link SystemDataStreamDescriptor}
      */
     public boolean isSystemDataStream(String name) {
-        return systemDataStreamAutomaton.test(name);
+        return systemDataStreamPredicate.test(name);
     }
 
     /**
      * Determines whether the provided name matches that of an index that backs a system data stream.
      */
     public boolean isSystemIndexBackingDataStream(String name) {
-        return systemDataStreamIndicesAutomaton.run(name);
+        return systemDataStreamIndicesRunAutomaton.run(name);
+    }
+
+    /**
+     * @return An {@link Automaton} that tests whether strings are names of system indices, aliases, or
+     * data streams.
+     */
+    public Automaton getSystemNameAutomaton() {
+        return systemNameAutomaton;
     }
 
     public boolean isNetNewSystemIndex(String indexName) {
@@ -293,11 +309,11 @@ public Map<String, Feature> getFeatures() {
         return featureDescriptors;
     }
 
-    private static CharacterRunAutomaton buildIndexCharacterRunAutomaton(Map<String, Feature> descriptors) {
+    private static Automaton buildIndexAutomaton(Map<String, Feature> descriptors) {
         Optional<Automaton> automaton = descriptors.values().stream()
             .map(SystemIndices::featureToIndexAutomaton)
             .reduce(Operations::union);
-        return new CharacterRunAutomaton(MinimizationOperations.minimize(automaton.orElse(EMPTY), Integer.MAX_VALUE));
+        return MinimizationOperations.minimize(automaton.orElse(EMPTY), Integer.MAX_VALUE);
     }
 
     private static CharacterRunAutomaton buildNetNewIndexCharacterRunAutomaton(Map<String, Feature> featureDescriptors) {
@@ -317,19 +333,26 @@ private static Automaton featureToIndexAutomaton(Feature feature) {
         return systemIndexAutomaton.orElse(EMPTY);
     }
 
-    private static Predicate<String> buildDataStreamNamePredicate(Map<String, Feature> descriptors) {
-        Set<String> systemDataStreamNames = descriptors.values().stream()
+    private static Automaton buildDataStreamAutomaton(Map<String, Feature> descriptors) {
+        Optional<Automaton> automaton = descriptors.values().stream()
             .flatMap(feature -> feature.getDataStreamDescriptors().stream())
             .map(SystemDataStreamDescriptor::getDataStreamName)
-            .collect(Collectors.toUnmodifiableSet());
-        return systemDataStreamNames::contains;
+            .map(dsName -> SystemIndexDescriptor.buildAutomaton(dsName, null))
+            .reduce(Operations::union);
+
+        return automaton.isPresent() ? MinimizationOperations.minimize(automaton.get(), Integer.MAX_VALUE) : EMPTY;
+    }
+
+    private static Predicate<String> buildDataStreamNamePredicate(Map<String, Feature> descriptors) {
+        CharacterRunAutomaton characterRunAutomaton = new CharacterRunAutomaton(buildDataStreamAutomaton(descriptors));
+        return characterRunAutomaton::run;
     }
 
-    private static CharacterRunAutomaton buildDataStreamBackingIndicesAutomaton(Map<String, Feature> descriptors) {
+    private static Automaton buildDataStreamBackingIndicesAutomaton(Map<String, Feature> descriptors) {
         Optional<Automaton> automaton = descriptors.values().stream()
             .map(SystemIndices::featureToDataStreamBackingIndicesAutomaton)
             .reduce(Operations::union);
-        return new CharacterRunAutomaton(automaton.orElse(EMPTY));
+        return MinimizationOperations.minimize(automaton.orElse(EMPTY), Integer.MAX_VALUE);
     }
 
     private static Automaton featureToDataStreamBackingIndicesAutomaton(Feature feature) {
@@ -343,7 +366,7 @@ private static Automaton featureToDataStreamBackingIndicesAutomaton(Feature feat
     }
 
     public SystemDataStreamDescriptor validateDataStreamAccess(String dataStreamName, ThreadContext threadContext) {
-        if (systemDataStreamAutomaton.test(dataStreamName)) {
+        if (systemDataStreamPredicate.test(dataStreamName)) {
             SystemDataStreamDescriptor dataStreamDescriptor = featureDescriptors.values().stream()
                 .flatMap(feature -> feature.getDataStreamDescriptors().stream())
                 .filter(descriptor -> descriptor.getDataStreamName().equals(dataStreamName))
@@ -405,6 +428,7 @@ IllegalArgumentException dataStreamAccessException(@Nullable String product, Str
     /**
      * Determines what level of system index access should be allowed in the current context.
      *
+     * @param threadContext the current thread context that has headers associated with the current request
      * @return {@link SystemIndexAccessLevel#ALL} if unrestricted system index access should be allowed,
      * {@link SystemIndexAccessLevel#RESTRICTED} if a subset of system index access should be allowed, or
      * {@link SystemIndexAccessLevel#NONE} if no system index access should be allowed.

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/async/AsyncTaskIndexService.java

@@ -142,7 +142,7 @@ private static XContentBuilder mappings() {
 
     public static SystemIndexDescriptor getSystemIndexDescriptor() {
         return SystemIndexDescriptor.builder()
-            .setIndexPattern(XPackPlugin.ASYNC_RESULTS_INDEX)
+            .setIndexPattern(XPackPlugin.ASYNC_RESULTS_INDEX + "*")
             .setDescription("Async search results")
             .setPrimaryIndex(XPackPlugin.ASYNC_RESULTS_INDEX)
             .setMappings(mappings())