Introduce system index types including external

This commit introduces system index types that will be used to
differentiate behavior. Previously system indices were all treated the
same regardless of whether they belonged to Elasticsearch, a stack
component, or one of our solutions. Upon further discussion and
analysis this decision was not in the best interest of the various
teams and instead a new type of system index was needed. These system
indices will be referred to as external system indices. Within external
system indices, an option exists for these indices to be managed by
Elasticsearch or to be managed by the external product.

In order to represent this within Elasticsearch, each system index will
have a type and this type will be used to control behavior.

Closes elastic#67383

modules/kibana/src/main/java/org/elasticsearch/kibana/KibanaPlugin.java

@@ -8,130 +8,53 @@
 
 package org.elasticsearch.kibana;
 
-import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.common.settings.ClusterSettings;
-import org.elasticsearch.common.settings.IndexScopedSettings;
-import org.elasticsearch.common.settings.Setting;
-import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.settings.SettingsFilter;
-import org.elasticsearch.index.reindex.RestDeleteByQueryAction;
 import org.elasticsearch.indices.SystemIndexDescriptor;
+import org.elasticsearch.indices.SystemIndexDescriptor.Type;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.SystemIndexPlugin;
-import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.RestController;
-import org.elasticsearch.rest.RestHandler;
-import org.elasticsearch.rest.action.admin.indices.RestCreateIndexAction;
-import org.elasticsearch.rest.action.admin.indices.RestGetAliasesAction;
-import org.elasticsearch.rest.action.admin.indices.RestGetIndicesAction;
-import org.elasticsearch.rest.action.admin.indices.RestIndexPutAliasAction;
-import org.elasticsearch.rest.action.admin.indices.RestRefreshAction;
-import org.elasticsearch.rest.action.admin.indices.RestUpdateSettingsAction;
-import org.elasticsearch.rest.action.document.RestBulkAction;
-import org.elasticsearch.rest.action.document.RestDeleteAction;
-import org.elasticsearch.rest.action.document.RestGetAction;
-import org.elasticsearch.rest.action.document.RestIndexAction;
-import org.elasticsearch.rest.action.document.RestIndexAction.AutoIdHandler;
-import org.elasticsearch.rest.action.document.RestIndexAction.CreateHandler;
-import org.elasticsearch.rest.action.document.RestMultiGetAction;
-import org.elasticsearch.rest.action.document.RestUpdateAction;
-import org.elasticsearch.rest.action.search.RestClearScrollAction;
-import org.elasticsearch.rest.action.search.RestSearchAction;
-import org.elasticsearch.rest.action.search.RestSearchScrollAction;
 
 import java.util.Collection;
 import java.util.List;
-import java.util.function.Function;
-import java.util.function.Supplier;
-import java.util.stream.Collectors;
 
 public class KibanaPlugin extends Plugin implements SystemIndexPlugin {
 
-    public static final Setting<List<String>> KIBANA_INDEX_NAMES_SETTING = Setting.listSetting(
-        "kibana.system_indices",
-        List.of(".kibana", ".kibana_*", ".reporting-*", ".apm-agent-configuration", ".apm-custom-link"),
-        Function.identity(),
-        Property.NodeScope
-    );
+    public static final SystemIndexDescriptor KIBANA_INDEX_DESCRIPTOR = SystemIndexDescriptor.builder()
+        .setIndexPattern(".kibana_*")
+        .setDescription("Kibana saved objects system index")
+        .setAliasName(".kibana")
+        .setType(Type.EXTERNAL_UNMANAGED)
+        .setAllowedStackComponents(List.of("kibana"))
+        .build();
+
+    public static final SystemIndexDescriptor REPORTING_INDEX_DESCRIPTOR = SystemIndexDescriptor.builder()
+        .setIndexPattern(".reporting-*")
+        .setDescription("system index for reporting")
+        .setType(Type.EXTERNAL_UNMANAGED)
+        .setAllowedStackComponents(List.of("kibana"))
+        .build();
+
+    public static final SystemIndexDescriptor APM_AGENT_CONFIG_INDEX_DESCRIPTOR = SystemIndexDescriptor.builder()
+        .setIndexPattern(".apm-agent-configuration")
+        .setDescription("system index for APM agent configuration")
+        .setType(Type.EXTERNAL_UNMANAGED)
+        .setAllowedStackComponents(List.of("kibana"))
+        .build();
+
+    public static final SystemIndexDescriptor APM_CUSTOM_LINK_INDEX_DESCRIPTOR = SystemIndexDescriptor.builder()
+        .setIndexPattern(".apm-custom-link")
+        .setDescription("system index for APM custom links")
+        .setType(Type.EXTERNAL_UNMANAGED)
+        .setAllowedStackComponents(List.of("kibana"))
+        .build();
 
     @Override
     public Collection<SystemIndexDescriptor> getSystemIndexDescriptors(Settings settings) {
-        return KIBANA_INDEX_NAMES_SETTING.get(settings)
-            .stream()
-            .map(pattern -> new SystemIndexDescriptor(pattern, "System index used by kibana"))
-            .collect(Collectors.toUnmodifiableList());
-    }
-
-    @Override
-    public List<RestHandler> getRestHandlers(
-        Settings settings,
-        RestController restController,
-        ClusterSettings clusterSettings,
-        IndexScopedSettings indexScopedSettings,
-        SettingsFilter settingsFilter,
-        IndexNameExpressionResolver indexNameExpressionResolver,
-        Supplier<DiscoveryNodes> nodesInCluster
-    ) {
-        // TODO need to figure out what subset of system indices Kibana should have access to via these APIs
         return List.of(
-            // Based on https://github.com/elastic/kibana/issues/49764
-            // apis needed to perform migrations... ideally these will go away
-            new KibanaWrappedRestHandler(new RestCreateIndexAction()),
-            new KibanaWrappedRestHandler(new RestGetAliasesAction()),
-            new KibanaWrappedRestHandler(new RestIndexPutAliasAction()),
-            new KibanaWrappedRestHandler(new RestRefreshAction()),
-
-            // apis needed to access saved objects
-            new KibanaWrappedRestHandler(new RestGetAction()),
-            new KibanaWrappedRestHandler(new RestMultiGetAction(settings)),
-            new KibanaWrappedRestHandler(new RestSearchAction()),
-            new KibanaWrappedRestHandler(new RestBulkAction(settings)),
-            new KibanaWrappedRestHandler(new RestDeleteAction()),
-            new KibanaWrappedRestHandler(new RestDeleteByQueryAction()),
-
-            // api used for testing
-            new KibanaWrappedRestHandler(new RestUpdateSettingsAction()),
-
-            // apis used specifically by reporting
-            new KibanaWrappedRestHandler(new RestGetIndicesAction()),
-            new KibanaWrappedRestHandler(new RestIndexAction()),
-            new KibanaWrappedRestHandler(new CreateHandler()),
-            new KibanaWrappedRestHandler(new AutoIdHandler(nodesInCluster)),
-            new KibanaWrappedRestHandler(new RestUpdateAction()),
-            new KibanaWrappedRestHandler(new RestSearchScrollAction()),
-            new KibanaWrappedRestHandler(new RestClearScrollAction())
+            KIBANA_INDEX_DESCRIPTOR,
+            REPORTING_INDEX_DESCRIPTOR,
+            APM_AGENT_CONFIG_INDEX_DESCRIPTOR,
+            APM_CUSTOM_LINK_INDEX_DESCRIPTOR
         );
-
-    }
-
-    @Override
-    public List<Setting<?>> getSettings() {
-        return List.of(KIBANA_INDEX_NAMES_SETTING);
-    }
-
-    static class KibanaWrappedRestHandler extends BaseRestHandler.Wrapper {
-
-        KibanaWrappedRestHandler(BaseRestHandler delegate) {
-            super(delegate);
-        }
-
-        @Override
-        public String getName() {
-            return "kibana_" + super.getName();
-        }
-
-        @Override
-        public boolean allowSystemIndexAccessByDefault() {
-            return true;
-        }
-
-        @Override
-        public List<Route> routes() {
-            return super.routes().stream()
-                .map(route -> new Route(route.getMethod(), "/_kibana" + route.getPath()))
-                .collect(Collectors.toUnmodifiableList());
-        }
     }
 }

modules/kibana/src/test/java/org/elasticsearch/kibana/KibanaPluginTests.java

@@ -12,35 +12,19 @@
 import org.elasticsearch.indices.SystemIndexDescriptor;
 import org.elasticsearch.test.ESTestCase;
 
-import java.util.List;
 import java.util.stream.Collectors;
 
 import static org.hamcrest.Matchers.contains;
-import static org.hamcrest.Matchers.is;
 
 public class KibanaPluginTests extends ESTestCase {
 
     public void testKibanaIndexNames() {
-        assertThat(new KibanaPlugin().getSettings(), contains(KibanaPlugin.KIBANA_INDEX_NAMES_SETTING));
         assertThat(
             new KibanaPlugin().getSystemIndexDescriptors(Settings.EMPTY)
                 .stream()
                 .map(SystemIndexDescriptor::getIndexPattern)
                 .collect(Collectors.toUnmodifiableList()),
-            contains(".kibana", ".kibana_*", ".reporting-*", ".apm-agent-configuration", ".apm-custom-link")
-        );
-
-        final List<String> names = List.of("." + randomAlphaOfLength(4), "." + randomAlphaOfLength(5));
-        final List<String> namesFromDescriptors = new KibanaPlugin().getSystemIndexDescriptors(
-            Settings.builder().putList(KibanaPlugin.KIBANA_INDEX_NAMES_SETTING.getKey(), names).build()
-        ).stream().map(SystemIndexDescriptor::getIndexPattern).collect(Collectors.toUnmodifiableList());
-        assertThat(namesFromDescriptors, is(names));
-
-        assertThat(
-            new KibanaPlugin().getSystemIndexDescriptors(Settings.EMPTY)
-                .stream()
-                .anyMatch(systemIndexDescriptor -> systemIndexDescriptor.matchesIndexPattern(".kibana-event-log-7-1")),
-            is(false)
+            contains(".kibana_*", ".reporting-*", ".apm-agent-configuration", ".apm-custom-link")
         );
     }
 }

modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexSourceTargetValidationTests.java

@@ -24,11 +24,12 @@
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.indices.SystemIndices;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.indices.SystemIndices;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.HashMap;
+import java.util.Map;
 
 import static java.util.Collections.emptyMap;
 import static org.hamcrest.Matchers.containsString;
@@ -52,7 +53,7 @@ public class ReindexSourceTargetValidationTests extends ESTestCase {
                 .put(index("source", "source_multi"), true)
                 .put(index("source2", "source_multi"), true)).build();
     private static final IndexNameExpressionResolver INDEX_NAME_EXPRESSION_RESOLVER =
-        new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY));
+        new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY), new SystemIndices(Map.of()));
     private static final AutoCreateIndex AUTO_CREATE_INDEX = new AutoCreateIndex(Settings.EMPTY,
             new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), INDEX_NAME_EXPRESSION_RESOLVER,
             new SystemIndices(new HashMap<>()));

server/src/internalClusterTest/java/org/elasticsearch/action/IndicesRequestIT.java

@@ -72,6 +72,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.indices.SystemIndices;
 import org.elasticsearch.plugins.NetworkPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.PluginsService;
@@ -381,7 +382,7 @@ public void testFlush() {
         internalCluster().coordOnlyNodeClient().admin().indices().flush(flushRequest).actionGet();
 
         clearInterceptedActions();
-        String[] indices = new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY))
+        String[] indices = new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY), new SystemIndices(Map.of()))
                 .concreteIndexNames(client().admin().cluster().prepareState().get().getState(), flushRequest);
         assertIndicesSubset(Arrays.asList(indices), indexShardActions);
     }
@@ -406,7 +407,7 @@ public void testRefresh() {
         internalCluster().coordOnlyNodeClient().admin().indices().refresh(refreshRequest).actionGet();
 
         clearInterceptedActions();
-        String[] indices = new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY))
+        String[] indices = new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY), new SystemIndices(Map.of()))
                 .concreteIndexNames(client().admin().cluster().prepareState().get().getState(), refreshRequest);
         assertIndicesSubset(Arrays.asList(indices), indexShardActions);
     }

server/src/internalClusterTest/java/org/elasticsearch/indices/TestSystemIndexDescriptor.java

@@ -16,6 +16,7 @@
 
 import java.io.IOException;
 import java.io.UncheckedIOException;
+import java.util.List;
 import java.util.concurrent.atomic.AtomicBoolean;
 
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
@@ -38,7 +39,8 @@ public class TestSystemIndexDescriptor extends SystemIndexDescriptor {
         .build();
 
     TestSystemIndexDescriptor() {
-        super(INDEX_NAME + "*", PRIMARY_INDEX_NAME, "Test system index", null, SETTINGS, INDEX_NAME, 0, "version", "stack", null);
+        super(INDEX_NAME + "*", PRIMARY_INDEX_NAME, "Test system index", null, SETTINGS, INDEX_NAME, 0, "version", "stack", null,
+            Type.INTERNAL, List.of());
     }
 
     @Override

server/src/internalClusterTest/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java

@@ -37,6 +37,7 @@
 import org.elasticsearch.index.seqno.RetentionLeaseActions;
 import org.elasticsearch.index.seqno.RetentionLeases;
 import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.indices.SystemIndices;
 import org.elasticsearch.indices.recovery.PeerRecoveryTargetService;
 import org.elasticsearch.indices.recovery.RecoveryState;
 import org.elasticsearch.node.Node;
@@ -71,6 +72,7 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Locale;
+import java.util.Map;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -585,7 +587,8 @@ public void testRestoreShrinkIndex() throws Exception {
     public void testSnapshotWithDateMath() {
         final String repo = "repo";
 
-        final IndexNameExpressionResolver nameExpressionResolver = new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY));
+        final IndexNameExpressionResolver nameExpressionResolver =
+            new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY), new SystemIndices(Map.of()));
         final String snapshotName = "<snapshot-{now/d}>";
 
         logger.info("-->  creating repository");