Protect newly introduced system indices fully (elastic#74186)

This change updates the way we handle net new system indices, which are
those that have been newly introduced and do not require any BWC
guarantees around non-system access. These indices will not be included
in wildcard expansions for user searches and operations. Direct access
to these indices will also not be allowed for user searches.

The first index of this type is the GeoIp index, which this change sets
the new flag on.

Closes elastic#72572

modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTaskExecutor.java

@@ -37,7 +37,7 @@
  */
 public final class GeoIpDownloaderTaskExecutor extends PersistentTasksExecutor<GeoIpTaskParams> implements ClusterStateListener {
 
-    static final boolean ENABLED_DEFAULT = "true".equals(System.getProperty("ingest.geoip.downloader.enabled.default"));
+    static final boolean ENABLED_DEFAULT = "true".equals(System.getProperty("ingest.geoip.downloader.enabled.default", "true"));
     public static final Setting<Boolean> ENABLED_SETTING = Setting.boolSetting("ingest.geoip.downloader.enabled", ENABLED_DEFAULT,
         Setting.Property.Dynamic, Setting.Property.NodeScope);
 

modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java

@@ -180,6 +180,7 @@ public Collection<SystemIndexDescriptor> getSystemIndexDescriptors(Settings sett
             .setOrigin("geoip")
             .setVersionMetaKey("version")
             .setPrimaryIndex(DATABASES_INDEX)
+            .setNetNew()
             .build();
         return Collections.singleton(geoipDatabasesIndex);
     }

qa/system-indices/build.gradle

@@ -0,0 +1,29 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+apply plugin: 'elasticsearch.internal-es-plugin'
+apply plugin: 'elasticsearch.java-rest-test'
+
+esplugin {
+  name 'system-indices-qa'
+  description 'Plugin for performing QA of system indices'
+  classname 'org.elasticsearch.system.indices.SystemIndicesQA'
+  licenseFile rootProject.file('licenses/SSPL-1.0+ELASTIC-LICENSE-2.0.txt')
+  noticeFile rootProject.file('NOTICE.txt')
+}
+
+tasks.named("test").configure { enabled = false }
+tasks.named("javaRestTest").configure {
+  dependsOn "buildZip"
+}
+
+testClusters.all {
+  testDistribution = 'DEFAULT'
+  setting 'xpack.security.enabled', 'true'
+  user username: 'rest_user', password: 'password', role: 'superuser'
+}

qa/system-indices/src/javaRestTest/java/org/elasticsearch/system/indices/NetNewSystemIndicesIT.java

@@ -0,0 +1,105 @@
+
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.system.indices;
+
+import org.apache.http.util.EntityUtils;
+import org.elasticsearch.Version;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.ResponseException;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.test.rest.ESRestTestCase;
+import org.junit.After;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.not;
+
+public class NetNewSystemIndicesIT extends ESRestTestCase {
+
+    static final String BASIC_AUTH_VALUE = basicAuthHeaderValue("rest_user", new SecureString("password".toCharArray()));
+
+    @Override
+    protected Settings restClientSettings() {
+        return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", BASIC_AUTH_VALUE).build();
+    }
+
+    public void testCreatingSystemIndex() throws Exception {
+        ResponseException e = expectThrows(
+            ResponseException.class,
+            () -> client().performRequest(new Request("PUT", "/.net-new-system-index-" + Version.CURRENT.major))
+        );
+        assertThat(EntityUtils.toString(e.getResponse().getEntity()), containsString("system"));
+
+        Response response = client().performRequest(new Request("PUT", "/_net_new_sys_index/_create"));
+        assertThat(response.getStatusLine().getStatusCode(), is(200));
+    }
+
+    public void testIndexDoc() throws Exception {
+        String id = randomAlphaOfLength(4);
+
+        ResponseException e = expectThrows(ResponseException.class, () -> {
+            Request request = new Request("PUT", "/.net-new-system-index-" + Version.CURRENT.major + "/_doc" + id);
+            request.setJsonEntity("{}");
+            client().performRequest(request);
+        });
+        assertThat(EntityUtils.toString(e.getResponse().getEntity()), containsString("system"));
+
+        Request request = new Request("PUT", "/_net_new_sys_index/" + id);
+        request.setJsonEntity("{}");
+        Response response = client().performRequest(request);
+        assertThat(response.getStatusLine().getStatusCode(), is(200));
+    }
+
+    public void testSearch() throws Exception {
+        // search before indexing doc
+        Request searchRequest = new Request("GET", "/_search");
+        searchRequest.setJsonEntity("{ \"query\": { \"match_all\": {} } }");
+        searchRequest.addParameter("size", "10000");
+        Response searchResponse = client().performRequest(searchRequest);
+        assertThat(searchResponse.getStatusLine().getStatusCode(), is(200));
+        assertThat(EntityUtils.toString(searchResponse.getEntity()), not(containsString(".net-new")));
+
+        // create a doc
+        String id = randomAlphaOfLength(4);
+        Request request = new Request("PUT", "/_net_new_sys_index/" + id);
+        request.setJsonEntity("{}");
+        request.addParameter("refresh", "true");
+        Response response = client().performRequest(request);
+        assertThat(response.getStatusLine().getStatusCode(), is(200));
+
+        // search again
+        searchResponse = client().performRequest(searchRequest);
+        assertThat(searchResponse.getStatusLine().getStatusCode(), is(200));
+        assertThat(EntityUtils.toString(searchResponse.getEntity()), not(containsString(".net-new")));
+
+        // index wildcard search
+        searchRequest = new Request("GET", "/.net-new-system-index*/_search");
+        searchRequest.setJsonEntity("{ \"query\": { \"match_all\": {} } }");
+        searchRequest.addParameter("size", "10000");
+        searchResponse = client().performRequest(searchRequest);
+        assertThat(searchResponse.getStatusLine().getStatusCode(), is(200));
+        assertThat(EntityUtils.toString(searchResponse.getEntity()), not(containsString(".net-new")));
+
+        // direct index search
+        Request directRequest = new Request("GET", "/.net-new-system-index-" + Version.CURRENT.major + "/_search");
+        directRequest.setJsonEntity("{ \"query\": { \"match_all\": {} } }");
+        directRequest.addParameter("size", "10000");
+        ResponseException e = expectThrows(ResponseException.class, () -> client().performRequest(directRequest));
+        assertThat(EntityUtils.toString(e.getResponse().getEntity()), containsString("system"));
+    }
+
+    @After
+    public void resetFeatures() throws Exception {
+        client().performRequest(new Request("POST", "/_features/_reset"));
+    }
+}

qa/system-indices/src/main/java/org/elasticsearch/system/indices/SystemIndicesQA.java

@@ -0,0 +1,164 @@
+
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.system.indices;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.IndexScopedSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.settings.SettingsFilter;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.indices.SystemIndexDescriptor;
+import org.elasticsearch.plugins.ActionPlugin;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.plugins.SystemIndexPlugin;
+import org.elasticsearch.rest.BaseRestHandler;
+import org.elasticsearch.rest.RestController;
+import org.elasticsearch.rest.RestHandler;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.RestRequest.Method;
+import org.elasticsearch.rest.action.RestToXContentListener;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Collection;
+import java.util.List;
+import java.util.function.Supplier;
+
+import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+import static org.elasticsearch.index.mapper.MapperService.SINGLE_MAPPING_NAME;
+import static org.elasticsearch.rest.RestRequest.Method.POST;
+import static org.elasticsearch.rest.RestRequest.Method.PUT;
+
+public class SystemIndicesQA extends Plugin implements SystemIndexPlugin, ActionPlugin {
+
+    @Override
+    public String getFeatureName() {
+        return "system indices qa";
+    }
+
+    @Override
+    public String getFeatureDescription() {
+        return "plugin used to perform qa on system index behavior";
+    }
+
+    @Override
+    public Collection<SystemIndexDescriptor> getSystemIndexDescriptors(Settings settings) {
+        return List.of(
+            SystemIndexDescriptor.builder()
+                .setNetNew()
+                .setIndexPattern(".net-new-system-index*")
+                .setDescription("net new system index")
+                .setMappings(mappings())
+                .setSettings(
+                    Settings.builder()
+                        .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+                        .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+                        .put(IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS, "0-1")
+                        .build()
+                )
+                .setOrigin("net-new")
+                .setVersionMetaKey("version")
+                .setPrimaryIndex(".net-new-system-index-" + Version.CURRENT.major)
+                .build()
+        );
+    }
+
+    private static XContentBuilder mappings() {
+        try {
+            return jsonBuilder().startObject()
+                .startObject(SINGLE_MAPPING_NAME)
+                .startObject("_meta")
+                .field("version", Version.CURRENT)
+                .endObject()
+                .field("dynamic", "strict")
+                .startObject("properties")
+                .startObject("name")
+                .field("type", "keyword")
+                .endObject()
+                .endObject()
+                .endObject()
+                .endObject();
+        } catch (IOException e) {
+            throw new UncheckedIOException("Failed to build mappings for net new system index", e);
+        }
+    }
+
+    @Override
+    public List<RestHandler> getRestHandlers(
+        Settings settings,
+        RestController restController,
+        ClusterSettings clusterSettings,
+        IndexScopedSettings indexScopedSettings,
+        SettingsFilter settingsFilter,
+        IndexNameExpressionResolver indexNameExpressionResolver,
+        Supplier<DiscoveryNodes> nodesInCluster
+    ) {
+        return List.of(new CreateNetNewSystemIndexHandler(), new IndexDocHandler());
+    }
+
+    private static class CreateNetNewSystemIndexHandler extends BaseRestHandler {
+
+        @Override
+        public String getName() {
+            return "create net new system index for qa";
+        }
+
+        @Override
+        public List<Route> routes() {
+            return List.of(Route.builder(Method.PUT, "/_net_new_sys_index/_create").build());
+        }
+
+        @Override
+        protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
+            return channel -> client.admin()
+                .indices()
+                .create(new CreateIndexRequest(".net-new-system-index-" + Version.CURRENT.major), new RestToXContentListener<>(channel));
+        }
+
+        @Override
+        public boolean allowSystemIndexAccessByDefault() {
+            return true;
+        }
+    }
+
+    private static class IndexDocHandler extends BaseRestHandler {
+        @Override
+        public String getName() {
+            return "index doc into net new for qa";
+        }
+
+        @Override
+        public List<Route> routes() {
+            return List.of(new Route(POST, "/_net_new_sys_index/{id}"), new Route(PUT, "/_net_new_sys_index/{id}"));
+        }
+
+        @Override
+        protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
+            IndexRequest indexRequest = new IndexRequest(".net-new-system-index-" + Version.CURRENT.major);
+            indexRequest.source(request.requiredContent(), request.getXContentType());
+            indexRequest.id(request.param("id"));
+            indexRequest.setRefreshPolicy(request.param("refresh"));
+
+            return channel -> client.index(indexRequest, new RestToXContentListener<>(channel));
+        }
+
+        @Override
+        public boolean allowSystemIndexAccessByDefault() {
+            return true;
+        }
+    }
+}

server/src/internalClusterTest/java/org/elasticsearch/indices/TestSystemIndexDescriptor.java

@@ -40,7 +40,7 @@ public class TestSystemIndexDescriptor extends SystemIndexDescriptor {
 
     TestSystemIndexDescriptor() {
         super(INDEX_NAME + "*", PRIMARY_INDEX_NAME, "Test system index", getOldMappings(), SETTINGS, INDEX_NAME, 0, "version", "stack",
-            Version.CURRENT.minimumCompatibilityVersion(), Type.INTERNAL_MANAGED, List.of(), List.of(), null);
+            Version.CURRENT.minimumCompatibilityVersion(), Type.INTERNAL_MANAGED, List.of(), List.of(), null, false);
     }
 
     @Override