[8.x] Add lookup index mode (#115143) (#115596)

* Add lookup index mode (#115143)

This change introduces a new index mode, lookup, for indices intended 
for lookup operations in ES|QL. Lookup indices must have a single shard
and be replicated to all data nodes by default. Aside from these
requirements, they function as standard indices. Documentation will be
added later when the lookup operator in ES|QL is implemented.

* default shard

* minimal

* compile

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/10_basic.yml

@@ -149,3 +149,70 @@
       indices.exists_alias:
         name: logs_2022-12-31
   - is_true: ''
+
+---
+"Create lookup index":
+  - requires:
+      test_runner_features: [ capabilities, default_shards ]
+      capabilities:
+        - method: PUT
+          path: /{index}
+          capabilities: [ lookup_index_mode ]
+      reason: "Support for 'lookup' index mode capability required"
+  - do:
+      indices.create:
+        index: "test_lookup"
+        body:
+          settings:
+            index.mode: lookup
+
+  - do:
+      indices.get_settings:
+        index: test_lookup
+
+  - match: { test_lookup.settings.index.number_of_shards: "1"}
+  - match: { test_lookup.settings.index.auto_expand_replicas: "0-all"}
+
+---
+"Create lookup index with one shard":
+  - requires:
+      test_runner_features: [ capabilities, default_shards ]
+      capabilities:
+        - method: PUT
+          path: /{index}
+          capabilities: [ lookup_index_mode ]
+      reason: "Support for 'lookup' index mode capability required"
+  - do:
+      indices.create:
+        index: "test_lookup"
+        body:
+          settings:
+            index:
+              mode: lookup
+              number_of_shards: 1
+
+  - do:
+      indices.get_settings:
+        index: test_lookup
+
+  - match: { test_lookup.settings.index.number_of_shards: "1"}
+  - match: { test_lookup.settings.index.auto_expand_replicas: "0-all"}
+
+---
+"Create lookup index with two shards":
+  - requires:
+      test_runner_features: [ capabilities ]
+      capabilities:
+        - method: PUT
+          path: /{index}
+          capabilities: [ lookup_index_mode ]
+      reason: "Support for 'lookup' index mode capability required"
+  - do:
+      catch: /illegal_argument_exception/
+      indices.create:
+        index: test_lookup
+        body:
+          settings:
+            index.mode: lookup
+            index.number_of_shards: 2
+

server/src/internalClusterTest/java/org/elasticsearch/index/LookupIndexModeIT.java

@@ -0,0 +1,219 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.index;
+
+import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
+import org.elasticsearch.action.admin.indices.create.TransportCreateIndexAction;
+import org.elasticsearch.action.admin.indices.shrink.ResizeAction;
+import org.elasticsearch.action.admin.indices.shrink.ResizeRequest;
+import org.elasticsearch.action.admin.indices.shrink.ResizeType;
+import org.elasticsearch.action.fieldcaps.FieldCapabilitiesIndexResponse;
+import org.elasticsearch.action.fieldcaps.FieldCapabilitiesRequest;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.query.MatchQueryBuilder;
+import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.test.ESIntegTestCase;
+
+import java.util.Map;
+
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasSize;
+
+public class LookupIndexModeIT extends ESIntegTestCase {
+
+    @Override
+    protected int numberOfShards() {
+        return 1;
+    }
+
+    public void testBasic() {
+        internalCluster().ensureAtLeastNumDataNodes(1);
+        Settings.Builder lookupSettings = Settings.builder().put("index.mode", "lookup");
+        if (randomBoolean()) {
+            lookupSettings.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1);
+        }
+        CreateIndexRequest createRequest = new CreateIndexRequest("hosts");
+        createRequest.settings(lookupSettings);
+        createRequest.simpleMapping("ip", "type=ip", "os", "type=keyword");
+        assertAcked(client().admin().indices().execute(TransportCreateIndexAction.TYPE, createRequest));
+        Settings settings = client().admin().indices().prepareGetSettings("hosts").get().getIndexToSettings().get("hosts");
+        assertThat(settings.get("index.mode"), equalTo("lookup"));
+        assertThat(settings.get("index.auto_expand_replicas"), equalTo("0-all"));
+        Map<String, String> allHosts = Map.of(
+            "192.168.1.2",
+            "Windows",
+            "192.168.1.3",
+            "MacOS",
+            "192.168.1.4",
+            "Linux",
+            "192.168.1.5",
+            "Android",
+            "192.168.1.6",
+            "iOS",
+            "192.168.1.7",
+            "Windows",
+            "192.168.1.8",
+            "MacOS",
+            "192.168.1.9",
+            "Linux",
+            "192.168.1.10",
+            "Linux",
+            "192.168.1.11",
+            "Windows"
+        );
+        for (Map.Entry<String, String> e : allHosts.entrySet()) {
+            client().prepareIndex("hosts").setSource("ip", e.getKey(), "os", e.getValue()).get();
+        }
+        refresh("hosts");
+        assertAcked(client().admin().indices().prepareCreate("events").setSettings(Settings.builder().put("index.mode", "logsdb")).get());
+        int numDocs = between(1, 10);
+        for (int i = 0; i < numDocs; i++) {
+            String ip = randomFrom(allHosts.keySet());
+            String message = randomFrom("login", "logout", "shutdown", "restart");
+            client().prepareIndex("events").setSource("@timestamp", "2024-01-01", "ip", ip, "message", message).get();
+        }
+        refresh("events");
+        // _search
+        {
+            SearchResponse resp = prepareSearch("events", "hosts").setQuery(new MatchQueryBuilder("_index_mode", "lookup"))
+                .setSize(10000)
+                .get();
+            for (SearchHit hit : resp.getHits()) {
+                assertThat(hit.getIndex(), equalTo("hosts"));
+            }
+            assertHitCount(resp, allHosts.size());
+            resp.decRef();
+        }
+        // field_caps
+        {
+            FieldCapabilitiesRequest request = new FieldCapabilitiesRequest();
+            request.indices("events", "hosts");
+            request.fields("*");
+            request.setMergeResults(false);
+            request.indexFilter(new MatchQueryBuilder("_index_mode", "lookup"));
+            var resp = client().fieldCaps(request).actionGet();
+            assertThat(resp.getIndexResponses(), hasSize(1));
+            FieldCapabilitiesIndexResponse indexResponse = resp.getIndexResponses().get(0);
+            assertThat(indexResponse.getIndexMode(), equalTo(IndexMode.LOOKUP));
+            assertThat(indexResponse.getIndexName(), equalTo("hosts"));
+        }
+    }
+
+    public void testRejectMoreThanOneShard() {
+        int numberOfShards = between(2, 5);
+        IllegalArgumentException error = expectThrows(IllegalArgumentException.class, () -> {
+            client().admin()
+                .indices()
+                .prepareCreate("hosts")
+                .setSettings(Settings.builder().put("index.mode", "lookup").put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numberOfShards))
+                .setMapping("ip", "type=ip", "os", "type=keyword")
+                .get();
+        });
+        assertThat(
+            error.getMessage(),
+            equalTo("index with [lookup] mode must have [index.number_of_shards] set to 1 or unset; provided " + numberOfShards)
+        );
+    }
+
+    public void testResizeLookupIndex() {
+        Settings.Builder createSettings = Settings.builder().put("index.mode", "lookup");
+        if (randomBoolean()) {
+            createSettings.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1);
+        }
+        CreateIndexRequest createIndexRequest = new CreateIndexRequest("lookup-1").settings(createSettings);
+        assertAcked(client().admin().indices().execute(TransportCreateIndexAction.TYPE, createIndexRequest));
+        client().admin().indices().prepareAddBlock(IndexMetadata.APIBlock.WRITE, "lookup-1").get();
+
+        ResizeRequest clone = new ResizeRequest("lookup-2", "lookup-1");
+        clone.setResizeType(ResizeType.CLONE);
+        assertAcked(client().admin().indices().execute(ResizeAction.INSTANCE, clone).actionGet());
+        Settings settings = client().admin().indices().prepareGetSettings("lookup-2").get().getIndexToSettings().get("lookup-2");
+        assertThat(settings.get("index.mode"), equalTo("lookup"));
+        assertThat(settings.get("index.number_of_shards"), equalTo("1"));
+        assertThat(settings.get("index.auto_expand_replicas"), equalTo("0-all"));
+
+        ResizeRequest split = new ResizeRequest("lookup-3", "lookup-1");
+        split.setResizeType(ResizeType.SPLIT);
+        split.getTargetIndexRequest().settings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 3));
+        IllegalArgumentException error = expectThrows(
+            IllegalArgumentException.class,
+            () -> client().admin().indices().execute(ResizeAction.INSTANCE, split).actionGet()
+        );
+        assertThat(
+            error.getMessage(),
+            equalTo("index with [lookup] mode must have [index.number_of_shards] set to 1 or unset; provided 3")
+        );
+    }
+
+    public void testResizeRegularIndexToLookup() {
+        String dataNode = internalCluster().startDataOnlyNode();
+        assertAcked(
+            client().admin()
+                .indices()
+                .prepareCreate("regular-1")
+                .setSettings(
+                    Settings.builder()
+                        .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 2)
+                        .put("index.routing.allocation.require._name", dataNode)
+                )
+                .setMapping("ip", "type=ip", "os", "type=keyword")
+                .get()
+        );
+        client().admin().indices().prepareAddBlock(IndexMetadata.APIBlock.WRITE, "regular-1").get();
+        client().admin()
+            .indices()
+            .prepareUpdateSettings("regular-1")
+            .setSettings(Settings.builder().put("index.number_of_replicas", 0))
+            .get();
+
+        ResizeRequest clone = new ResizeRequest("lookup-3", "regular-1");
+        clone.setResizeType(ResizeType.CLONE);
+        clone.getTargetIndexRequest().settings(Settings.builder().put("index.mode", "lookup"));
+        IllegalArgumentException error = expectThrows(
+            IllegalArgumentException.class,
+            () -> client().admin().indices().execute(ResizeAction.INSTANCE, clone).actionGet()
+        );
+        assertThat(
+            error.getMessage(),
+            equalTo("index with [lookup] mode must have [index.number_of_shards] set to 1 or unset; provided 2")
+        );
+
+        ResizeRequest shrink = new ResizeRequest("lookup-4", "regular-1");
+        shrink.setResizeType(ResizeType.SHRINK);
+        shrink.getTargetIndexRequest()
+            .settings(Settings.builder().put("index.mode", "lookup").put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1));
+
+        error = expectThrows(
+            IllegalArgumentException.class,
+            () -> client().admin().indices().execute(ResizeAction.INSTANCE, shrink).actionGet()
+        );
+        assertThat(error.getMessage(), equalTo("can't change index.mode of index [regular-1] from [standard] to [lookup]"));
+    }
+
+    public void testDoNotOverrideAutoExpandReplicas() {
+        internalCluster().ensureAtLeastNumDataNodes(1);
+        Settings.Builder createSettings = Settings.builder().put("index.mode", "lookup");
+        if (randomBoolean()) {
+            createSettings.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1);
+        }
+        createSettings.put("index.auto_expand_replicas", "3-5");
+        CreateIndexRequest createRequest = new CreateIndexRequest("hosts");
+        createRequest.settings(createSettings);
+        createRequest.simpleMapping("ip", "type=ip", "os", "type=keyword");
+        assertAcked(client().admin().indices().execute(TransportCreateIndexAction.TYPE, createRequest));
+        Settings settings = client().admin().indices().prepareGetSettings("hosts").get().getIndexToSettings().get("hosts");
+        assertThat(settings.get("index.mode"), equalTo("lookup"));
+        assertThat(settings.get("index.auto_expand_replicas"), equalTo("3-5"));
+    }
+}

server/src/main/java/org/elasticsearch/TransportVersions.java

@@ -181,6 +181,7 @@ static TransportVersion def(int id) {
     public static final TransportVersion INFERENCE_DONT_PERSIST_ON_READ = def(8_776_00_0);
     public static final TransportVersion SIMULATE_MAPPING_ADDITION = def(8_777_00_0);
     public static final TransportVersion INTRODUCE_ALL_APPLICABLE_SELECTOR = def(8_778_00_0);
+    public static final TransportVersion INDEX_MODE_LOOKUP = def(8_779_00_0);
 
     /*
      * STOP! READ THIS FIRST! No, really,

server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java

@@ -308,7 +308,12 @@ private void onlyCreateIndex(
         final CreateIndexClusterStateUpdateRequest request,
         final ActionListener<AcknowledgedResponse> listener
     ) {
-        normalizeRequestSetting(request);
+        try {
+            normalizeRequestSetting(request);
+        } catch (Exception e) {
+            listener.onFailure(e);
+            return;
+        }
 
         var delegate = new AllocationActionListener<>(listener, threadPool.getThreadContext());
         submitUnbatchedTask(
@@ -1599,6 +1604,15 @@ static IndexMetadata validateResize(
             // of if the source shards are divisible by the number of target shards
             IndexMetadata.getRoutingFactor(sourceMetadata.getNumberOfShards(), INDEX_NUMBER_OF_SHARDS_SETTING.get(targetIndexSettings));
         }
+        if (targetIndexSettings.hasValue(IndexSettings.MODE.getKey())) {
+            IndexMode oldMode = Objects.requireNonNullElse(sourceMetadata.getIndexMode(), IndexMode.STANDARD);
+            IndexMode newMode = IndexSettings.MODE.get(targetIndexSettings);
+            if (newMode != oldMode) {
+                throw new IllegalArgumentException(
+                    "can't change index.mode of index [" + sourceIndex + "] from [" + oldMode + "] to [" + newMode + "]"
+                );
+            }
+        }
         return sourceMetadata;
     }