Add ability to retrieve document metadata via fields option

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/330_fetch_fields.yml

@@ -1007,3 +1007,60 @@ error for flattened includes whole path:
           fields:
             - field: flattened.bar
               format: "yyyy/MM/dd"
+
+---
+test fetching metadata fields:
+  - skip:
+      version: ' - 7.15.99'
+      reason:  'fetching metadata via fields was introduced in 7.16'
+
+  - do:
+      indices.create:
+        index:  test
+        body:
+          settings:
+            index.number_of_shards: 1
+          mappings:
+            properties:
+              field:
+                type: keyword
+              idAlias:
+                type: alias
+                path: _id
+
+  - do:
+      index:
+        index:   test
+        id:      1
+        refresh: true
+        body:
+          field: foo
+
+  - do:
+      search:
+        index: test
+        body:
+          fields: [ "*" ]
+
+  - length: { hits.hits.0.fields : 2 }
+  - match: { hits.hits.0.fields.field.0: "foo" }
+  - match: { hits.hits.0.fields.idAlias.0: "1" }
+
+  - do:
+      search:
+        index: test
+        body:
+          fields: [ "_*" ]
+
+  - is_false: hits.hits.0.fields
+
+  - do:
+      search:
+        index: test
+        body:
+          fields: [ "_id", "_index", "_version" ]
+
+  - length: { hits.hits.0.fields : 3 }
+  - match: { hits.hits.0.fields._id.0: "1" }
+  - match: { hits.hits.0.fields._index.0: "test" }
+  - match: { hits.hits.0.fields._version.0: 1 }

server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java

@@ -109,7 +109,7 @@ public boolean isSearchable() {
 
         @Override
         public ValueFetcher valueFetcher(SearchExecutionContext context, String format) {
-            throw new UnsupportedOperationException("Cannot fetch values for internal field [" + name() + "].");
+            return new StoredValueFetcher(context.lookup(), NAME);
         }
 
         @Override

server/src/main/java/org/elasticsearch/index/mapper/IgnoredFieldMapper.java

@@ -68,7 +68,7 @@ public Query existsQuery(SearchExecutionContext context) {
 
         @Override
         public ValueFetcher valueFetcher(SearchExecutionContext context, String format) {
-            throw new UnsupportedOperationException("Cannot fetch values for internal field [" + name() + "].");
+            return new StoredValueFetcher(context.lookup(), NAME);
         }
     }
 

server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java

@@ -16,8 +16,11 @@
 import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
 import org.elasticsearch.search.lookup.SearchLookup;
+import org.elasticsearch.search.lookup.SourceLookup;
 
+import java.io.IOException;
 import java.util.Collections;
+import java.util.List;
 import java.util.function.Supplier;
 
 public class IndexFieldMapper extends MetadataFieldMapper {
@@ -65,7 +68,15 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, S
 
         @Override
         public ValueFetcher valueFetcher(SearchExecutionContext context, String format) {
-            throw new UnsupportedOperationException("Cannot fetch values for internal field [" + name() + "].");
+            return new ValueFetcher() {
+
+                private final List<Object> indexName = Collections.singletonList(context.getFullyQualifiedIndex().getName());
+
+                @Override
+                public List<Object> fetchValues(SourceLookup lookup, List<Object> ignoredValues) throws IOException {
+                    return indexName;
+                }
+            };
         }
     }
 

server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java

@@ -84,7 +84,7 @@ public String typeName() {
 
         @Override
         public ValueFetcher valueFetcher(SearchExecutionContext context, String format) {
-            throw new UnsupportedOperationException("Cannot fetch values for internal field [" + name() + "].");
+            return new StoredValueFetcher(context.lookup(), NAME);
         }
     }
 

server/src/main/java/org/elasticsearch/index/mapper/StoredValueFetcher.java

@@ -0,0 +1,44 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.index.mapper;
+
+import org.apache.lucene.index.LeafReaderContext;
+import org.elasticsearch.search.lookup.LeafSearchLookup;
+import org.elasticsearch.search.lookup.SearchLookup;
+import org.elasticsearch.search.lookup.SourceLookup;
+
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * Value fetcher that loads from stored values.
+ */
+public final class StoredValueFetcher implements ValueFetcher {
+
+    private final SearchLookup lookup;
+    private LeafSearchLookup leafSearchLookup;
+    private final String fieldname;
+
+    public StoredValueFetcher(SearchLookup lookup, String fieldname) {
+        this.lookup = lookup;
+        this.fieldname = fieldname;
+    }
+
+    @Override
+    public void setNextReader(LeafReaderContext context) {
+        this.leafSearchLookup = lookup.getLeafSearchLookup(context);
+    }
+
+    @Override
+    public List<Object> fetchValues(SourceLookup lookup, List<Object> ignoredValues) throws IOException {
+        leafSearchLookup.setDocument(lookup.docId());
+        return leafSearchLookup.fields().get(fieldname).getValues();
+    }
+
+}

server/src/main/java/org/elasticsearch/index/mapper/VersionFieldMapper.java

@@ -11,10 +11,15 @@
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.NumericDocValuesField;
 import org.apache.lucene.search.Query;
-import org.elasticsearch.index.query.SearchExecutionContext;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.IndexNumericFieldData.NumericType;
+import org.elasticsearch.index.fielddata.plain.SortedNumericIndexFieldData;
 import org.elasticsearch.index.query.QueryShardException;
+import org.elasticsearch.index.query.SearchExecutionContext;
+import org.elasticsearch.search.lookup.SearchLookup;
 
 import java.util.Collections;
+import java.util.function.Supplier;
 
 /** Mapper for the _version field. */
 public class VersionFieldMapper extends MetadataFieldMapper {
@@ -46,7 +51,13 @@ public Query termQuery(Object value, SearchExecutionContext context) {
 
         @Override
         public ValueFetcher valueFetcher(SearchExecutionContext context, String format) {
-            throw new UnsupportedOperationException("Cannot fetch values for internal field [" + name() + "].");
+            return new DocValueFetcher(docValueFormat(format, null), context.getForField(this));
+        }
+
+        @Override
+        public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier<SearchLookup> searchLookup) {
+            failIfNoDocValues();
+            return new SortedNumericIndexFieldData.Builder(name(), NumericType.LONG);
         }
     }
 

server/src/test/java/org/elasticsearch/index/mapper/IdFieldMapperTests.java

@@ -10,12 +10,20 @@
 
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.search.IndexSearcher;
+import org.elasticsearch.core.List;
+import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.search.lookup.SearchLookup;
 
 import java.io.IOException;
+import java.util.Collections;
 
 import static org.elasticsearch.index.mapper.IdFieldMapper.ID_FIELD_DATA_DEPRECATION_MESSAGE;
 import static org.hamcrest.Matchers.containsString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
 
 public class IdFieldMapperTests extends MapperServiceTestCase {
 
@@ -60,4 +68,25 @@ public void testEnableFieldData() throws IOException {
         assertWarnings(ID_FIELD_DATA_DEPRECATION_MESSAGE);
         assertTrue(ft.isAggregatable());
     }
+
+    public void testFetchIdFieldValue() throws IOException {
+        MapperService mapperService = createMapperService(
+            fieldMapping(b -> b.field("type", "keyword"))
+        );
+        String id = randomAlphaOfLength(12);
+        withLuceneIndex(mapperService, iw -> {
+            iw.addDocument(mapperService.documentMapper().parse(source(id, b -> b.field("field", "value"), null)).rootDoc());
+        }, iw -> {
+            SearchLookup lookup = new SearchLookup(mapperService::fieldType, fieldDataLookup());
+            SearchExecutionContext searchExecutionContext = mock(SearchExecutionContext.class);
+            when(searchExecutionContext.lookup()).thenReturn(lookup);
+            IdFieldMapper.IdFieldType ft = (IdFieldMapper.IdFieldType) mapperService.fieldType("_id");
+            ValueFetcher valueFetcher = ft.valueFetcher(searchExecutionContext, null);
+            IndexSearcher searcher = newSearcher(iw);
+            LeafReaderContext context = searcher.getIndexReader().leaves().get(0);
+            lookup.source().setSegmentAndDocument(context, 0);
+            valueFetcher.setNextReader(context);
+            assertEquals(Collections.singletonList(id), valueFetcher.fetchValues(lookup.source(), List.of()));
+        });
+    }
 }

server/src/test/java/org/elasticsearch/index/mapper/IgnoredFieldMapperTests.java

@@ -0,0 +1,68 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.index.mapper;
+
+import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.search.IndexSearcher;
+import org.elasticsearch.core.List;
+import org.elasticsearch.index.query.SearchExecutionContext;
+import org.elasticsearch.search.lookup.SearchLookup;
+
+import java.io.IOException;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class IgnoredFieldMapperTests extends MapperServiceTestCase {
+
+    public void testIncludeInObjectNotAllowed() throws Exception {
+        DocumentMapper docMapper = createDocumentMapper(mapping(b -> {}));
+
+        Exception e = expectThrows(MapperParsingException.class,
+            () -> docMapper.parse(source(b -> b.field("_ignored", 1))));
+
+        assertThat(e.getCause().getMessage(),
+            containsString("Field [_ignored] is a metadata field and cannot be added inside a document"));
+    }
+
+    public void testDefaults() throws IOException {
+        DocumentMapper mapper = createDocumentMapper(
+            mapping(b -> b.startObject("field").field("type", "keyword").field("ignore_above", 3).endObject())
+        );
+        ParsedDocument document = mapper.parse(source(b -> b.field("field", "value")));
+        IndexableField[] fields = document.rootDoc().getFields(IgnoredFieldMapper.NAME);
+        assertEquals(1, fields.length);
+        assertEquals(IndexOptions.DOCS, fields[0].fieldType().indexOptions());
+        assertTrue(fields[0].fieldType().stored());
+    }
+
+    public void testFetchIgnoredFieldValue() throws IOException {
+        MapperService mapperService = createMapperService(
+            fieldMapping(b -> b.field("type", "keyword").field("ignore_above", 3))
+        );
+        withLuceneIndex(mapperService, iw -> {
+            iw.addDocument(mapperService.documentMapper().parse(source(b -> b.field("field", "value"))).rootDoc());
+        }, iw -> {
+            SearchLookup lookup = new SearchLookup(mapperService::fieldType, fieldDataLookup());
+            SearchExecutionContext searchExecutionContext = mock(SearchExecutionContext.class);
+            when(searchExecutionContext.lookup()).thenReturn(lookup);
+            IgnoredFieldMapper.IgnoredFieldType ft = (IgnoredFieldMapper.IgnoredFieldType) mapperService.fieldType("_ignored");
+            ValueFetcher valueFetcher = ft.valueFetcher(searchExecutionContext, null);
+            IndexSearcher searcher = newSearcher(iw);
+            LeafReaderContext context = searcher.getIndexReader().leaves().get(0);
+            lookup.source().setSegmentAndDocument(context, 0);
+            valueFetcher.setNextReader(context);
+            assertEquals(List.of("field"), valueFetcher.fetchValues(lookup.source(), List.of()));
+        });
+    }
+
+}

server/src/test/java/org/elasticsearch/index/mapper/IndexFieldMapperTests.java

@@ -8,9 +8,22 @@
 
 package org.elasticsearch.index.mapper;
 
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.search.IndexSearcher;
+import org.elasticsearch.core.List;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.fielddata.IndexFieldDataCache;
+import org.elasticsearch.index.query.SearchExecutionContext;
+import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
+import org.elasticsearch.search.lookup.SearchLookup;
+
+import java.io.IOException;
+import java.util.Collections;
+
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.nullValue;
+import static org.mockito.Mockito.when;
 
 public class IndexFieldMapperTests extends MapperServiceTestCase {
 
@@ -27,4 +40,29 @@ public void testIndexNotConfigurable() {
         assertThat(e.getMessage(), containsString("_index is not configurable"));
     }
 
+    public void testFetchFieldValue() throws IOException {
+        MapperService mapperService = createMapperService(
+            fieldMapping(b -> b.field("type", "keyword"))
+        );
+        String index = randomAlphaOfLength(12);
+        withLuceneIndex(mapperService, iw -> {
+            SourceToParse source = source(index, "id", b -> b.field("field", "value"), "", org.elasticsearch.core.Map.of());
+            iw.addDocument(mapperService.documentMapper().parse(source).rootDoc());
+        }, iw -> {
+            IndexFieldMapper.IndexFieldType ft = (IndexFieldMapper.IndexFieldType) mapperService.fieldType("_index");
+            SearchLookup lookup = new SearchLookup(mapperService::fieldType, fieldDataLookup());
+            SearchExecutionContext searchExecutionContext = createSearchExecutionContext(mapperService);
+            when(searchExecutionContext.getForField(ft)).thenReturn(
+                ft.fielddataBuilder(index, () -> lookup).build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService())
+            );
+            when(searchExecutionContext.getFullyQualifiedIndex()).thenReturn(new Index(index, "indexUUid"));
+            ValueFetcher valueFetcher = ft.valueFetcher(searchExecutionContext, null);
+            IndexSearcher searcher = newSearcher(iw);
+            LeafReaderContext context = searcher.getIndexReader().leaves().get(0);
+            lookup.source().setSegmentAndDocument(context, 0);
+            valueFetcher.setNextReader(context);
+            assertEquals(List.of(index), valueFetcher.fetchValues(lookup.source(), Collections.emptyList()));
+        });
+    }
+
 }