From 1533428ffbadc82547809c3607667d0ae3085340 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Wed, 8 Jul 2020 14:27:15 -0400 Subject: [PATCH 01/16] Teach _search to read search time mappings This adds support for search time mappings to the _search API like so: ``` curl -XPOST -HContent-Type:application/json -uelastic:password 'localhost:9200/test/_search?pretty&error_trace' -d'{ "mappings": { "properties": { "cat": { "type": "script", "runtime_type": "keyword", "script": "value(\"cat\")" } } }, "docvalue_fields": ["cat"] }' ``` It just lifts the `mappings` field from the create index and mappings update requests and drops it into the `_search` request. It then gives each mapped field a chance to convert itself into a runtime field. Most mapping can not and will throw an exception that is translated into an HTTP 400 response. The `script` converts itself properly and is available at runtime. --- .../org/elasticsearch/index/IndexService.java | 13 +++- .../index/mapper/DocumentMapper.java | 11 ++-- .../index/mapper/DocumentMapperParser.java | 10 ++- .../elasticsearch/index/mapper/Mapper.java | 8 +++ .../index/mapper/MapperService.java | 32 ++++++++++ .../index/query/QueryShardContext.java | 40 +++++++++--- .../search/DefaultSearchContext.java | 5 +- .../search/builder/SearchSourceBuilder.java | 36 ++++++++++- .../fetch/subphase/FetchDocValuesPhase.java | 2 +- .../MetadataCreateIndexServiceTests.java | 2 +- .../index/mapper/DateFieldTypeTests.java | 6 +- .../mapper/FieldNamesFieldTypeTests.java | 2 +- .../index/mapper/IndexFieldTypeTests.java | 2 +- .../index/mapper/MapperServiceTests.java | 62 ++++++++++++++++++- .../index/mapper/NumberFieldTypeTests.java | 2 +- .../index/mapper/RangeFieldTypeTests.java | 2 +- .../query/IntervalQueryBuilderTests.java | 2 +- .../index/query/QueryShardContextTests.java | 4 +- .../index/query/RangeQueryRewriteTests.java | 6 +- .../bucket/histogram/ExtendedBoundsTests.java | 2 +- .../ScriptedMetricAggregatorTests.java | 3 +- .../highlight/HighlightBuilderTests.java | 2 +- .../rescore/QueryRescorerBuilderTests.java | 4 +- .../search/sort/AbstractSortTestCase.java | 2 +- .../AbstractSuggestionBuilderTestCase.java | 2 +- .../aggregations/AggregatorTestCase.java | 2 +- .../test/AbstractBuilderTestCase.java | 2 +- .../search/MockSearchServiceTests.java | 2 +- .../DocumentSubsetBitsetCacheTests.java | 2 +- ...ityIndexReaderWrapperIntegrationTests.java | 4 +- .../job/RollupIndexerIndexingTests.java | 2 +- .../mapper/ScriptFieldMapper.java | 5 ++ .../mapper/WildcardFieldMapperTests.java | 2 +- 33 files changed, 233 insertions(+), 50 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/IndexService.java b/server/src/main/java/org/elasticsearch/index/IndexService.java index 5adf9d4dedabb..59e74c53dd94a 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexService.java +++ b/server/src/main/java/org/elasticsearch/index/IndexService.java @@ -575,14 +575,25 @@ public IndexSettings getIndexSettings() { * {@link IndexReader}-specific optimizations, such as rewriting containing range queries. */ public QueryShardContext newQueryShardContext(int shardId, IndexSearcher searcher, LongSupplier nowInMillis, String clusterAlias) { + return newQueryShardContext(shardId, searcher, nowInMillis, clusterAlias, null); + } + + public QueryShardContext newQueryShardContext( + int shardId, + IndexSearcher searcher, + LongSupplier nowInMillis, + String clusterAlias, + Map extraMapping + ) { final SearchIndexNameMatcher indexNameMatcher = new SearchIndexNameMatcher(index().getName(), clusterAlias, clusterService, expressionResolver); return new QueryShardContext( shardId, indexSettings, bigArrays, indexCache.bitsetFilterCache(), indexFieldData::getForField, mapperService(), similarityService(), scriptService, xContentRegistry, namedWriteableRegistry, client, searcher, nowInMillis, clusterAlias, - indexNameMatcher, allowExpensiveQueries, valuesSourceRegistry); + indexNameMatcher, allowExpensiveQueries, valuesSourceRegistry, extraMapping); } + /** * The {@link ThreadPool} to use for this index. */ diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java index a3ec5a933e258..8555e0acc4329 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java @@ -103,14 +103,17 @@ public Builder put(MetadataFieldMapper.Builder mapper) { return this; } - public DocumentMapper build(MapperService mapperService) { + public Mapping buildMapping(Version indexVersionCreated) { Objects.requireNonNull(rootObjectMapper, "Mapper builder must have the root object mapper set"); - Mapping mapping = new Mapping( - mapperService.getIndexSettings().getIndexVersionCreated(), + return new Mapping( + indexVersionCreated, rootObjectMapper, metadataMappers.values().toArray(new MetadataFieldMapper[metadataMappers.values().size()]), meta); - return new DocumentMapper(mapperService, mapping); + } + + public DocumentMapper build(MapperService mapperService) { + return new DocumentMapper(mapperService, buildMapping(mapperService.getIndexSettings().getIndexVersionCreated())); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java index d4f16bcaf8458..aa0cb3617babe 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java @@ -80,11 +80,15 @@ public DocumentMapper parse(@Nullable String type, CompressedXContent source) th if (mapping == null) { mapping = new HashMap<>(); } - return parse(type, mapping); + return parse(type, mapping).build(mapperService); + } + + public Mapping parseMapping(String type, Map mapping) { + return parse(type, mapping).buildMapping(Version.CURRENT); } @SuppressWarnings({"unchecked"}) - private DocumentMapper parse(String type, Map mapping) throws MapperParsingException { + private DocumentMapper.Builder parse(String type, Map mapping) throws MapperParsingException { if (type == null) { throw new MapperParsingException("Failed to derive type"); } @@ -134,7 +138,7 @@ private DocumentMapper parse(String type, Map mapping) throws Ma checkNoRemainingFields(mapping, parserContext.indexVersionCreated(), "Root mapping definition has unsupported parameters: "); - return docBuilder.build(mapperService); + return docBuilder; } public static void checkNoRemainingFields(String fieldName, Map fieldNodeMap, Version indexVersionCreated) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java b/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java index 7b432031238c6..7feec19042691 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java @@ -26,6 +26,7 @@ import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.similarity.SimilarityProvider; +import java.util.List; import java.util.Map; import java.util.Objects; import java.util.function.Function; @@ -172,4 +173,11 @@ public final String simpleName() { * Both {@code this} and {@code mergeWith} will be left unmodified. */ public abstract Mapper merge(Mapper mergeWith); + /** + * Called when this {@linkplain Mapper} is parsed on the {@code _search} + * request to make fields available just for this search. + */ + public List convertToSearchTimeMappings() { + throw new IllegalArgumentException("fields with type [" + typeName() + "] can't be added to [_search]"); + } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java index c6be5abeb47a1..45a81f9a544c5 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -618,6 +618,38 @@ public Analyzer searchQuoteAnalyzer() { return this.searchQuoteAnalyzer; } + /** + * Builds a {@linkplain Function} to lookup mappers for a request, adding + * any {@code extraMapping} provided. + * @param extraMapping extra mappings parse and to add to the request + * lookup or {@code null} if there aren't any extra mappings + */ + public Function newFieldTypeLookup(Map extraMapping) { + if (extraMapping == null || extraMapping.size() == 0) { + return this::fieldType; + } + // TODO parsing an entire mapping is almost certainly "too big" + Mapping mapping = documentMapperParser().parseMapping("extra", extraMapping); + Map extra = new HashMap<>(); + for (Mapper m : mapping.root()) { + // TODO this should dig the sub fields out of object mappers. Probably. + for (MappedFieldType ft: m.convertToSearchTimeMappings()) { + MappedFieldType fromIndexMapping = fieldType(ft.name()); + if (fromIndexMapping != null) { + throw new IllegalArgumentException("No shadowing?! Was [" + fromIndexMapping + "] attempted to add [" + mapping + "]"); + } + extra.put(ft.name(), ft); + } + } + return fullName -> { + MappedFieldType searchTime = extra.get(fullName); + if (searchTime != null) { + return searchTime; + } + return fieldType(fullName); + }; + } + /** * Returns true if fielddata is enabled for the {@link IdFieldMapper} field, false otherwise. */ diff --git a/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java b/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java index b550c6b5552b2..4e4ada7901099 100644 --- a/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java @@ -70,6 +70,7 @@ import java.util.function.BiConsumer; import java.util.function.BiFunction; import java.util.function.BooleanSupplier; +import java.util.function.Function; import java.util.function.LongSupplier; import java.util.function.Predicate; @@ -102,6 +103,7 @@ public class QueryShardContext extends QueryRewriteContext { private boolean mapUnmappedFieldAsString; private NestedScope nestedScope; private final ValuesSourceRegistry valuesSourceRegistry; + private final Function lookupFieldType; public QueryShardContext(int shardId, IndexSettings indexSettings, @@ -119,18 +121,38 @@ public QueryShardContext(int shardId, String clusterAlias, Predicate indexNameMatcher, BooleanSupplier allowExpensiveQueries, - ValuesSourceRegistry valuesSourceRegistry) { - this(shardId, indexSettings, bigArrays, bitsetFilterCache, indexFieldDataLookup, mapperService, similarityService, - scriptService, xContentRegistry, namedWriteableRegistry, client, searcher, nowInMillis, indexNameMatcher, - new Index(RemoteClusterAware.buildRemoteIndexName(clusterAlias, indexSettings.getIndex().getName()), - indexSettings.getIndex().getUUID()), allowExpensiveQueries, valuesSourceRegistry); + ValuesSourceRegistry valuesSourceRegistry, + Map extraMappings) { + this( + shardId, + indexSettings, + bigArrays, + bitsetFilterCache, + indexFieldDataLookup, + mapperService, + similarityService, + scriptService, + xContentRegistry, + namedWriteableRegistry, + client, + searcher, + nowInMillis, + indexNameMatcher, + new Index( + RemoteClusterAware.buildRemoteIndexName(clusterAlias, indexSettings.getIndex().getName()), + indexSettings.getIndex().getUUID() + ), + allowExpensiveQueries, + valuesSourceRegistry, + mapperService.newFieldTypeLookup(extraMappings) + ); } public QueryShardContext(QueryShardContext source) { this(source.shardId, source.indexSettings, source.bigArrays, source.bitsetFilterCache, source.indexFieldDataService, source.mapperService, source.similarityService, source.scriptService, source.getXContentRegistry(), source.getWriteableRegistry(), source.client, source.searcher, source.nowInMillis, source.indexNameMatcher, - source.fullyQualifiedIndex, source.allowExpensiveQueries, source.valuesSourceRegistry); + source.fullyQualifiedIndex, source.allowExpensiveQueries, source.valuesSourceRegistry, source.lookupFieldType); } private QueryShardContext(int shardId, @@ -149,7 +171,8 @@ private QueryShardContext(int shardId, Predicate indexNameMatcher, Index fullyQualifiedIndex, BooleanSupplier allowExpensiveQueries, - ValuesSourceRegistry valuesSourceRegistry) { + ValuesSourceRegistry valuesSourceRegistry, + Function lookupFieldType) { super(xContentRegistry, namedWriteableRegistry, client, nowInMillis); this.shardId = shardId; this.similarityService = similarityService; @@ -166,6 +189,7 @@ private QueryShardContext(int shardId, this.fullyQualifiedIndex = fullyQualifiedIndex; this.allowExpensiveQueries = allowExpensiveQueries; this.valuesSourceRegistry = valuesSourceRegistry; + this.lookupFieldType = lookupFieldType; } private void reset() { @@ -241,7 +265,7 @@ public MappedFieldType fieldMapper(String name) { if (name.equals(TypeFieldMapper.NAME)) { deprecationLogger.deprecate("query_with_types", TYPES_DEPRECATION_MESSAGE); } - return failIfFieldMappingNotFound(name, mapperService.fieldType(name)); + return failIfFieldMappingNotFound(name, lookupFieldType.apply(name)); } public ObjectMapper getObjectMapper(String name) { diff --git a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java index 1ff399860a0f0..f0b6d2c47e646 100644 --- a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java +++ b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java @@ -175,8 +175,9 @@ final class DefaultSearchContext extends SearchContext { engineSearcher.getQueryCache(), engineSearcher.getQueryCachingPolicy(), lowLevelCancellation); this.relativeTimeSupplier = relativeTimeSupplier; this.timeout = timeout; + Map extraMapping = request.source() == null ? null : request.source().extraMapping(); queryShardContext = indexService.newQueryShardContext(request.shardId().id(), searcher, - request::nowInMillis, shardTarget.getClusterAlias()); + request::nowInMillis, shardTarget.getClusterAlias(), extraMapping); queryBoost = request.indexBoost(); this.lowLevelCancellation = lowLevelCancellation; } @@ -775,7 +776,7 @@ public FetchSearchResult fetchResult() { @Override public MappedFieldType fieldType(String name) { - return mapperService().fieldType(name); + return queryShardContext.fieldMapper(name); } @Override diff --git a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java index 9bf4abe0b67e2..c352eb92662df 100644 --- a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java @@ -20,6 +20,8 @@ package org.elasticsearch.search.builder; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.Version; +import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.common.Booleans; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.ParseField; @@ -62,6 +64,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Objects; import static org.elasticsearch.index.query.AbstractQueryBuilder.parseInnerQueryBuilder; @@ -185,6 +188,8 @@ public static HighlightBuilder highlight() { private CollapseBuilder collapse = null; + private Map extraMapping; + /** * Constructs a new search source builder. */ @@ -239,6 +244,9 @@ public SearchSourceBuilder(StreamInput in) throws IOException { sliceBuilder = in.readOptionalWriteable(SliceBuilder::new); collapse = in.readOptionalWriteable(CollapseBuilder::new); trackTotalHitsUpTo = in.readOptionalInt(); + if (in.getVersion().onOrAfter(Version.V_8_0_0)) { + extraMapping = in.readMap(); + } } @Override @@ -293,6 +301,13 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalWriteable(sliceBuilder); out.writeOptionalWriteable(collapse); out.writeOptionalInt(trackTotalHitsUpTo); + if (out.getVersion().onOrAfter(Version.V_8_0_0)) { + out.writeMap(extraMapping); + } else { + throw new IllegalArgumentException( + "Defining [" + CreateIndexRequest.MAPPINGS.getPreferredName() + "] in the search request is not supported before 8.0.0" + ); + } } /** @@ -895,6 +910,21 @@ public List stats() { return stats; } + /** + * Add a search time mapping. + */ + public SearchSourceBuilder extraMapping(Map extraMapping) { + this.extraMapping = extraMapping; + return this; + } + + /** + * The search time mappings. + */ + public Map extraMapping() { + return extraMapping; + } + public SearchSourceBuilder ext(List searchExtBuilders) { this.extBuilders = Objects.requireNonNull(searchExtBuilders, "searchExtBuilders must not be null"); return this; @@ -996,6 +1026,7 @@ private SearchSourceBuilder shallowCopy(QueryBuilder queryBuilder, QueryBuilder rewrittenBuilder.version = version; rewrittenBuilder.seqNoAndPrimaryTerm = seqNoAndPrimaryTerm; rewrittenBuilder.collapse = collapse; + rewrittenBuilder.extraMapping = extraMapping; return rewrittenBuilder; } @@ -1104,6 +1135,8 @@ public void parseXContent(XContentParser parser, boolean checkTrailingTokens) th sliceBuilder = SliceBuilder.fromXContent(parser); } else if (COLLAPSE.match(currentFieldName, parser.getDeprecationHandler())) { collapse = CollapseBuilder.fromXContent(parser); + } else if (CreateIndexRequest.MAPPINGS.match(currentFieldName, parser.getDeprecationHandler())) { + extraMapping = parser.map(); } else { throw new ParsingException(parser.getTokenLocation(), "Unknown key for a " + token + " in [" + currentFieldName + "].", parser.getTokenLocation()); @@ -1551,7 +1584,8 @@ public boolean equals(Object obj) { && Objects.equals(profile, other.profile) && Objects.equals(extBuilders, other.extBuilders) && Objects.equals(collapse, other.collapse) - && Objects.equals(trackTotalHitsUpTo, other.trackTotalHitsUpTo); + && Objects.equals(trackTotalHitsUpTo, other.trackTotalHitsUpTo) + && Objects.equals(extraMapping, other.extraMapping); } @Override diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchDocValuesPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchDocValuesPhase.java index 81838e447d280..7b143ad65cdc8 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchDocValuesPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchDocValuesPhase.java @@ -71,7 +71,7 @@ public void hitsExecute(SearchContext context, SearchHit[] hits) throws IOExcept for (FieldAndFormat fieldAndFormat : context.docValuesContext().fields()) { String field = fieldAndFormat.field; - MappedFieldType fieldType = context.mapperService().fieldType(field); + MappedFieldType fieldType = context.fieldType(field); if (fieldType != null) { final IndexFieldData indexFieldData = context.getForField(fieldType); final boolean isNanosecond; diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java index ab49aea8fa188..fde6520d11cbb 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java @@ -125,7 +125,7 @@ public void setupCreateIndexRequestAndAliasValidator() { queryShardContext = new QueryShardContext(0, new IndexSettings(IndexMetadata.builder("test").settings(indexSettings).build(), indexSettings), BigArrays.NON_RECYCLING_INSTANCE, null, null, null, null, null, xContentRegistry(), writableRegistry(), - null, null, () -> randomNonNegativeLong(), null, null, () -> true, null); + null, null, () -> randomNonNegativeLong(), null, null, () -> true, null, null); } private ClusterState createClusterState(String name, int numShards, int numReplicas, Settings settings) { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java index e780516237aee..49f8a95e957f1 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java @@ -174,7 +174,7 @@ public void testTermQuery() { QueryShardContext context = new QueryShardContext(0, new IndexSettings(IndexMetadata.builder("foo").settings(indexSettings).build(), indexSettings), BigArrays.NON_RECYCLING_INSTANCE, null, null, null, null, null, - xContentRegistry(), writableRegistry(), null, null, () -> nowInMillis, null, null, () -> true, null); + xContentRegistry(), writableRegistry(), null, null, () -> nowInMillis, null, null, () -> true, null, null); MappedFieldType ft = new DateFieldType("field"); String date = "2015-10-12T14:10:55"; long instant = DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(date)).toInstant().toEpochMilli(); @@ -196,7 +196,7 @@ public void testRangeQuery() throws IOException { QueryShardContext context = new QueryShardContext(0, new IndexSettings(IndexMetadata.builder("foo").settings(indexSettings).build(), indexSettings), BigArrays.NON_RECYCLING_INSTANCE, null, null, null, null, null, xContentRegistry(), writableRegistry(), - null, null, () -> nowInMillis, null, null, () -> true, null); + null, null, () -> nowInMillis, null, null, () -> true, null, null); MappedFieldType ft = new DateFieldType("field"); String date1 = "2015-10-12T14:10:55"; String date2 = "2016-04-28T11:33:52"; @@ -240,7 +240,7 @@ public void testRangeQueryWithIndexSort() { QueryShardContext context = new QueryShardContext(0, indexSettings, BigArrays.NON_RECYCLING_INSTANCE, null, null, null, null, null, xContentRegistry(), writableRegistry(), - null, null, () -> 0L, null, null, () -> true, null); + null, null, () -> 0L, null, null, () -> true, null, null); MappedFieldType ft = new DateFieldType("field"); String date1 = "2015-10-12T14:10:55"; diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldTypeTests.java index d0b71cc2d5c46..326c8fc8bcd28 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldTypeTests.java @@ -52,7 +52,7 @@ public void testTermQuery() { QueryShardContext queryShardContext = new QueryShardContext(0, indexSettings, BigArrays.NON_RECYCLING_INSTANCE, null, null, mapperService, - null, null, null, null, null, null, () -> 0L, null, null, () -> true, null); + null, null, null, null, null, null, () -> 0L, null, null, () -> true, null, null); fieldNamesFieldType.setEnabled(true); Query termQuery = fieldNamesFieldType.termQuery("field_name", queryShardContext); assertEquals(new TermQuery(new Term(FieldNamesFieldMapper.CONTENT_TYPE, "field_name")), termQuery); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldTypeTests.java index fb592b1d4d7da..add997668bcbd 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldTypeTests.java @@ -73,6 +73,6 @@ private QueryShardContext createContext() { Predicate indexNameMatcher = pattern -> Regex.simpleMatch(pattern, "index"); return new QueryShardContext(0, indexSettings, null, null, null, null, null, null, xContentRegistry(), writableRegistry(), - null, null, System::currentTimeMillis, null, indexNameMatcher, () -> true, null); + null, null, System::currentTimeMillis, null, indexNameMatcher, () -> true, null, null); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java b/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java index b130f1efd3567..5051129b2847f 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java @@ -37,11 +37,15 @@ import org.elasticsearch.index.analysis.ReloadableCustomAnalyzer; import org.elasticsearch.index.analysis.TokenFilterFactory; import org.elasticsearch.index.mapper.KeywordFieldMapper.KeywordFieldType; +import org.elasticsearch.index.mapper.Mapper.BuilderContext; +import org.elasticsearch.index.mapper.Mapper.TypeParser; import org.elasticsearch.index.mapper.MapperService.MergeReason; import org.elasticsearch.index.mapper.NumberFieldMapper.NumberFieldType; +import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType; import org.elasticsearch.indices.InvalidTypeNameException; import org.elasticsearch.indices.analysis.AnalysisModule.AnalysisProvider; import org.elasticsearch.plugins.AnalysisPlugin; +import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESSingleNodeTestCase; import org.elasticsearch.test.InternalSettingsPlugin; @@ -49,19 +53,25 @@ import java.io.IOException; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.TreeMap; +import java.util.function.Function; import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.spy; public class MapperServiceTests extends ESSingleNodeTestCase { @Override protected Collection> getPlugins() { - return List.of(InternalSettingsPlugin.class, ReloadableFilterPlugin.class); + return List.of(InternalSettingsPlugin.class, ReloadableFilterPlugin.class, TestSearchTimeFieldPlugin.class); } public void testTypeValidation() { @@ -410,6 +420,33 @@ public void testReloadSearchAnalyzers() throws IOException { mapperService.fieldType("otherField").getTextSearchInfo().getSearchQuoteAnalyzer())); } + public void testNewFieldTypeLookupEmpty() throws IOException { + MapperService mapperService = createIndex("test1", Settings.EMPTY, "_doc", "f1", "type=long").mapperService(); + Function lookup = mapperService.newFieldTypeLookup(randomBoolean() ? null : new HashMap<>()); + assertThat(lookup.apply("f1"), notNullValue()); + assertThat(lookup.apply("f2"), nullValue()); + } + + public void testNewFieldTypeLookupInvalidExtra() throws IOException { + MapperService mapperService = createIndex("test1", Settings.EMPTY, "_doc", "f1", "type=long").mapperService(); + Exception e = expectThrows(IllegalArgumentException.class, () -> mapperService.newFieldTypeLookup( + new TreeMap<>(Map.of("properties", new TreeMap<>(Map.of("st1", new TreeMap<>(Map.of("type", "long")))))) + )); + assertThat(e.getMessage(), equalTo("fields with type [long] can't be added to [_search]")); + } + + public void testNewFieldTypeLookupValidExtra() throws IOException { + MapperService mapperService = createIndex("test1", Settings.EMPTY, "_doc", "f1", "type=long").mapperService(); + Function lookup = mapperService.newFieldTypeLookup( + new TreeMap<>(Map.of("properties", new TreeMap<>(Map.of("st1", new TreeMap<>(Map.of("type", "test_search_time")))))) + ); + assertThat(lookup.apply("f1"), notNullValue()); + assertThat(lookup.apply("f2"), nullValue()); + assertThat(lookup.apply("st1"), notNullValue()); + assertThat(lookup.apply("st2"), nullValue()); + } + + private boolean assertSameContainedFilters(TokenFilterFactory[] originalTokenFilter, NamedAnalyzer updatedAnalyzer) { ReloadableCustomAnalyzer updatedReloadableAnalyzer = (ReloadableCustomAnalyzer) updatedAnalyzer.analyzer(); TokenFilterFactory[] newTokenFilters = updatedReloadableAnalyzer.getComponents().getTokenFilters(); @@ -460,4 +497,27 @@ public AnalysisMode getAnalysisMode() { } } + /** + * Creates a totally broken search time field that returns a {@code long}'s + * {@link MappedFieldType} and doesn't crash. Searches would never work + * with it though. + */ + public static final class TestSearchTimeFieldPlugin extends Plugin implements MapperPlugin { + @Override + @SuppressWarnings("rawtypes") + public Map getMappers() { + return Map.of("test_search_time", (name, node, parserContext) -> { + Mapper.Builder delegate = new NumberFieldMapper.TypeParser(NumberType.LONG).parse(name, node, parserContext); + return new Mapper.Builder(name) { + @Override + public Mapper build(BuilderContext context) { + FieldMapper longMapper = spy((FieldMapper) delegate.build(context)); + doReturn(List.of(longMapper.fieldType())).when(longMapper).convertToSearchTimeMappings(); + return longMapper; + } + }; + }); + } + } + } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java index 407fdca4d3b51..43dabbb446f9a 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java @@ -473,7 +473,7 @@ public void doTestIndexSortRangeQueries(NumberType type, Supplier valueS QueryShardContext context = new QueryShardContext(0, indexSettings, BigArrays.NON_RECYCLING_INSTANCE, null, null, null, null, null, xContentRegistry(), writableRegistry(), - null, null, () -> 0L, null, null, () -> true, null); + null, null, () -> 0L, null, null, () -> true, null, null); final int iters = 10; for (int iter = 0; iter < iters; ++iter) { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java index 9dcdb4e32fc28..a2b78fbd77b34 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java @@ -213,7 +213,7 @@ private QueryShardContext createContext() { .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).build(); IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(randomAlphaOfLengthBetween(1, 10), indexSettings); return new QueryShardContext(0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, null, null, null, null, null, - xContentRegistry(), writableRegistry(), null, null, () -> nowInMillis, null, null, () -> true, null); + xContentRegistry(), writableRegistry(), null, null, () -> nowInMillis, null, null, () -> true, null, null); } public void testDateRangeQueryUsingMappingFormat() { diff --git a/server/src/test/java/org/elasticsearch/index/query/IntervalQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/IntervalQueryBuilderTests.java index d155b193f91eb..eec58a4c536a4 100644 --- a/server/src/test/java/org/elasticsearch/index/query/IntervalQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/IntervalQueryBuilderTests.java @@ -423,7 +423,7 @@ public FactoryType compile(Script script, ScriptContext true, null); + null, scriptService, null, null, null, null, null, null, null, () -> true, null, null); String json = "{ \"intervals\" : { \"" + TEXT_FIELD_NAME + "\": { " + "\"match\" : { " + diff --git a/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java b/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java index e119f9fa6e0a3..ca04ee335f1f3 100644 --- a/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java @@ -149,7 +149,7 @@ public void testIndexSortedOnField() { QueryShardContext context = new QueryShardContext( 0, indexSettings, BigArrays.NON_RECYCLING_INSTANCE, null, null, null, null, null, NamedXContentRegistry.EMPTY, new NamedWriteableRegistry(Collections.emptyList()), - null, null, () -> 0L, null, null, () -> true, null); + null, null, () -> 0L, null, null, () -> true, null, null); assertTrue(context.indexSortedOnField("sort_field")); assertFalse(context.indexSortedOnField("second_sort_field")); @@ -175,6 +175,6 @@ public static QueryShardContext createQueryShardContext(String indexUuid, String (mappedFieldType, idxName) -> mappedFieldType.fielddataBuilder(idxName).build(indexSettings, mappedFieldType, null, null, null), mapperService, null, null, NamedXContentRegistry.EMPTY, new NamedWriteableRegistry(Collections.emptyList()), - null, null, () -> nowInMillis, clusterAlias, null, () -> true, null); + null, null, () -> nowInMillis, clusterAlias, null, () -> true, null, null); } } diff --git a/server/src/test/java/org/elasticsearch/index/query/RangeQueryRewriteTests.java b/server/src/test/java/org/elasticsearch/index/query/RangeQueryRewriteTests.java index 263a9eb95942f..583d70862d18d 100644 --- a/server/src/test/java/org/elasticsearch/index/query/RangeQueryRewriteTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/RangeQueryRewriteTests.java @@ -41,7 +41,7 @@ public void testRewriteMissingField() throws Exception { IndexReader reader = new MultiReader(); QueryRewriteContext context = new QueryShardContext(0, indexService.getIndexSettings(), BigArrays.NON_RECYCLING_INSTANCE, null, null, indexService.mapperService(), null, null, xContentRegistry(), writableRegistry(), - null, new IndexSearcher(reader), null, null, null, () -> true, null); + null, new IndexSearcher(reader), null, null, null, () -> true, null, null); RangeQueryBuilder range = new RangeQueryBuilder("foo"); assertEquals(Relation.DISJOINT, range.getRelation(context)); } @@ -59,7 +59,7 @@ public void testRewriteMissingReader() throws Exception { new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE); QueryRewriteContext context = new QueryShardContext(0, indexService.getIndexSettings(), null, null, null, indexService.mapperService(), null, null, xContentRegistry(), writableRegistry(), - null, null, null, null, null, () -> true, null); + null, null, null, null, null, () -> true, null, null); RangeQueryBuilder range = new RangeQueryBuilder("foo"); // can't make assumptions on a missing reader, so it must return INTERSECT assertEquals(Relation.INTERSECTS, range.getRelation(context)); @@ -79,7 +79,7 @@ public void testRewriteEmptyReader() throws Exception { IndexReader reader = new MultiReader(); QueryRewriteContext context = new QueryShardContext(0, indexService.getIndexSettings(), BigArrays.NON_RECYCLING_INSTANCE, null, null, indexService.mapperService(), null, null, xContentRegistry(), writableRegistry(), - null, new IndexSearcher(reader), null, null, null, () -> true, null); + null, new IndexSearcher(reader), null, null, null, () -> true, null, null); RangeQueryBuilder range = new RangeQueryBuilder("foo"); // no values -> DISJOINT assertEquals(Relation.DISJOINT, range.getRelation(context)); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/ExtendedBoundsTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/ExtendedBoundsTests.java index be9c9947b6b37..93df7d7529dcc 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/ExtendedBoundsTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/ExtendedBoundsTests.java @@ -97,7 +97,7 @@ public void testParseAndValidate() { QueryShardContext qsc = new QueryShardContext(0, new IndexSettings(IndexMetadata.builder("foo").settings(indexSettings).build(), indexSettings), BigArrays.NON_RECYCLING_INSTANCE, null, null, null, null, null, xContentRegistry(), writableRegistry(), - null, null, () -> now, null, null, () -> true, null); + null, null, () -> now, null, null, () -> true, null, null); DateFormatter formatter = DateFormatter.forPattern("dateOptionalTime"); DocValueFormat format = new DocValueFormat.DateTime(formatter, ZoneOffset.UTC, DateFieldMapper.Resolution.MILLISECONDS); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricAggregatorTests.java index d03e64e1b120d..b7ad8d9d4de85 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricAggregatorTests.java @@ -566,7 +566,8 @@ protected QueryShardContext queryShardContextMock(IndexSearcher searcher, null, null, () -> true, - valuesSourceRegistry + valuesSourceRegistry, + null ); } } diff --git a/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java b/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java index 445a9fe95258e..40c234c2a340a 100644 --- a/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java @@ -280,7 +280,7 @@ public void testBuildSearchContextHighlight() throws IOException { // shard context will only need indicesQueriesRegistry for building Query objects nested in highlighter QueryShardContext mockShardContext = new QueryShardContext(0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, null, null, null, null, null, xContentRegistry(), namedWriteableRegistry, - null, null, System::currentTimeMillis, null, null, () -> true, null) { + null, null, System::currentTimeMillis, null, null, () -> true, null, null) { @Override public MappedFieldType fieldMapper(String name) { TextFieldMapper.Builder builder = new TextFieldMapper.Builder(name); diff --git a/server/src/test/java/org/elasticsearch/search/rescore/QueryRescorerBuilderTests.java b/server/src/test/java/org/elasticsearch/search/rescore/QueryRescorerBuilderTests.java index 3922bf40d4269..d0a544b4cad2e 100644 --- a/server/src/test/java/org/elasticsearch/search/rescore/QueryRescorerBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/rescore/QueryRescorerBuilderTests.java @@ -144,7 +144,7 @@ public void testBuildRescoreSearchContext() throws ElasticsearchParseException, // shard context will only need indicesQueriesRegistry for building Query objects nested in query rescorer QueryShardContext mockShardContext = new QueryShardContext(0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, null, null, null, null, null, - xContentRegistry(), namedWriteableRegistry, null, null, () -> nowInMillis, null, null, () -> true, null) { + xContentRegistry(), namedWriteableRegistry, null, null, () -> nowInMillis, null, null, () -> true, null, null) { @Override public MappedFieldType fieldMapper(String name) { TextFieldMapper.Builder builder = new TextFieldMapper.Builder(name); @@ -188,7 +188,7 @@ public void testRewritingKeepsSettings() throws IOException { // shard context will only need indicesQueriesRegistry for building Query objects nested in query rescorer QueryShardContext mockShardContext = new QueryShardContext(0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, null, null, null, null, null, - xContentRegistry(), namedWriteableRegistry, null, null, () -> nowInMillis, null, null, () -> true, null) { + xContentRegistry(), namedWriteableRegistry, null, null, () -> nowInMillis, null, null, () -> true, null, null) { @Override public MappedFieldType fieldMapper(String name) { TextFieldMapper.Builder builder = new TextFieldMapper.Builder(name); diff --git a/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java b/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java index 58a65840559fc..407bac630347c 100644 --- a/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java +++ b/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java @@ -200,7 +200,7 @@ protected final QueryShardContext createMockShardContext(IndexSearcher searcher) }; return new QueryShardContext(0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, bitsetFilterCache, indexFieldDataLookup, null, null, scriptService, xContentRegistry(), namedWriteableRegistry, null, searcher, - () -> randomNonNegativeLong(), null, null, () -> true, null) { + () -> randomNonNegativeLong(), null, null, () -> true, null, null) { @Override public MappedFieldType fieldMapper(String name) { diff --git a/server/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java b/server/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java index 64da85173b075..92d879546e82b 100644 --- a/server/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java +++ b/server/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java @@ -173,7 +173,7 @@ public void testBuild() throws IOException { ((Script) invocation.getArguments()[0]).getIdOrCode())); QueryShardContext mockShardContext = new QueryShardContext(0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, null, null, mapperService, null, scriptService, xContentRegistry(), namedWriteableRegistry, null, null, - System::currentTimeMillis, null, null, () -> true, null); + System::currentTimeMillis, null, null, () -> true, null, null); SuggestionContext suggestionContext = suggestionBuilder.build(mockShardContext); assertEquals(toBytesRef(suggestionBuilder.text()), suggestionContext.getText()); diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java index 47ef3456d3bbd..9c64c4e36b027 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java @@ -368,7 +368,7 @@ protected QueryShardContext queryShardContextMock(IndexSearcher searcher, getIndexFieldDataLookup(mapperService, circuitBreakerService), mapperService, null, getMockScriptService(), xContentRegistry(), writableRegistry(), null, searcher, System::currentTimeMillis, null, null, () -> true, - valuesSourceRegistry); + valuesSourceRegistry, null); } /** diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java index aa41bcad47c32..06ab387dc2a3c 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java @@ -416,7 +416,7 @@ public void close() throws IOException { QueryShardContext createShardContext(IndexSearcher searcher) { return new QueryShardContext(0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, bitsetFilterCache, indexFieldDataService::getForField, mapperService, similarityService, scriptService, xContentRegistry, - namedWriteableRegistry, this.client, searcher, () -> nowInMillis, null, indexNameMatcher(), () -> true, null); + namedWriteableRegistry, this.client, searcher, () -> nowInMillis, null, indexNameMatcher(), () -> true, null, null); } ScriptModule createScriptModule(List scriptPlugins) { diff --git a/test/framework/src/test/java/org/elasticsearch/search/MockSearchServiceTests.java b/test/framework/src/test/java/org/elasticsearch/search/MockSearchServiceTests.java index 94bd637781c83..03d529de7e144 100644 --- a/test/framework/src/test/java/org/elasticsearch/search/MockSearchServiceTests.java +++ b/test/framework/src/test/java/org/elasticsearch/search/MockSearchServiceTests.java @@ -43,7 +43,7 @@ public void testAssertNoInFlightContext() { final long nowInMillis = randomNonNegativeLong(); SearchContext s = new TestSearchContext(new QueryShardContext(0, new IndexSettings(EMPTY_INDEX_METADATA, Settings.EMPTY), BigArrays.NON_RECYCLING_INSTANCE, null, null, null, null, null, - xContentRegistry(), writableRegistry(), null, null, () -> nowInMillis, null, null, () -> true, null)) { + xContentRegistry(), writableRegistry(), null, null, () -> nowInMillis, null, null, () -> true, null, null)) { @Override public SearchShardTarget shardTarget() { diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetBitsetCacheTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetBitsetCacheTests.java index 31f2e096a1cee..d2cf4452cb350 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetBitsetCacheTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetBitsetCacheTests.java @@ -523,7 +523,7 @@ private TestIndexContext testIndex(MapperService mapperService, Client client) t final QueryShardContext shardContext = new QueryShardContext(shardId.id(), indexSettings, BigArrays.NON_RECYCLING_INSTANCE, null, null, mapperService, null, null, xContentRegistry(), writableRegistry(), - client, new IndexSearcher(directoryReader), () -> nowInMillis, null, null, () -> true, null); + client, new IndexSearcher(directoryReader), () -> nowInMillis, null, null, () -> true, null, null); context = new TestIndexContext(directory, iw, directoryReader, shardContext, leaf); return context; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapperIntegrationTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapperIntegrationTests.java index 6a1ba5dcc3eb5..74c2544c14ed7 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapperIntegrationTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapperIntegrationTests.java @@ -91,7 +91,7 @@ public void testDLS() throws Exception { final long nowInMillis = randomNonNegativeLong(); QueryShardContext realQueryShardContext = new QueryShardContext(shardId.id(), indexSettings, BigArrays.NON_RECYCLING_INSTANCE, null, null, mapperService, null, null, xContentRegistry(), writableRegistry(), - client, null, () -> nowInMillis, null, null, () -> true, null); + client, null, () -> nowInMillis, null, null, () -> true, null, null); QueryShardContext queryShardContext = spy(realQueryShardContext); DocumentSubsetBitsetCache bitsetCache = new DocumentSubsetBitsetCache(Settings.EMPTY, Executors.newSingleThreadExecutor()); XPackLicenseState licenseState = mock(XPackLicenseState.class); @@ -223,7 +223,7 @@ public void testDLSWithLimitedPermissions() throws Exception { final long nowInMillis = randomNonNegativeLong(); QueryShardContext realQueryShardContext = new QueryShardContext(shardId.id(), indexSettings, BigArrays.NON_RECYCLING_INSTANCE, null, null, mapperService, null, null, xContentRegistry(), writableRegistry(), - client, null, () -> nowInMillis, null, null, () -> true, null); + client, null, () -> nowInMillis, null, null, () -> true, null, null); QueryShardContext queryShardContext = spy(realQueryShardContext); DocumentSubsetBitsetCache bitsetCache = new DocumentSubsetBitsetCache(Settings.EMPTY, Executors.newSingleThreadExecutor()); diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java index 181c35ce1e495..dc5c62b54abc7 100644 --- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java +++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java @@ -90,7 +90,7 @@ private void setup() { settings = createIndexSettings(); queryShardContext = new QueryShardContext(0, settings, BigArrays.NON_RECYCLING_INSTANCE, null, null, null, null, null, - null, null, null, null, () -> 0L, null, null, () -> true, null); + null, null, null, null, () -> 0L, null, null, () -> true, null, null); } public void testSimpleDateHisto() throws Exception { diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptFieldMapper.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptFieldMapper.java index c6fcd43635865..d5b7163c32713 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptFieldMapper.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptFieldMapper.java @@ -57,6 +57,11 @@ protected String contentType() { return CONTENT_TYPE; } + @Override + public List convertToSearchTimeMappings() { + return List.of(fieldType()); + } + public static class Builder extends FieldMapper.Builder { private final ScriptService scriptService; diff --git a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java index 0942a93e67c12..c1d6109acce57 100644 --- a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java +++ b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java @@ -793,7 +793,7 @@ protected final QueryShardContext createMockShardContext() { }; return new QueryShardContext(0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, bitsetFilterCache, indexFieldDataLookup, null, null, null, xContentRegistry(), null, null, null, - () -> randomNonNegativeLong(), null, null, () -> true, null) { + () -> randomNonNegativeLong(), null, null, () -> true, null, null) { @Override public MappedFieldType fieldMapper(String name) { From dc70b428855b87aae52a2043ec9f767d55381beb Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Mon, 13 Jul 2020 15:42:14 -0400 Subject: [PATCH 02/16] Itr --- .../org/elasticsearch/index/IndexService.java | 4 +- .../index/mapper/FieldMapper.java | 8 +++ .../elasticsearch/index/mapper/Mapper.java | 9 --- .../index/mapper/MapperService.java | 51 +++++++++++++---- .../index/query/QueryShardContext.java | 30 ++-------- .../search/DefaultSearchContext.java | 4 +- .../search/builder/SearchSourceBuilder.java | 31 +++++----- .../index/mapper/MapperServiceTests.java | 57 ++++++++++++++----- 8 files changed, 117 insertions(+), 77 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/IndexService.java b/server/src/main/java/org/elasticsearch/index/IndexService.java index 59e74c53dd94a..e2d3dae2e093c 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexService.java +++ b/server/src/main/java/org/elasticsearch/index/IndexService.java @@ -583,14 +583,14 @@ public QueryShardContext newQueryShardContext( IndexSearcher searcher, LongSupplier nowInMillis, String clusterAlias, - Map extraMapping + Map runtimeMappings ) { final SearchIndexNameMatcher indexNameMatcher = new SearchIndexNameMatcher(index().getName(), clusterAlias, clusterService, expressionResolver); return new QueryShardContext( shardId, indexSettings, bigArrays, indexCache.bitsetFilterCache(), indexFieldData::getForField, mapperService(), similarityService(), scriptService, xContentRegistry, namedWriteableRegistry, client, searcher, nowInMillis, clusterAlias, - indexNameMatcher, allowExpensiveQueries, valuesSourceRegistry, extraMapping); + indexNameMatcher, allowExpensiveQueries, valuesSourceRegistry, runtimeMappings); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java index 070a192edacda..f9dafc6fbf847 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java @@ -448,6 +448,14 @@ protected final void doXContentAnalyzers(XContentBuilder builder, boolean includ } } + /** + * Called when this {@linkplain Mapper} is parsed on the {@code _search} + * request to check if this field can be a runtime field. + */ + public boolean isRuntimeField() { + return false; + } + protected static String indexOptionToString(IndexOptions indexOption) { switch (indexOption) { case DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS: diff --git a/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java b/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java index 7feec19042691..c23a8d47451ab 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java @@ -26,7 +26,6 @@ import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.similarity.SimilarityProvider; -import java.util.List; import java.util.Map; import java.util.Objects; import java.util.function.Function; @@ -172,12 +171,4 @@ public final String simpleName() { /** Return the merge of {@code mergeWith} into this. * Both {@code this} and {@code mergeWith} will be left unmodified. */ public abstract Mapper merge(Mapper mergeWith); - - /** - * Called when this {@linkplain Mapper} is parsed on the {@code _search} - * request to make fields available just for this search. - */ - public List convertToSearchTimeMappings() { - throw new IllegalArgumentException("fields with type [" + typeName() + "] can't be added to [_search]"); - } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java index 45a81f9a544c5..e9faaf9cb6bea 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -624,22 +624,49 @@ public Analyzer searchQuoteAnalyzer() { * @param extraMapping extra mappings parse and to add to the request * lookup or {@code null} if there aren't any extra mappings */ - public Function newFieldTypeLookup(Map extraMapping) { - if (extraMapping == null || extraMapping.size() == 0) { + public Function newFieldTypeLookup(Map runtimeMappings) { + if (runtimeMappings == null || runtimeMappings.size() == 0) { return this::fieldType; } - // TODO parsing an entire mapping is almost certainly "too big" - Mapping mapping = documentMapperParser().parseMapping("extra", extraMapping); + Mapper.BuilderContext builderContext = new Mapper.BuilderContext(indexSettings.getSettings(), new ContentPath(0)); Map extra = new HashMap<>(); - for (Mapper m : mapping.root()) { - // TODO this should dig the sub fields out of object mappers. Probably. - for (MappedFieldType ft: m.convertToSearchTimeMappings()) { - MappedFieldType fromIndexMapping = fieldType(ft.name()); - if (fromIndexMapping != null) { - throw new IllegalArgumentException("No shadowing?! Was [" + fromIndexMapping + "] attempted to add [" + mapping + "]"); - } - extra.put(ft.name(), ft); + Collection objectMappers = new ArrayList<>(); + Collection fieldMappers = new ArrayList<>(); + Collection fieldAliasMappers = new ArrayList<>(); + for (Map.Entry runtimeEntry : runtimeMappings.entrySet()) { + @SuppressWarnings("unchecked") // Safe because that is how we deserialized it + Map definition = (Map) runtimeEntry.getValue(); + String type = (String) definition.remove("type"); + if (type == null) { + throw new IllegalArgumentException("[type] is required for runtime mapping [" + runtimeEntry.getKey() + "]"); + } + Mapper.TypeParser parser = documentMapperParser().parserContext().typeParser(type); + if (parser == null) { + throw new IllegalArgumentException("[" + type + "] is unknown type for runtime mapping [" + runtimeEntry.getKey() + "]"); + } + Mapper.Builder builder = parser.parse(runtimeEntry.getKey(), definition, documentMapperParser().parserContext()); + Mapper mapper = builder.build(builderContext); + + MapperUtils.collect(mapper, objectMappers, fieldMappers, fieldAliasMappers); + + } + // We don't do anything with the collected ObjectMappers + for (FieldMapper fm : fieldMappers) { + if (false == fm.isRuntimeField()) { + throw new IllegalArgumentException( + "[" + fm.typeName() + "] are not supported in runtime mappings" + ); } + MappedFieldType fromIndexMapping = fieldType(fm.name()); + if (fromIndexMapping != null) { + throw new IllegalArgumentException( + "[" + fm.name() + "] can't be defined in the search's runtime mappings and the index's mappings" + ); + } + extra.put(fm.name(), fm.fieldType()); + } + if (false == fieldAliasMappers.isEmpty()) { + throw new IllegalArgumentException("aliases are not supported in runtime mappings"); } return fullName -> { MappedFieldType searchTime = extra.get(fullName); diff --git a/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java b/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java index 4e4ada7901099..1627666e6815f 100644 --- a/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java @@ -122,30 +122,12 @@ public QueryShardContext(int shardId, Predicate indexNameMatcher, BooleanSupplier allowExpensiveQueries, ValuesSourceRegistry valuesSourceRegistry, - Map extraMappings) { - this( - shardId, - indexSettings, - bigArrays, - bitsetFilterCache, - indexFieldDataLookup, - mapperService, - similarityService, - scriptService, - xContentRegistry, - namedWriteableRegistry, - client, - searcher, - nowInMillis, - indexNameMatcher, - new Index( - RemoteClusterAware.buildRemoteIndexName(clusterAlias, indexSettings.getIndex().getName()), - indexSettings.getIndex().getUUID() - ), - allowExpensiveQueries, - valuesSourceRegistry, - mapperService.newFieldTypeLookup(extraMappings) - ); + Map runtimeMappings) { + this(shardId, indexSettings, bigArrays, bitsetFilterCache, indexFieldDataLookup, mapperService, similarityService, + scriptService, xContentRegistry, namedWriteableRegistry, client, searcher, nowInMillis, indexNameMatcher, + new Index(RemoteClusterAware.buildRemoteIndexName(clusterAlias, indexSettings.getIndex().getName()), + indexSettings.getIndex().getUUID()), allowExpensiveQueries, valuesSourceRegistry, + mapperService.newFieldTypeLookup(runtimeMappings)); } public QueryShardContext(QueryShardContext source) { diff --git a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java index f0b6d2c47e646..b5947c91f18d0 100644 --- a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java +++ b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java @@ -175,9 +175,9 @@ final class DefaultSearchContext extends SearchContext { engineSearcher.getQueryCache(), engineSearcher.getQueryCachingPolicy(), lowLevelCancellation); this.relativeTimeSupplier = relativeTimeSupplier; this.timeout = timeout; - Map extraMapping = request.source() == null ? null : request.source().extraMapping(); + Map runtimeMappings = request.source() == null ? null : request.source().runtimeMappings(); queryShardContext = indexService.newQueryShardContext(request.shardId().id(), searcher, - request::nowInMillis, shardTarget.getClusterAlias(), extraMapping); + request::nowInMillis, shardTarget.getClusterAlias(), runtimeMappings); queryBoost = request.indexBoost(); this.lowLevelCancellation = lowLevelCancellation; } diff --git a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java index c352eb92662df..021deb9e0f8a3 100644 --- a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java @@ -110,6 +110,7 @@ public final class SearchSourceBuilder implements Writeable, ToXContentObject, R public static final ParseField SEARCH_AFTER = new ParseField("search_after"); public static final ParseField COLLAPSE = new ParseField("collapse"); public static final ParseField SLICE = new ParseField("slice"); + public static final ParseField RUNTIME_MAPPINGS = new ParseField("runtime_mappings"); public static SearchSourceBuilder fromXContent(XContentParser parser) throws IOException { return fromXContent(parser, true); @@ -188,7 +189,7 @@ public static HighlightBuilder highlight() { private CollapseBuilder collapse = null; - private Map extraMapping; + private Map runtimeMappings; /** * Constructs a new search source builder. @@ -245,7 +246,8 @@ public SearchSourceBuilder(StreamInput in) throws IOException { collapse = in.readOptionalWriteable(CollapseBuilder::new); trackTotalHitsUpTo = in.readOptionalInt(); if (in.getVersion().onOrAfter(Version.V_8_0_0)) { - extraMapping = in.readMap(); + // TODO update version after backporting runtime fields + runtimeMappings = in.readMap(); } } @@ -302,11 +304,10 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalWriteable(collapse); out.writeOptionalInt(trackTotalHitsUpTo); if (out.getVersion().onOrAfter(Version.V_8_0_0)) { - out.writeMap(extraMapping); + // TODO update version after backporting runtime fields + out.writeMap(runtimeMappings); } else { - throw new IllegalArgumentException( - "Defining [" + CreateIndexRequest.MAPPINGS.getPreferredName() + "] in the search request is not supported before 8.0.0" - ); + throw new IllegalArgumentException("[" + RUNTIME_MAPPINGS.getPreferredName() + "] are not supported on nodes older than 8.0.0"); } } @@ -911,18 +912,18 @@ public List stats() { } /** - * Add a search time mapping. + * Extra runtime field mappings. */ - public SearchSourceBuilder extraMapping(Map extraMapping) { - this.extraMapping = extraMapping; + public SearchSourceBuilder runtimeMappings(Map runtimeMappings) { + this.runtimeMappings = runtimeMappings; return this; } /** - * The search time mappings. + * Extra runtime field mappings. */ - public Map extraMapping() { - return extraMapping; + public Map runtimeMappings() { + return runtimeMappings; } public SearchSourceBuilder ext(List searchExtBuilders) { @@ -1026,7 +1027,7 @@ private SearchSourceBuilder shallowCopy(QueryBuilder queryBuilder, QueryBuilder rewrittenBuilder.version = version; rewrittenBuilder.seqNoAndPrimaryTerm = seqNoAndPrimaryTerm; rewrittenBuilder.collapse = collapse; - rewrittenBuilder.extraMapping = extraMapping; + rewrittenBuilder.runtimeMappings = runtimeMappings; return rewrittenBuilder; } @@ -1136,7 +1137,7 @@ public void parseXContent(XContentParser parser, boolean checkTrailingTokens) th } else if (COLLAPSE.match(currentFieldName, parser.getDeprecationHandler())) { collapse = CollapseBuilder.fromXContent(parser); } else if (CreateIndexRequest.MAPPINGS.match(currentFieldName, parser.getDeprecationHandler())) { - extraMapping = parser.map(); + runtimeMappings = parser.map(); } else { throw new ParsingException(parser.getTokenLocation(), "Unknown key for a " + token + " in [" + currentFieldName + "].", parser.getTokenLocation()); @@ -1585,7 +1586,7 @@ public boolean equals(Object obj) { && Objects.equals(extBuilders, other.extBuilders) && Objects.equals(collapse, other.collapse) && Objects.equals(trackTotalHitsUpTo, other.trackTotalHitsUpTo) - && Objects.equals(extraMapping, other.extraMapping); + && Objects.equals(runtimeMappings, other.runtimeMappings); } @Override diff --git a/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java b/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java index 5051129b2847f..6a9dc0a6e7f58 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java @@ -71,7 +71,7 @@ public class MapperServiceTests extends ESSingleNodeTestCase { @Override protected Collection> getPlugins() { - return List.of(InternalSettingsPlugin.class, ReloadableFilterPlugin.class, TestSearchTimeFieldPlugin.class); + return List.of(InternalSettingsPlugin.class, ReloadableFilterPlugin.class, TestRuntimeFieldPlugin.class); } public void testTypeValidation() { @@ -427,26 +427,57 @@ public void testNewFieldTypeLookupEmpty() throws IOException { assertThat(lookup.apply("f2"), nullValue()); } - public void testNewFieldTypeLookupInvalidExtra() throws IOException { + public void testNewFieldTypeLookupInvalidRuntimeMappings() throws IOException { MapperService mapperService = createIndex("test1", Settings.EMPTY, "_doc", "f1", "type=long").mapperService(); Exception e = expectThrows(IllegalArgumentException.class, () -> mapperService.newFieldTypeLookup( - new TreeMap<>(Map.of("properties", new TreeMap<>(Map.of("st1", new TreeMap<>(Map.of("type", "long")))))) + new TreeMap<>(Map.of("st1", new TreeMap<>(Map.of("type", "long")))) )); - assertThat(e.getMessage(), equalTo("fields with type [long] can't be added to [_search]")); + assertThat(e.getMessage(), equalTo("[long] are not supported in runtime mappings")); } - public void testNewFieldTypeLookupValidExtra() throws IOException { + public void testNewFieldTypeLookupNoType() throws IOException { + MapperService mapperService = createIndex("test1", Settings.EMPTY, "_doc", "f1", "type=long").mapperService(); + Exception e = expectThrows(IllegalArgumentException.class, () -> mapperService.newFieldTypeLookup( + new TreeMap<>(Map.of("r1", new TreeMap<>(Map.of("stuff", "foo")))) + )); + assertThat(e.getMessage(), equalTo("[type] is required for runtime mapping [r1]")); + } + + public void testNewFieldTypeLookupUnknownType() throws IOException { + MapperService mapperService = createIndex("test1", Settings.EMPTY, "_doc", "f1", "type=long").mapperService(); + Exception e = expectThrows(IllegalArgumentException.class, () -> mapperService.newFieldTypeLookup( + new TreeMap<>(Map.of("r1", new TreeMap<>(Map.of("type", "asdf")))) + )); + assertThat(e.getMessage(), equalTo("[asdf] is unknown type for runtime mapping [r1]")); + } + + public void testNewFieldTypeLookupAliases() throws IOException { + MapperService mapperService = createIndex("test1", Settings.EMPTY, "_doc", "f1", "type=long").mapperService(); + Exception e = expectThrows(IllegalArgumentException.class, () -> mapperService.newFieldTypeLookup( + new TreeMap<>(Map.of("r1", new TreeMap<>(Map.of("type", "alias", "path", "f1")))) + )); + assertThat(e.getMessage(), equalTo("aliases are not supported in runtime mappings")); + } + + public void testNewFieldTypeLookupShadowingRuntimeMappings() throws IOException { + MapperService mapperService = createIndex("test1", Settings.EMPTY, "_doc", "field", "type=long").mapperService(); + Exception e = expectThrows(IllegalArgumentException.class, () -> mapperService.newFieldTypeLookup( + new TreeMap<>(Map.of("field", new TreeMap<>(Map.of("type", "test_runtime")))) + )); + assertThat(e.getMessage(), equalTo("[field] can't be defined in the search's runtime mappings and the index's mappings")); + } + + public void testNewFieldTypeLookupValidRuntimeMappings() throws IOException { MapperService mapperService = createIndex("test1", Settings.EMPTY, "_doc", "f1", "type=long").mapperService(); Function lookup = mapperService.newFieldTypeLookup( - new TreeMap<>(Map.of("properties", new TreeMap<>(Map.of("st1", new TreeMap<>(Map.of("type", "test_search_time")))))) + new TreeMap<>(Map.of("r1", new TreeMap<>(Map.of("type", "test_runtime")))) ); assertThat(lookup.apply("f1"), notNullValue()); assertThat(lookup.apply("f2"), nullValue()); - assertThat(lookup.apply("st1"), notNullValue()); - assertThat(lookup.apply("st2"), nullValue()); + assertThat(lookup.apply("r1"), notNullValue()); + assertThat(lookup.apply("r2"), nullValue()); } - private boolean assertSameContainedFilters(TokenFilterFactory[] originalTokenFilter, NamedAnalyzer updatedAnalyzer) { ReloadableCustomAnalyzer updatedReloadableAnalyzer = (ReloadableCustomAnalyzer) updatedAnalyzer.analyzer(); TokenFilterFactory[] newTokenFilters = updatedReloadableAnalyzer.getComponents().getTokenFilters(); @@ -498,21 +529,21 @@ public AnalysisMode getAnalysisMode() { } /** - * Creates a totally broken search time field that returns a {@code long}'s + * Creates a totally broken runtime field that returns a {@code long}'s * {@link MappedFieldType} and doesn't crash. Searches would never work * with it though. */ - public static final class TestSearchTimeFieldPlugin extends Plugin implements MapperPlugin { + public static final class TestRuntimeFieldPlugin extends Plugin implements MapperPlugin { @Override @SuppressWarnings("rawtypes") public Map getMappers() { - return Map.of("test_search_time", (name, node, parserContext) -> { + return Map.of("test_runtime", (name, node, parserContext) -> { Mapper.Builder delegate = new NumberFieldMapper.TypeParser(NumberType.LONG).parse(name, node, parserContext); return new Mapper.Builder(name) { @Override public Mapper build(BuilderContext context) { FieldMapper longMapper = spy((FieldMapper) delegate.build(context)); - doReturn(List.of(longMapper.fieldType())).when(longMapper).convertToSearchTimeMappings(); + doReturn(true).when(longMapper).isRuntimeField(); return longMapper; } }; From 1938a0e95d3c68511f5c7ceb50bfaafdf424ced8 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Mon, 13 Jul 2020 15:54:05 -0400 Subject: [PATCH 03/16] Itr --- .../java/org/elasticsearch/index/mapper/MapperService.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java index e9faaf9cb6bea..d8aa668dfac0e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -621,7 +621,7 @@ public Analyzer searchQuoteAnalyzer() { /** * Builds a {@linkplain Function} to lookup mappers for a request, adding * any {@code extraMapping} provided. - * @param extraMapping extra mappings parse and to add to the request + * @param runtimeMappings extra mappings parse and to add to the request * lookup or {@code null} if there aren't any extra mappings */ public Function newFieldTypeLookup(Map runtimeMappings) { @@ -629,7 +629,6 @@ public Function newFieldTypeLookup(Map return this::fieldType; } Mapper.BuilderContext builderContext = new Mapper.BuilderContext(indexSettings.getSettings(), new ContentPath(0)); - Map extra = new HashMap<>(); Collection objectMappers = new ArrayList<>(); Collection fieldMappers = new ArrayList<>(); Collection fieldAliasMappers = new ArrayList<>(); @@ -651,6 +650,7 @@ public Function newFieldTypeLookup(Map } // We don't do anything with the collected ObjectMappers + Map extra = new HashMap<>(); for (FieldMapper fm : fieldMappers) { if (false == fm.isRuntimeField()) { throw new IllegalArgumentException( From 9d8e4b36a6b39517bd92f2dd477519ed5aefe460 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Mon, 13 Jul 2020 19:52:15 -0400 Subject: [PATCH 04/16] Tests --- .../test/runtime_fields/10_keyword.yml | 71 +++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/10_keyword.yml diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/10_keyword.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/10_keyword.yml new file mode 100644 index 0000000000000..0d3a1dfaeb91c --- /dev/null +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/10_keyword.yml @@ -0,0 +1,71 @@ +--- +setup: + - do: + indices.create: + index: sensor + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + properties: + timestamp: + type: date + temperature: + type: long + voltage: + type: float + node: + type: keyword + day_of_week: + type: script + runtime_type: keyword + script: value(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT)) + + - do: + bulk: + index: sensor + refresh: true + body: | + {"index":{}} + {"timestamp": 1516729294000, "temperature": 200, "voltage": 5.2, "node": "a"} + {"index":{}} + {"timestamp": 1516642894000, "temperature": 201, "voltage": 5.8, "node": "b"} + {"index":{}} + {"timestamp": 1516556494000, "temperature": 202, "voltage": 5.1, "node": "a"} + {"index":{}} + {"timestamp": 1516470094000, "temperature": 198, "voltage": 5.6, "node": "b"} + {"index":{}} + {"timestamp": 1516383694000, "temperature": 200, "voltage": 4.2, "node": "c"} + {"index":{}} + {"timestamp": 1516297294000, "temperature": 202, "voltage": 4.0, "node": "c"} + +--- +"term query": + - do: + search: + index: sensor + body: + query: + term: + day_of_week: Monday + - match: {hits.total.value: 1} + - match: {hits.hits.0._source.voltage: 5.8} + + +--- +"runtime defined": + - do: + search: + index: sensor + body: + runtime_mappings: + first_letter: + type: script + runtime_type: keyword + script: value(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT).charAt(0)) + query: + term: + day_of_week: M + - match: {hits.total.value: 1} + - match: {hits.hits.0._source.voltage: 5.8} From 7346b8baf11f6821bcb002b0bc79815ea41fbf68 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Mon, 13 Jul 2020 19:57:51 -0400 Subject: [PATCH 05/16] Parse plz --- .../org/elasticsearch/search/builder/SearchSourceBuilder.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java index 021deb9e0f8a3..3123b617c0d2a 100644 --- a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java @@ -1136,7 +1136,7 @@ public void parseXContent(XContentParser parser, boolean checkTrailingTokens) th sliceBuilder = SliceBuilder.fromXContent(parser); } else if (COLLAPSE.match(currentFieldName, parser.getDeprecationHandler())) { collapse = CollapseBuilder.fromXContent(parser); - } else if (CreateIndexRequest.MAPPINGS.match(currentFieldName, parser.getDeprecationHandler())) { + } else if (RUNTIME_MAPPINGS.match(currentFieldName, parser.getDeprecationHandler())) { runtimeMappings = parser.map(); } else { throw new ParsingException(parser.getTokenLocation(), "Unknown key for a " + token + " in [" + currentFieldName + "].", From 4891bda0f9fb532fdedc452072d5b51b9867f831 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Mon, 13 Jul 2020 19:58:12 -0400 Subject: [PATCH 06/16] Precommit --- .../org/elasticsearch/search/builder/SearchSourceBuilder.java | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java index 3123b617c0d2a..7bd5295f9d7d1 100644 --- a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java @@ -21,7 +21,6 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.Version; -import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.common.Booleans; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.ParseField; From 77f35cb0f6e2372b6ecda1e27ae2e0bb1daf50ee Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Mon, 13 Jul 2020 19:59:58 -0400 Subject: [PATCH 07/16] Fix test --- .../rest-api-spec/test/runtime_fields/10_keyword.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/10_keyword.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/10_keyword.yml index a83dfce0bc78f..7b01619f61142 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/10_keyword.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/10_keyword.yml @@ -62,9 +62,9 @@ setup: first_letter: type: script runtime_type: keyword - script: value(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT).charAt(0)) + script: value(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT).charAt(0).toString()) query: term: - day_of_week: M + first_letter: M - match: {hits.total.value: 1} - match: {hits.hits.0._source.voltage: 5.8} From 2011791a8dfd85eb7394af5912ac8e5166ce9be8 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Tue, 14 Jul 2020 10:57:50 -0400 Subject: [PATCH 08/16] Fix test --- .../elasticsearch/search/aggregations/AggregatorTestCase.java | 1 + 1 file changed, 1 insertion(+) diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java index 42d0f0082059f..00edfded1f4fb 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java @@ -308,6 +308,7 @@ public boolean shouldCache(Query query) { MapperService mapperService = mapperServiceMock(); when(mapperService.getIndexSettings()).thenReturn(indexSettings); when(mapperService.hasNested()).thenReturn(false); + when(mapperService.newFieldTypeLookup(Mockito.anyMap())).thenCallRealMethod(); when(searchContext.mapperService()).thenReturn(mapperService); IndexFieldDataService ifds = new IndexFieldDataService(indexSettings, new IndicesFieldDataCache(Settings.EMPTY, new IndexFieldDataCache.Listener() { From 2b9b34cb2d9aba215e9406348345228ebf593c06 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Tue, 14 Jul 2020 14:55:37 -0400 Subject: [PATCH 09/16] Revert some stuff --- .../java/org/elasticsearch/index/IndexService.java | 7 ++++++- .../elasticsearch/index/mapper/DocumentMapper.java | 11 ++++------- .../index/mapper/DocumentMapperParser.java | 10 +++------- .../java/org/elasticsearch/index/mapper/Mapper.java | 1 + 4 files changed, 14 insertions(+), 15 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/IndexService.java b/server/src/main/java/org/elasticsearch/index/IndexService.java index ff859e34ae13f..5112622ac5c01 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexService.java +++ b/server/src/main/java/org/elasticsearch/index/IndexService.java @@ -587,6 +587,12 @@ public QueryShardContext newQueryShardContext(int shardId, IndexSearcher searche return newQueryShardContext(shardId, searcher, nowInMillis, clusterAlias, null); } + /** + * Creates a new QueryShardContext. + * + * Passing a {@code null} {@link IndexSearcher} will return a valid context, however it won't be able to make + * {@link IndexReader}-specific optimizations, such as rewriting containing range queries. + */ public QueryShardContext newQueryShardContext( int shardId, IndexSearcher searcher, @@ -602,7 +608,6 @@ public QueryShardContext newQueryShardContext( indexNameMatcher, allowExpensiveQueries, valuesSourceRegistry, runtimeMappings); } - /** * The {@link ThreadPool} to use for this index. */ diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java index 8555e0acc4329..a3ec5a933e258 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java @@ -103,17 +103,14 @@ public Builder put(MetadataFieldMapper.Builder mapper) { return this; } - public Mapping buildMapping(Version indexVersionCreated) { + public DocumentMapper build(MapperService mapperService) { Objects.requireNonNull(rootObjectMapper, "Mapper builder must have the root object mapper set"); - return new Mapping( - indexVersionCreated, + Mapping mapping = new Mapping( + mapperService.getIndexSettings().getIndexVersionCreated(), rootObjectMapper, metadataMappers.values().toArray(new MetadataFieldMapper[metadataMappers.values().size()]), meta); - } - - public DocumentMapper build(MapperService mapperService) { - return new DocumentMapper(mapperService, buildMapping(mapperService.getIndexSettings().getIndexVersionCreated())); + return new DocumentMapper(mapperService, mapping); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java index aa0cb3617babe..d4f16bcaf8458 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java @@ -80,15 +80,11 @@ public DocumentMapper parse(@Nullable String type, CompressedXContent source) th if (mapping == null) { mapping = new HashMap<>(); } - return parse(type, mapping).build(mapperService); - } - - public Mapping parseMapping(String type, Map mapping) { - return parse(type, mapping).buildMapping(Version.CURRENT); + return parse(type, mapping); } @SuppressWarnings({"unchecked"}) - private DocumentMapper.Builder parse(String type, Map mapping) throws MapperParsingException { + private DocumentMapper parse(String type, Map mapping) throws MapperParsingException { if (type == null) { throw new MapperParsingException("Failed to derive type"); } @@ -138,7 +134,7 @@ private DocumentMapper.Builder parse(String type, Map mapping) t checkNoRemainingFields(mapping, parserContext.indexVersionCreated(), "Root mapping definition has unsupported parameters: "); - return docBuilder; + return docBuilder.build(mapperService); } public static void checkNoRemainingFields(String fieldName, Map fieldNodeMap, Version indexVersionCreated) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java b/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java index c23a8d47451ab..7b432031238c6 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java @@ -171,4 +171,5 @@ public final String simpleName() { /** Return the merge of {@code mergeWith} into this. * Both {@code this} and {@code mergeWith} will be left unmodified. */ public abstract Mapper merge(Mapper mergeWith); + } From 0546dc24a7e29b34b2abdd0433540e2a51631c18 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Tue, 14 Jul 2020 14:56:50 -0400 Subject: [PATCH 10/16] Revert --- .../main/java/org/elasticsearch/index/mapper/MapperService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java index 98fa8e276929e..4aa3cfc400e7d 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -620,7 +620,7 @@ public Analyzer searchQuoteAnalyzer() { /** * Builds a {@linkplain Function} to lookup mappers for a request, adding - * any {@code extraMapping} provided. + * any {@code runtimeMappings} provided. * @param runtimeMappings extra mappings parse and to add to the request * lookup or {@code null} if there aren't any extra mappings */ From a86ffaa7187ae7c00d0ea47f3b3031e532628513 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Tue, 14 Jul 2020 15:44:02 -0400 Subject: [PATCH 11/16] WIP --- .../org/elasticsearch/index/IndexService.java | 5 +- .../index/mapper/FieldTypeLookup.java | 39 +++++++++- .../index/mapper/MapperService.java | 78 ++++++++++++------- .../index/query/QueryShardContext.java | 16 ++-- .../fetch/subphase/FetchDocValuesPhase.java | 2 +- 5 files changed, 98 insertions(+), 42 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/IndexService.java b/server/src/main/java/org/elasticsearch/index/IndexService.java index 5112622ac5c01..5a73686a5660c 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexService.java +++ b/server/src/main/java/org/elasticsearch/index/IndexService.java @@ -603,9 +603,10 @@ public QueryShardContext newQueryShardContext( final SearchIndexNameMatcher indexNameMatcher = new SearchIndexNameMatcher(index().getName(), clusterAlias, clusterService, expressionResolver); return new QueryShardContext( - shardId, indexSettings, bigArrays, indexCache.bitsetFilterCache(), indexFieldData::getForField, mapperService(), + shardId, indexSettings, bigArrays, indexCache.bitsetFilterCache(), indexFieldData::getForField, + mapperService().forSearch(runtimeMappings), similarityService(), scriptService, xContentRegistry, namedWriteableRegistry, client, searcher, nowInMillis, clusterAlias, - indexNameMatcher, allowExpensiveQueries, valuesSourceRegistry, runtimeMappings); + indexNameMatcher, allowExpensiveQueries, valuesSourceRegistry); } /** diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldTypeLookup.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldTypeLookup.java index f4529b4643f2b..40e6284599970 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FieldTypeLookup.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldTypeLookup.java @@ -35,8 +35,8 @@ */ class FieldTypeLookup implements Iterable { - private final Map fullNameToFieldType = new HashMap<>(); - private final Map aliasToConcreteName = new HashMap<>(); + private final Map fullNameToFieldType; + private final Map aliasToConcreteName; private final DynamicKeyFieldTypeLookup dynamicKeyLookup; FieldTypeLookup() { @@ -48,6 +48,7 @@ class FieldTypeLookup implements Iterable { Map dynamicKeyMappers = new HashMap<>(); + fullNameToFieldType = new HashMap<>(fieldMappers.size()); for (FieldMapper fieldMapper : fieldMappers) { String fieldName = fieldMapper.name(); MappedFieldType fieldType = fieldMapper.fieldType(); @@ -57,6 +58,7 @@ class FieldTypeLookup implements Iterable { } } + aliasToConcreteName = new HashMap<>(fieldAliasMappers.size()); for (FieldAliasMapper fieldAliasMapper : fieldAliasMappers) { String aliasName = fieldAliasMapper.name(); String path = fieldAliasMapper.path(); @@ -66,6 +68,16 @@ class FieldTypeLookup implements Iterable { this.dynamicKeyLookup = new DynamicKeyFieldTypeLookup(dynamicKeyMappers, aliasToConcreteName); } + private FieldTypeLookup( + Map fullNameToFieldType, + Map aliasToConcreteName, + DynamicKeyFieldTypeLookup dynamicKeyLookup + ) { + this.fullNameToFieldType = fullNameToFieldType; + this.aliasToConcreteName = aliasToConcreteName; + this.dynamicKeyLookup = dynamicKeyLookup; + } + /** * Returns the mapped field type for the given field name. */ @@ -105,4 +117,27 @@ public Iterator iterator() { Iterator keyedFieldTypes = dynamicKeyLookup.fieldTypes(); return Iterators.concat(concreteFieldTypes, keyedFieldTypes); } + + /** + * Returns a copy of this lookup with runtime mappings merged into it. + */ + public FieldTypeLookup withRuntimeMappings(Collection runtimeMappings) { + Map mappers = new HashMap<>(fullNameToFieldType.size() + runtimeMappings.size()); + mappers.putAll(fullNameToFieldType); + for (FieldMapper fm : runtimeMappings) { + if (false == fm.isRuntimeField()) { + throw new IllegalArgumentException( + "[" + fm.typeName() + "] are not supported in runtime mappings" + ); + } + MappedFieldType fromIndexMapping = fullNameToFieldType.get(fm.name()); + if (fromIndexMapping != null) { + throw new IllegalArgumentException( + "[" + fm.name() + "] can't be defined in the search's runtime mappings and the index's mappings" + ); + } + mappers.put(fm.name(), fm.fieldType()); + } + return new FieldTypeLookup(mappers, aliasToConcreteName, dynamicKeyLookup); + } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java index 4aa3cfc400e7d..50eb0376b116f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -150,6 +150,38 @@ public MapperService(IndexSettings indexSettings, IndexAnalyzers indexAnalyzers, this.idFieldDataEnabled = idFieldDataEnabled; } + private MapperService( + IndexSettings indexSettings, + IndexAnalyzers indexAnalyzers, + DocumentMapper mapper, + FieldTypeLookup fieldTypes, + Map fullPathObjectMappers, + boolean hasNested, + DocumentMapperParser documentParser, + Version indexVersionCreated, + MapperAnalyzerWrapper indexAnalyzer, + MapperAnalyzerWrapper searchAnalyzer, + MapperAnalyzerWrapper searchQuoteAnalyzer, + Map unmappedFieldTypes, + MapperRegistry mapperRegistry, + BooleanSupplier idFieldDataEnabled + ) { + super(indexSettings); + this.indexAnalyzers = indexAnalyzers; + this.mapper = mapper; + this.fieldTypes = fieldTypes; + this.fullPathObjectMappers = fullPathObjectMappers; + this.hasNested = hasNested; + this.documentParser = documentParser; + this.indexVersionCreated = indexVersionCreated; + this.indexAnalyzer = indexAnalyzer; + this.searchAnalyzer = searchAnalyzer; + this.searchQuoteAnalyzer = searchQuoteAnalyzer; + this.unmappedFieldTypes = unmappedFieldTypes; + this.mapperRegistry = mapperRegistry; + this.idFieldDataEnabled = idFieldDataEnabled; + } + public boolean hasNested() { return this.hasNested; } @@ -624,9 +656,9 @@ public Analyzer searchQuoteAnalyzer() { * @param runtimeMappings extra mappings parse and to add to the request * lookup or {@code null} if there aren't any extra mappings */ - public Function newFieldTypeLookup(Map runtimeMappings) { + public MapperService forSearch(Map runtimeMappings) { if (runtimeMappings == null || runtimeMappings.size() == 0) { - return this::fieldType; + return this; } Mapper.BuilderContext builderContext = new Mapper.BuilderContext(indexSettings.getSettings(), new ContentPath(0)); Collection objectMappers = new ArrayList<>(); @@ -645,36 +677,30 @@ public Function newFieldTypeLookup(Map } Mapper.Builder builder = parser.parse(runtimeEntry.getKey(), definition, documentMapperParser().parserContext()); Mapper mapper = builder.build(builderContext); - + + // MapperUtils.collect will find the mappers declared in objectss MapperUtils.collect(mapper, objectMappers, fieldMappers, fieldAliasMappers); } - // We don't do anything with the collected ObjectMappers - Map extra = new HashMap<>(); - for (FieldMapper fm : fieldMappers) { - if (false == fm.isRuntimeField()) { - throw new IllegalArgumentException( - "[" + fm.typeName() + "] are not supported in runtime mappings" - ); - } - MappedFieldType fromIndexMapping = fieldType(fm.name()); - if (fromIndexMapping != null) { - throw new IllegalArgumentException( - "[" + fm.name() + "] can't be defined in the search's runtime mappings and the index's mappings" - ); - } - extra.put(fm.name(), fm.fieldType()); - } if (false == fieldAliasMappers.isEmpty()) { throw new IllegalArgumentException("aliases are not supported in runtime mappings"); } - return fullName -> { - MappedFieldType searchTime = extra.get(fullName); - if (searchTime != null) { - return searchTime; - } - return fieldType(fullName); - }; + return new MapperService( + indexSettings, + indexAnalyzers, + mapper, + fieldTypes.withRuntimeMappings(fieldMappers), + fullPathObjectMappers, + hasNested, + documentParser, + indexVersionCreated, + indexAnalyzer, + searchAnalyzer, + searchQuoteAnalyzer, + unmappedFieldTypes, + mapperRegistry, + idFieldDataEnabled + ); } /** diff --git a/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java b/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java index 1627666e6815f..b655b900a35c2 100644 --- a/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java @@ -70,7 +70,6 @@ import java.util.function.BiConsumer; import java.util.function.BiFunction; import java.util.function.BooleanSupplier; -import java.util.function.Function; import java.util.function.LongSupplier; import java.util.function.Predicate; @@ -103,7 +102,6 @@ public class QueryShardContext extends QueryRewriteContext { private boolean mapUnmappedFieldAsString; private NestedScope nestedScope; private final ValuesSourceRegistry valuesSourceRegistry; - private final Function lookupFieldType; public QueryShardContext(int shardId, IndexSettings indexSettings, @@ -121,20 +119,18 @@ public QueryShardContext(int shardId, String clusterAlias, Predicate indexNameMatcher, BooleanSupplier allowExpensiveQueries, - ValuesSourceRegistry valuesSourceRegistry, - Map runtimeMappings) { + ValuesSourceRegistry valuesSourceRegistry) { this(shardId, indexSettings, bigArrays, bitsetFilterCache, indexFieldDataLookup, mapperService, similarityService, scriptService, xContentRegistry, namedWriteableRegistry, client, searcher, nowInMillis, indexNameMatcher, new Index(RemoteClusterAware.buildRemoteIndexName(clusterAlias, indexSettings.getIndex().getName()), - indexSettings.getIndex().getUUID()), allowExpensiveQueries, valuesSourceRegistry, - mapperService.newFieldTypeLookup(runtimeMappings)); + indexSettings.getIndex().getUUID()), allowExpensiveQueries, valuesSourceRegistry); } public QueryShardContext(QueryShardContext source) { this(source.shardId, source.indexSettings, source.bigArrays, source.bitsetFilterCache, source.indexFieldDataService, source.mapperService, source.similarityService, source.scriptService, source.getXContentRegistry(), source.getWriteableRegistry(), source.client, source.searcher, source.nowInMillis, source.indexNameMatcher, - source.fullyQualifiedIndex, source.allowExpensiveQueries, source.valuesSourceRegistry, source.lookupFieldType); + source.fullyQualifiedIndex, source.allowExpensiveQueries, source.valuesSourceRegistry); } private QueryShardContext(int shardId, @@ -153,8 +149,7 @@ private QueryShardContext(int shardId, Predicate indexNameMatcher, Index fullyQualifiedIndex, BooleanSupplier allowExpensiveQueries, - ValuesSourceRegistry valuesSourceRegistry, - Function lookupFieldType) { + ValuesSourceRegistry valuesSourceRegistry) { super(xContentRegistry, namedWriteableRegistry, client, nowInMillis); this.shardId = shardId; this.similarityService = similarityService; @@ -171,7 +166,6 @@ private QueryShardContext(int shardId, this.fullyQualifiedIndex = fullyQualifiedIndex; this.allowExpensiveQueries = allowExpensiveQueries; this.valuesSourceRegistry = valuesSourceRegistry; - this.lookupFieldType = lookupFieldType; } private void reset() { @@ -247,7 +241,7 @@ public MappedFieldType fieldMapper(String name) { if (name.equals(TypeFieldMapper.NAME)) { deprecationLogger.deprecate("query_with_types", TYPES_DEPRECATION_MESSAGE); } - return failIfFieldMappingNotFound(name, lookupFieldType.apply(name)); + return failIfFieldMappingNotFound(name, mapperService.fieldType(name)); } public ObjectMapper getObjectMapper(String name) { diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchDocValuesPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchDocValuesPhase.java index 7b143ad65cdc8..81838e447d280 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchDocValuesPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchDocValuesPhase.java @@ -71,7 +71,7 @@ public void hitsExecute(SearchContext context, SearchHit[] hits) throws IOExcept for (FieldAndFormat fieldAndFormat : context.docValuesContext().fields()) { String field = fieldAndFormat.field; - MappedFieldType fieldType = context.fieldType(field); + MappedFieldType fieldType = context.mapperService().fieldType(field); if (fieldType != null) { final IndexFieldData indexFieldData = context.getForField(fieldType); final boolean isNanosecond; From 3c12ed0f84e33560f66f4c223d44ff47d3f0ebb8 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Tue, 21 Jul 2020 10:56:11 -0400 Subject: [PATCH 12/16] itr --- .../test/runtime_fields/10_keyword.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/10_keyword.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/10_keyword.yml index fb1a2d91042c5..7292690c56aab 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/10_keyword.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/10_keyword.yml @@ -121,13 +121,13 @@ setup: sort: timestamp docvalue_fields: [first_letter] - match: {hits.total.value: 6} - - match: {hits.hits.0.fields.day_of_week: [T] } - - match: {hits.hits.1.fields.day_of_week: [F] } - - match: {hits.hits.2.fields.day_of_week: [S] } - - match: {hits.hits.3.fields.day_of_week: [S] } - - match: {hits.hits.4.fields.day_of_week: [M] } - - match: {hits.hits.5.fields.day_of_week: [T] } - - match: {hits.hits.6.fields.day_of_week: [W] } + - match: {hits.hits.0.fields.first_letter: [T] } + - match: {hits.hits.1.fields.first_letter: [F] } + - match: {hits.hits.2.fields.first_letter: [S] } + - match: {hits.hits.3.fields.first_letter: [S] } + - match: {hits.hits.4.fields.first_letter: [M] } + - match: {hits.hits.5.fields.first_letter: [T] } + - match: {hits.hits.6.fields.first_letter: [W] } --- "terms agg": From 09aa1f4a5545746c9b12e665ac9dd5afc3f791ef Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Tue, 21 Jul 2020 11:17:01 -0400 Subject: [PATCH 13/16] iter --- .../search/DefaultSearchContext.java | 2 +- .../index/mapper/MapperServiceTests.java | 1 - .../test/runtime_fields/10_keyword.yml | 69 ++++++++++++------- 3 files changed, 45 insertions(+), 27 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java index b5947c91f18d0..c99938280ad36 100644 --- a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java +++ b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java @@ -467,7 +467,7 @@ public IndexShard indexShard() { @Override public MapperService mapperService() { - return indexService.mapperService(); + return queryShardContext.getMapperService(); } @Override diff --git a/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java b/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java index 94c9c9cd86d72..45357b922a1b7 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java @@ -477,7 +477,6 @@ public void testNewFieldTypeLookupValidRuntimeMappings() throws IOException { assertThat(mapperService.fieldType("r1"), notNullValue()); assertThat(mapperService.fieldType("r2"), nullValue()); assertThat(mapperService.simpleMatchToFullName("*1"), equalTo(Set.of("f1", "r1"))); - } private boolean assertSameContainedFilters(TokenFilterFactory[] originalTokenFilter, NamedAnalyzer updatedAnalyzer) { diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/10_keyword.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/10_keyword.yml index 7292690c56aab..469cecea7195a 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/10_keyword.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/10_keyword.yml @@ -104,31 +104,6 @@ setup: - match: {hits.hits.0.fields.days_starting_with_t: [Thursday] } - match: {hits.hits.0.fields.prefixed_node: [node_c] } ---- -"runtime defined": - - do: - search: - index: sensor - body: - runtime_mappings: - first_letter: - type: script - runtime_type: keyword - script: | - for (String node : doc['day_of_week']) { - value(node.charAt(0).toString()); - } - sort: timestamp - docvalue_fields: [first_letter] - - match: {hits.total.value: 6} - - match: {hits.hits.0.fields.first_letter: [T] } - - match: {hits.hits.1.fields.first_letter: [F] } - - match: {hits.hits.2.fields.first_letter: [S] } - - match: {hits.hits.3.fields.first_letter: [S] } - - match: {hits.hits.4.fields.first_letter: [M] } - - match: {hits.hits.5.fields.first_letter: [T] } - - match: {hits.hits.6.fields.first_letter: [W] } - --- "terms agg": - do: @@ -248,3 +223,47 @@ setup: day_of_week: M*ay - match: {hits.total.value: 1} - match: {hits.hits.0._source.voltage: 5.8} + +--- +"runtime defined docvalue_fields": + - do: + search: + index: sensor + body: + runtime_mappings: + first_letter: + type: script + runtime_type: keyword + script: | + for (String node : doc['day_of_week']) { + value(node.charAt(0).toString()); + } + sort: timestamp + docvalue_fields: [first_letter] + - match: {hits.total.value: 6} + - match: {hits.hits.0.fields.first_letter: [T] } + - match: {hits.hits.1.fields.first_letter: [F] } + - match: {hits.hits.2.fields.first_letter: [S] } + - match: {hits.hits.3.fields.first_letter: [S] } + - match: {hits.hits.4.fields.first_letter: [M] } + - match: {hits.hits.5.fields.first_letter: [T] } + +--- +"runtime defined query": + - do: + search: + index: sensor + body: + runtime_mappings: + first_letter: + type: script + runtime_type: keyword + script: | + for (String node : doc['day_of_week']) { + value(node.charAt(0).toString()); + } + query: + term: + first_letter: M + - match: {hits.total.value: 1} + - match: {hits.hits.0._source.voltage: 5.8} From 65a53262aa8f80413ab2c1e9333678727359b683 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Tue, 21 Jul 2020 12:37:18 -0400 Subject: [PATCH 14/16] Tests --- .../search/builder/SearchSourceBuilder.java | 6 +++++- .../action/search/SearchRequestTests.java | 21 +++++++++++++++++++ .../search/DefaultSearchContextTests.java | 4 ++++ .../search/RandomSearchRequestGenerator.java | 4 ++++ 4 files changed, 34 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java index 7bd5295f9d7d1..80652179a5905 100644 --- a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java @@ -306,7 +306,11 @@ public void writeTo(StreamOutput out) throws IOException { // TODO update version after backporting runtime fields out.writeMap(runtimeMappings); } else { - throw new IllegalArgumentException("[" + RUNTIME_MAPPINGS.getPreferredName() + "] are not supported on nodes older than 8.0.0"); + if (runtimeMappings != null && false == runtimeMappings.isEmpty()) { + throw new IllegalArgumentException( + "[" + RUNTIME_MAPPINGS.getPreferredName() + "] are not supported on nodes older than 8.0.0" + ); + } } } diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchRequestTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchRequestTests.java index 5ff66a0709701..7629c6993e2fb 100644 --- a/server/src/test/java/org/elasticsearch/action/search/SearchRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/SearchRequestTests.java @@ -37,6 +37,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.Map; import static java.util.Collections.emptyMap; import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode; @@ -80,6 +81,12 @@ public void testSerialization() throws Exception { public void testRandomVersionSerialization() throws IOException { SearchRequest searchRequest = createSearchRequest(); Version version = VersionUtils.randomVersion(random()); + if (version.before(Version.V_8_0_0)) { + // Runtime mappings aren't supported before 8.0.0 and will fail the serialization + if (searchRequest.source() != null) { + searchRequest.source().runtimeMappings(null); + } + } SearchRequest deserializedRequest = copyWriteable(searchRequest, namedWriteableRegistry, SearchRequest::new, version); assertEquals(searchRequest.isCcsMinimizeRoundtrips(), deserializedRequest.isCcsMinimizeRoundtrips()); assertEquals(searchRequest.getLocalClusterAlias(), deserializedRequest.getLocalClusterAlias()); @@ -87,6 +94,20 @@ public void testRandomVersionSerialization() throws IOException { assertEquals(searchRequest.isFinalReduce(), deserializedRequest.isFinalReduce()); } + public void testRuntimeMappingsNotSupported() throws IOException { + SearchRequest searchRequest = createSearchRequest(); + if (searchRequest.source() == null) { + searchRequest.source(new SearchSourceBuilder()); + } + searchRequest.source().runtimeMappings(Map.of("foo", "bar")); + Version version = randomValueOtherThanMany(v -> v.onOrAfter(Version.V_8_0_0), () -> VersionUtils.randomVersion(random())); + Exception e = expectThrows( + IllegalArgumentException.class, + () -> copyWriteable(searchRequest, namedWriteableRegistry, SearchRequest::new, version) + ); + assertThat(e.getMessage(), equalTo("[runtime_mappings] are not supported on nodes older than 8.0.0")); + } + public void testIllegalArguments() { SearchRequest searchRequest = new SearchRequest(); assertNotNull(searchRequest.indices()); diff --git a/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java b/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java index 5be92101e6360..f9d62cf6ac46c 100644 --- a/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java +++ b/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java @@ -62,6 +62,8 @@ import java.util.UUID; import static org.hamcrest.Matchers.equalTo; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyInt; import static org.mockito.Matchers.anyObject; import static org.mockito.Matchers.anyString; import static org.mockito.Matchers.eq; @@ -103,6 +105,8 @@ public void testPreProcess() throws Exception { MapperService mapperService = mock(MapperService.class); when(mapperService.hasNested()).thenReturn(randomBoolean()); when(indexService.mapperService()).thenReturn(mapperService); + when(indexService.newQueryShardContext(anyInt(), any(), any(), any(), any())).thenReturn(queryShardContext); + when(queryShardContext.getMapperService()).thenReturn(mapperService); IndexMetadata indexMetadata = IndexMetadata.builder("index").settings(settings).build(); IndexSettings indexSettings = new IndexSettings(indexMetadata, Settings.EMPTY); diff --git a/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java b/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java index 43e282af5f227..91e7acb9c8f8d 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java @@ -52,6 +52,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.function.Supplier; import static java.util.Collections.emptyMap; @@ -363,6 +364,9 @@ public static SearchSourceBuilder randomSearchSourceBuilder( if (randomBoolean()) { builder.collapse(randomCollapseBuilder.get()); } + if (randomBoolean()) { + builder.runtimeMappings(Map.of("foo", Map.of("bar", "baz"))); + } return builder; } } From acefc5dc50f5f301962f5d16be5849a5b1116470 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Tue, 21 Jul 2020 15:54:48 -0400 Subject: [PATCH 15/16] Oops --- .../org/elasticsearch/search/builder/SearchSourceBuilder.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java index 80652179a5905..1affaa02b63ad 100644 --- a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java @@ -1337,6 +1337,10 @@ public XContentBuilder innerToXContent(XContentBuilder builder, Params params) t if (collapse != null) { builder.field(COLLAPSE.getPreferredName(), collapse); } + + if (runtimeMappings != null && false == runtimeMappings.isEmpty()) { + builder.field(RUNTIME_MAPPINGS.getPreferredName(), runtimeMappings); + } return builder; } From 394ba8f39e70b4610e4efdb78a25c4b35bacd55f Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Tue, 21 Jul 2020 15:58:51 -0400 Subject: [PATCH 16/16] sub-objects --- .../index/mapper/MapperServiceTests.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java b/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java index 45357b922a1b7..6b4892d967f13 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java @@ -479,6 +479,17 @@ public void testNewFieldTypeLookupValidRuntimeMappings() throws IOException { assertThat(mapperService.simpleMatchToFullName("*1"), equalTo(Set.of("f1", "r1"))); } + public void testNewFieldTypeLookupRuntimeMappingsInObject() throws IOException { + Map inner = new TreeMap<>(Map.of("inner", new TreeMap<>(Map.of("type", "test_runtime")))); + MapperService mapperService = createIndex("test1", Settings.EMPTY, "_doc", "f1", "type=long").mapperService() + .forSearch(new TreeMap<>(Map.of("r1", new TreeMap<>(Map.of("type", "object", "properties", inner))))); + assertThat(mapperService.fieldType("f1"), notNullValue()); + assertThat(mapperService.fieldType("f2"), nullValue()); + assertThat(mapperService.fieldType("r1.inner"), notNullValue()); + assertThat(mapperService.fieldType("r2"), nullValue()); + assertThat(mapperService.simpleMatchToFullName("*1*"), equalTo(Set.of("f1", "r1.inner"))); + } + private boolean assertSameContainedFilters(TokenFilterFactory[] originalTokenFilter, NamedAnalyzer updatedAnalyzer) { ReloadableCustomAnalyzer updatedReloadableAnalyzer = (ReloadableCustomAnalyzer) updatedAnalyzer.analyzer(); TokenFilterFactory[] newTokenFilters = updatedReloadableAnalyzer.getComponents().getTokenFilters();