diff --git a/server/src/main/java/org/elasticsearch/index/IndexService.java b/server/src/main/java/org/elasticsearch/index/IndexService.java index 19ea0542f7fe2..5a73686a5660c 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexService.java +++ b/server/src/main/java/org/elasticsearch/index/IndexService.java @@ -584,10 +584,27 @@ public IndexSettings getIndexSettings() { * {@link IndexReader}-specific optimizations, such as rewriting containing range queries. */ public QueryShardContext newQueryShardContext(int shardId, IndexSearcher searcher, LongSupplier nowInMillis, String clusterAlias) { + return newQueryShardContext(shardId, searcher, nowInMillis, clusterAlias, null); + } + + /** + * Creates a new QueryShardContext. + * + * Passing a {@code null} {@link IndexSearcher} will return a valid context, however it won't be able to make + * {@link IndexReader}-specific optimizations, such as rewriting containing range queries. + */ + public QueryShardContext newQueryShardContext( + int shardId, + IndexSearcher searcher, + LongSupplier nowInMillis, + String clusterAlias, + Map runtimeMappings + ) { final SearchIndexNameMatcher indexNameMatcher = new SearchIndexNameMatcher(index().getName(), clusterAlias, clusterService, expressionResolver); return new QueryShardContext( - shardId, indexSettings, bigArrays, indexCache.bitsetFilterCache(), indexFieldData::getForField, mapperService(), + shardId, indexSettings, bigArrays, indexCache.bitsetFilterCache(), indexFieldData::getForField, + mapperService().forSearch(runtimeMappings), similarityService(), scriptService, xContentRegistry, namedWriteableRegistry, client, searcher, nowInMillis, clusterAlias, indexNameMatcher, allowExpensiveQueries, valuesSourceRegistry); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java index 30f38ce16d8a3..131446634ef49 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java @@ -448,6 +448,14 @@ protected final void doXContentAnalyzers(XContentBuilder builder, boolean includ } } + /** + * Called when this {@linkplain Mapper} is parsed on the {@code _search} + * request to check if this field can be a runtime field. + */ + public boolean isRuntimeField() { + return false; + } + protected static String indexOptionToString(IndexOptions indexOption) { switch (indexOption) { case DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS: diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldTypeLookup.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldTypeLookup.java index f4529b4643f2b..40e6284599970 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FieldTypeLookup.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldTypeLookup.java @@ -35,8 +35,8 @@ */ class FieldTypeLookup implements Iterable { - private final Map fullNameToFieldType = new HashMap<>(); - private final Map aliasToConcreteName = new HashMap<>(); + private final Map fullNameToFieldType; + private final Map aliasToConcreteName; private final DynamicKeyFieldTypeLookup dynamicKeyLookup; FieldTypeLookup() { @@ -48,6 +48,7 @@ class FieldTypeLookup implements Iterable { Map dynamicKeyMappers = new HashMap<>(); + fullNameToFieldType = new HashMap<>(fieldMappers.size()); for (FieldMapper fieldMapper : fieldMappers) { String fieldName = fieldMapper.name(); MappedFieldType fieldType = fieldMapper.fieldType(); @@ -57,6 +58,7 @@ class FieldTypeLookup implements Iterable { } } + aliasToConcreteName = new HashMap<>(fieldAliasMappers.size()); for (FieldAliasMapper fieldAliasMapper : fieldAliasMappers) { String aliasName = fieldAliasMapper.name(); String path = fieldAliasMapper.path(); @@ -66,6 +68,16 @@ class FieldTypeLookup implements Iterable { this.dynamicKeyLookup = new DynamicKeyFieldTypeLookup(dynamicKeyMappers, aliasToConcreteName); } + private FieldTypeLookup( + Map fullNameToFieldType, + Map aliasToConcreteName, + DynamicKeyFieldTypeLookup dynamicKeyLookup + ) { + this.fullNameToFieldType = fullNameToFieldType; + this.aliasToConcreteName = aliasToConcreteName; + this.dynamicKeyLookup = dynamicKeyLookup; + } + /** * Returns the mapped field type for the given field name. */ @@ -105,4 +117,27 @@ public Iterator iterator() { Iterator keyedFieldTypes = dynamicKeyLookup.fieldTypes(); return Iterators.concat(concreteFieldTypes, keyedFieldTypes); } + + /** + * Returns a copy of this lookup with runtime mappings merged into it. + */ + public FieldTypeLookup withRuntimeMappings(Collection runtimeMappings) { + Map mappers = new HashMap<>(fullNameToFieldType.size() + runtimeMappings.size()); + mappers.putAll(fullNameToFieldType); + for (FieldMapper fm : runtimeMappings) { + if (false == fm.isRuntimeField()) { + throw new IllegalArgumentException( + "[" + fm.typeName() + "] are not supported in runtime mappings" + ); + } + MappedFieldType fromIndexMapping = fullNameToFieldType.get(fm.name()); + if (fromIndexMapping != null) { + throw new IllegalArgumentException( + "[" + fm.name() + "] can't be defined in the search's runtime mappings and the index's mappings" + ); + } + mappers.put(fm.name(), fm.fieldType()); + } + return new FieldTypeLookup(mappers, aliasToConcreteName, dynamicKeyLookup); + } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java index f4b7efa092a45..9a27e090bbef0 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -150,6 +150,38 @@ public MapperService(IndexSettings indexSettings, IndexAnalyzers indexAnalyzers, this.idFieldDataEnabled = idFieldDataEnabled; } + private MapperService( + IndexSettings indexSettings, + IndexAnalyzers indexAnalyzers, + DocumentMapper mapper, + FieldTypeLookup fieldTypes, + Map fullPathObjectMappers, + boolean hasNested, + DocumentMapperParser documentParser, + Version indexVersionCreated, + MapperAnalyzerWrapper indexAnalyzer, + MapperAnalyzerWrapper searchAnalyzer, + MapperAnalyzerWrapper searchQuoteAnalyzer, + Map unmappedFieldTypes, + MapperRegistry mapperRegistry, + BooleanSupplier idFieldDataEnabled + ) { + super(indexSettings); + this.indexAnalyzers = indexAnalyzers; + this.mapper = mapper; + this.fieldTypes = fieldTypes; + this.fullPathObjectMappers = fullPathObjectMappers; + this.hasNested = hasNested; + this.documentParser = documentParser; + this.indexVersionCreated = indexVersionCreated; + this.indexAnalyzer = indexAnalyzer; + this.searchAnalyzer = searchAnalyzer; + this.searchQuoteAnalyzer = searchQuoteAnalyzer; + this.unmappedFieldTypes = unmappedFieldTypes; + this.mapperRegistry = mapperRegistry; + this.idFieldDataEnabled = idFieldDataEnabled; + } + public boolean hasNested() { return this.hasNested; } @@ -618,6 +650,59 @@ public Analyzer searchQuoteAnalyzer() { return this.searchQuoteAnalyzer; } + /** + * Builds a {@linkplain Function} to lookup mappers for a request, adding + * any {@code runtimeMappings} provided. + * @param runtimeMappings extra mappings parse and to add to the request + * lookup or {@code null} if there aren't any extra mappings + */ + public MapperService forSearch(Map runtimeMappings) { + if (runtimeMappings == null || runtimeMappings.size() == 0) { + return this; + } + Mapper.BuilderContext builderContext = new Mapper.BuilderContext(indexSettings.getSettings(), new ContentPath(0)); + Collection objectMappers = new ArrayList<>(); + Collection fieldMappers = new ArrayList<>(); + Collection fieldAliasMappers = new ArrayList<>(); + for (Map.Entry runtimeEntry : runtimeMappings.entrySet()) { + @SuppressWarnings("unchecked") // Safe because that is how we deserialized it + Map definition = (Map) runtimeEntry.getValue(); + String type = (String) definition.remove("type"); + if (type == null) { + throw new IllegalArgumentException("[type] is required for runtime mapping [" + runtimeEntry.getKey() + "]"); + } + Mapper.TypeParser parser = documentMapperParser().parserContext().typeParser(type); + if (parser == null) { + throw new IllegalArgumentException("[" + type + "] is unknown type for runtime mapping [" + runtimeEntry.getKey() + "]"); + } + Mapper.Builder builder = parser.parse(runtimeEntry.getKey(), definition, documentMapperParser().parserContext()); + Mapper mapper = builder.build(builderContext); + + // MapperUtils.collect will find the mappers declared in objectss + MapperUtils.collect(mapper, objectMappers, fieldMappers, fieldAliasMappers); + + } + if (false == fieldAliasMappers.isEmpty()) { + throw new IllegalArgumentException("aliases are not supported in runtime mappings"); + } + return new MapperService( + indexSettings, + indexAnalyzers, + mapper, + fieldTypes.withRuntimeMappings(fieldMappers), + fullPathObjectMappers, + hasNested, + documentParser, + indexVersionCreated, + indexAnalyzer, + searchAnalyzer, + searchQuoteAnalyzer, + unmappedFieldTypes, + mapperRegistry, + idFieldDataEnabled + ); + } + /** * Returns true if fielddata is enabled for the {@link IdFieldMapper} field, false otherwise. */ diff --git a/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java b/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java index be449f625c399..691a2ab97cc86 100644 --- a/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java @@ -121,9 +121,9 @@ public QueryShardContext(int shardId, BooleanSupplier allowExpensiveQueries, ValuesSourceRegistry valuesSourceRegistry) { this(shardId, indexSettings, bigArrays, bitsetFilterCache, indexFieldDataLookup, mapperService, similarityService, - scriptService, xContentRegistry, namedWriteableRegistry, client, searcher, nowInMillis, indexNameMatcher, - new Index(RemoteClusterAware.buildRemoteIndexName(clusterAlias, indexSettings.getIndex().getName()), - indexSettings.getIndex().getUUID()), allowExpensiveQueries, valuesSourceRegistry); + scriptService, xContentRegistry, namedWriteableRegistry, client, searcher, nowInMillis, indexNameMatcher, + new Index(RemoteClusterAware.buildRemoteIndexName(clusterAlias, indexSettings.getIndex().getName()), + indexSettings.getIndex().getUUID()), allowExpensiveQueries, valuesSourceRegistry); } public QueryShardContext(QueryShardContext source) { diff --git a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java index 1ff399860a0f0..c99938280ad36 100644 --- a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java +++ b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java @@ -175,8 +175,9 @@ final class DefaultSearchContext extends SearchContext { engineSearcher.getQueryCache(), engineSearcher.getQueryCachingPolicy(), lowLevelCancellation); this.relativeTimeSupplier = relativeTimeSupplier; this.timeout = timeout; + Map runtimeMappings = request.source() == null ? null : request.source().runtimeMappings(); queryShardContext = indexService.newQueryShardContext(request.shardId().id(), searcher, - request::nowInMillis, shardTarget.getClusterAlias()); + request::nowInMillis, shardTarget.getClusterAlias(), runtimeMappings); queryBoost = request.indexBoost(); this.lowLevelCancellation = lowLevelCancellation; } @@ -466,7 +467,7 @@ public IndexShard indexShard() { @Override public MapperService mapperService() { - return indexService.mapperService(); + return queryShardContext.getMapperService(); } @Override @@ -775,7 +776,7 @@ public FetchSearchResult fetchResult() { @Override public MappedFieldType fieldType(String name) { - return mapperService().fieldType(name); + return queryShardContext.fieldMapper(name); } @Override diff --git a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java index 9bf4abe0b67e2..1affaa02b63ad 100644 --- a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java @@ -20,6 +20,7 @@ package org.elasticsearch.search.builder; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.Version; import org.elasticsearch.common.Booleans; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.ParseField; @@ -62,6 +63,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Objects; import static org.elasticsearch.index.query.AbstractQueryBuilder.parseInnerQueryBuilder; @@ -107,6 +109,7 @@ public final class SearchSourceBuilder implements Writeable, ToXContentObject, R public static final ParseField SEARCH_AFTER = new ParseField("search_after"); public static final ParseField COLLAPSE = new ParseField("collapse"); public static final ParseField SLICE = new ParseField("slice"); + public static final ParseField RUNTIME_MAPPINGS = new ParseField("runtime_mappings"); public static SearchSourceBuilder fromXContent(XContentParser parser) throws IOException { return fromXContent(parser, true); @@ -185,6 +188,8 @@ public static HighlightBuilder highlight() { private CollapseBuilder collapse = null; + private Map runtimeMappings; + /** * Constructs a new search source builder. */ @@ -239,6 +244,10 @@ public SearchSourceBuilder(StreamInput in) throws IOException { sliceBuilder = in.readOptionalWriteable(SliceBuilder::new); collapse = in.readOptionalWriteable(CollapseBuilder::new); trackTotalHitsUpTo = in.readOptionalInt(); + if (in.getVersion().onOrAfter(Version.V_8_0_0)) { + // TODO update version after backporting runtime fields + runtimeMappings = in.readMap(); + } } @Override @@ -293,6 +302,16 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalWriteable(sliceBuilder); out.writeOptionalWriteable(collapse); out.writeOptionalInt(trackTotalHitsUpTo); + if (out.getVersion().onOrAfter(Version.V_8_0_0)) { + // TODO update version after backporting runtime fields + out.writeMap(runtimeMappings); + } else { + if (runtimeMappings != null && false == runtimeMappings.isEmpty()) { + throw new IllegalArgumentException( + "[" + RUNTIME_MAPPINGS.getPreferredName() + "] are not supported on nodes older than 8.0.0" + ); + } + } } /** @@ -895,6 +914,21 @@ public List stats() { return stats; } + /** + * Extra runtime field mappings. + */ + public SearchSourceBuilder runtimeMappings(Map runtimeMappings) { + this.runtimeMappings = runtimeMappings; + return this; + } + + /** + * Extra runtime field mappings. + */ + public Map runtimeMappings() { + return runtimeMappings; + } + public SearchSourceBuilder ext(List searchExtBuilders) { this.extBuilders = Objects.requireNonNull(searchExtBuilders, "searchExtBuilders must not be null"); return this; @@ -996,6 +1030,7 @@ private SearchSourceBuilder shallowCopy(QueryBuilder queryBuilder, QueryBuilder rewrittenBuilder.version = version; rewrittenBuilder.seqNoAndPrimaryTerm = seqNoAndPrimaryTerm; rewrittenBuilder.collapse = collapse; + rewrittenBuilder.runtimeMappings = runtimeMappings; return rewrittenBuilder; } @@ -1104,6 +1139,8 @@ public void parseXContent(XContentParser parser, boolean checkTrailingTokens) th sliceBuilder = SliceBuilder.fromXContent(parser); } else if (COLLAPSE.match(currentFieldName, parser.getDeprecationHandler())) { collapse = CollapseBuilder.fromXContent(parser); + } else if (RUNTIME_MAPPINGS.match(currentFieldName, parser.getDeprecationHandler())) { + runtimeMappings = parser.map(); } else { throw new ParsingException(parser.getTokenLocation(), "Unknown key for a " + token + " in [" + currentFieldName + "].", parser.getTokenLocation()); @@ -1300,6 +1337,10 @@ public XContentBuilder innerToXContent(XContentBuilder builder, Params params) t if (collapse != null) { builder.field(COLLAPSE.getPreferredName(), collapse); } + + if (runtimeMappings != null && false == runtimeMappings.isEmpty()) { + builder.field(RUNTIME_MAPPINGS.getPreferredName(), runtimeMappings); + } return builder; } @@ -1551,7 +1592,8 @@ public boolean equals(Object obj) { && Objects.equals(profile, other.profile) && Objects.equals(extBuilders, other.extBuilders) && Objects.equals(collapse, other.collapse) - && Objects.equals(trackTotalHitsUpTo, other.trackTotalHitsUpTo); + && Objects.equals(trackTotalHitsUpTo, other.trackTotalHitsUpTo) + && Objects.equals(runtimeMappings, other.runtimeMappings); } @Override diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchRequestTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchRequestTests.java index 5ff66a0709701..7629c6993e2fb 100644 --- a/server/src/test/java/org/elasticsearch/action/search/SearchRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/SearchRequestTests.java @@ -37,6 +37,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.Map; import static java.util.Collections.emptyMap; import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode; @@ -80,6 +81,12 @@ public void testSerialization() throws Exception { public void testRandomVersionSerialization() throws IOException { SearchRequest searchRequest = createSearchRequest(); Version version = VersionUtils.randomVersion(random()); + if (version.before(Version.V_8_0_0)) { + // Runtime mappings aren't supported before 8.0.0 and will fail the serialization + if (searchRequest.source() != null) { + searchRequest.source().runtimeMappings(null); + } + } SearchRequest deserializedRequest = copyWriteable(searchRequest, namedWriteableRegistry, SearchRequest::new, version); assertEquals(searchRequest.isCcsMinimizeRoundtrips(), deserializedRequest.isCcsMinimizeRoundtrips()); assertEquals(searchRequest.getLocalClusterAlias(), deserializedRequest.getLocalClusterAlias()); @@ -87,6 +94,20 @@ public void testRandomVersionSerialization() throws IOException { assertEquals(searchRequest.isFinalReduce(), deserializedRequest.isFinalReduce()); } + public void testRuntimeMappingsNotSupported() throws IOException { + SearchRequest searchRequest = createSearchRequest(); + if (searchRequest.source() == null) { + searchRequest.source(new SearchSourceBuilder()); + } + searchRequest.source().runtimeMappings(Map.of("foo", "bar")); + Version version = randomValueOtherThanMany(v -> v.onOrAfter(Version.V_8_0_0), () -> VersionUtils.randomVersion(random())); + Exception e = expectThrows( + IllegalArgumentException.class, + () -> copyWriteable(searchRequest, namedWriteableRegistry, SearchRequest::new, version) + ); + assertThat(e.getMessage(), equalTo("[runtime_mappings] are not supported on nodes older than 8.0.0")); + } + public void testIllegalArguments() { SearchRequest searchRequest = new SearchRequest(); assertNotNull(searchRequest.indices()); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java b/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java index b130f1efd3567..6b4892d967f13 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java @@ -37,11 +37,15 @@ import org.elasticsearch.index.analysis.ReloadableCustomAnalyzer; import org.elasticsearch.index.analysis.TokenFilterFactory; import org.elasticsearch.index.mapper.KeywordFieldMapper.KeywordFieldType; +import org.elasticsearch.index.mapper.Mapper.BuilderContext; +import org.elasticsearch.index.mapper.Mapper.TypeParser; import org.elasticsearch.index.mapper.MapperService.MergeReason; import org.elasticsearch.index.mapper.NumberFieldMapper.NumberFieldType; +import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType; import org.elasticsearch.indices.InvalidTypeNameException; import org.elasticsearch.indices.analysis.AnalysisModule.AnalysisProvider; import org.elasticsearch.plugins.AnalysisPlugin; +import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESSingleNodeTestCase; import org.elasticsearch.test.InternalSettingsPlugin; @@ -49,19 +53,25 @@ import java.io.IOException; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.TreeMap; import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.spy; public class MapperServiceTests extends ESSingleNodeTestCase { @Override protected Collection> getPlugins() { - return List.of(InternalSettingsPlugin.class, ReloadableFilterPlugin.class); + return List.of(InternalSettingsPlugin.class, ReloadableFilterPlugin.class, TestRuntimeFieldPlugin.class); } public void testTypeValidation() { @@ -410,6 +420,76 @@ public void testReloadSearchAnalyzers() throws IOException { mapperService.fieldType("otherField").getTextSearchInfo().getSearchQuoteAnalyzer())); } + public void testNewFieldTypeLookupEmpty() throws IOException { + MapperService mapperService = createIndex("test1", Settings.EMPTY, "_doc", "f1", "type=long").mapperService() + .forSearch(randomBoolean() ? null : new HashMap<>()); + assertThat(mapperService.fieldType("f1"), notNullValue()); + assertThat(mapperService.fieldType("f2"), nullValue()); + assertThat(mapperService.simpleMatchToFullName("f*"), equalTo(Set.of("f1"))); + } + + public void testNewFieldTypeLookupInvalidRuntimeMappings() throws IOException { + MapperService mapperService = createIndex("test1", Settings.EMPTY, "_doc", "f1", "type=long").mapperService(); + Exception e = expectThrows(IllegalArgumentException.class, () -> mapperService.forSearch( + new TreeMap<>(Map.of("st1", new TreeMap<>(Map.of("type", "long")))) + )); + assertThat(e.getMessage(), equalTo("[long] are not supported in runtime mappings")); + } + + public void testNewFieldTypeLookupNoType() throws IOException { + MapperService mapperService = createIndex("test1", Settings.EMPTY, "_doc", "f1", "type=long").mapperService(); + Exception e = expectThrows(IllegalArgumentException.class, () -> mapperService.forSearch( + new TreeMap<>(Map.of("r1", new TreeMap<>(Map.of("stuff", "foo")))) + )); + assertThat(e.getMessage(), equalTo("[type] is required for runtime mapping [r1]")); + } + + public void testNewFieldTypeLookupUnknownType() throws IOException { + MapperService mapperService = createIndex("test1", Settings.EMPTY, "_doc", "f1", "type=long").mapperService(); + Exception e = expectThrows(IllegalArgumentException.class, () -> mapperService.forSearch( + new TreeMap<>(Map.of("r1", new TreeMap<>(Map.of("type", "asdf")))) + )); + assertThat(e.getMessage(), equalTo("[asdf] is unknown type for runtime mapping [r1]")); + } + + public void testNewFieldTypeLookupAliases() throws IOException { + MapperService mapperService = createIndex("test1", Settings.EMPTY, "_doc", "f1", "type=long").mapperService(); + Exception e = expectThrows(IllegalArgumentException.class, () -> mapperService.forSearch( + new TreeMap<>(Map.of("r1", new TreeMap<>(Map.of("type", "alias", "path", "f1")))) + )); + assertThat(e.getMessage(), equalTo("aliases are not supported in runtime mappings")); + } + + public void testNewFieldTypeLookupShadowingRuntimeMappings() throws IOException { + MapperService mapperService = createIndex("test1", Settings.EMPTY, "_doc", "field", "type=long").mapperService(); + Exception e = expectThrows(IllegalArgumentException.class, () -> mapperService.forSearch( + new TreeMap<>(Map.of("field", new TreeMap<>(Map.of("type", "test_runtime")))) + )); + assertThat(e.getMessage(), equalTo("[field] can't be defined in the search's runtime mappings and the index's mappings")); + } + + public void testNewFieldTypeLookupValidRuntimeMappings() throws IOException { + MapperService mapperService = createIndex("test1", Settings.EMPTY, "_doc", "f1", "type=long").mapperService().forSearch( + new TreeMap<>(Map.of("r1", new TreeMap<>(Map.of("type", "test_runtime")))) + ); + assertThat(mapperService.fieldType("f1"), notNullValue()); + assertThat(mapperService.fieldType("f2"), nullValue()); + assertThat(mapperService.fieldType("r1"), notNullValue()); + assertThat(mapperService.fieldType("r2"), nullValue()); + assertThat(mapperService.simpleMatchToFullName("*1"), equalTo(Set.of("f1", "r1"))); + } + + public void testNewFieldTypeLookupRuntimeMappingsInObject() throws IOException { + Map inner = new TreeMap<>(Map.of("inner", new TreeMap<>(Map.of("type", "test_runtime")))); + MapperService mapperService = createIndex("test1", Settings.EMPTY, "_doc", "f1", "type=long").mapperService() + .forSearch(new TreeMap<>(Map.of("r1", new TreeMap<>(Map.of("type", "object", "properties", inner))))); + assertThat(mapperService.fieldType("f1"), notNullValue()); + assertThat(mapperService.fieldType("f2"), nullValue()); + assertThat(mapperService.fieldType("r1.inner"), notNullValue()); + assertThat(mapperService.fieldType("r2"), nullValue()); + assertThat(mapperService.simpleMatchToFullName("*1*"), equalTo(Set.of("f1", "r1.inner"))); + } + private boolean assertSameContainedFilters(TokenFilterFactory[] originalTokenFilter, NamedAnalyzer updatedAnalyzer) { ReloadableCustomAnalyzer updatedReloadableAnalyzer = (ReloadableCustomAnalyzer) updatedAnalyzer.analyzer(); TokenFilterFactory[] newTokenFilters = updatedReloadableAnalyzer.getComponents().getTokenFilters(); @@ -460,4 +540,27 @@ public AnalysisMode getAnalysisMode() { } } + /** + * Creates a totally broken runtime field that returns a {@code long}'s + * {@link MappedFieldType} and doesn't crash. Searches would never work + * with it though. + */ + public static final class TestRuntimeFieldPlugin extends Plugin implements MapperPlugin { + @Override + @SuppressWarnings("rawtypes") + public Map getMappers() { + return Map.of("test_runtime", (name, node, parserContext) -> { + Mapper.Builder delegate = new NumberFieldMapper.TypeParser(NumberType.LONG).parse(name, node, parserContext); + return new Mapper.Builder(name) { + @Override + public Mapper build(BuilderContext context) { + FieldMapper longMapper = spy((FieldMapper) delegate.build(context)); + doReturn(true).when(longMapper).isRuntimeField(); + return longMapper; + } + }; + }); + } + } + } diff --git a/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java b/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java index 5be92101e6360..f9d62cf6ac46c 100644 --- a/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java +++ b/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java @@ -62,6 +62,8 @@ import java.util.UUID; import static org.hamcrest.Matchers.equalTo; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyInt; import static org.mockito.Matchers.anyObject; import static org.mockito.Matchers.anyString; import static org.mockito.Matchers.eq; @@ -103,6 +105,8 @@ public void testPreProcess() throws Exception { MapperService mapperService = mock(MapperService.class); when(mapperService.hasNested()).thenReturn(randomBoolean()); when(indexService.mapperService()).thenReturn(mapperService); + when(indexService.newQueryShardContext(anyInt(), any(), any(), any(), any())).thenReturn(queryShardContext); + when(queryShardContext.getMapperService()).thenReturn(mapperService); IndexMetadata indexMetadata = IndexMetadata.builder("index").settings(settings).build(); IndexSettings indexSettings = new IndexSettings(indexMetadata, Settings.EMPTY); diff --git a/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java b/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java index 43e282af5f227..91e7acb9c8f8d 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java @@ -52,6 +52,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.function.Supplier; import static java.util.Collections.emptyMap; @@ -363,6 +364,9 @@ public static SearchSourceBuilder randomSearchSourceBuilder( if (randomBoolean()) { builder.collapse(randomCollapseBuilder.get()); } + if (randomBoolean()) { + builder.runtimeMappings(Map.of("foo", Map.of("bar", "baz"))); + } return builder; } } diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptFieldMapper.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptFieldMapper.java index 79d918776d7fe..27984b4ca8851 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptFieldMapper.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptFieldMapper.java @@ -62,6 +62,11 @@ protected String contentType() { return CONTENT_TYPE; } + @Override + public boolean isRuntimeField() { + return true; + } + public static class Builder extends ParametrizedFieldMapper.Builder { static final Map> FIELD_TYPE_RESOLVER = Map.of( diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/10_keyword.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/10_keyword.yml index 3d9a86d699e05..29451959aa89c 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/10_keyword.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/10_keyword.yml @@ -262,3 +262,47 @@ setup: day_of_week: M*ay - match: {hits.total.value: 1} - match: {hits.hits.0._source.voltage: 5.8} + +--- +"runtime defined docvalue_fields": + - do: + search: + index: sensor + body: + runtime_mappings: + first_letter: + type: script + runtime_type: keyword + script: | + for (String node : doc['day_of_week']) { + value(node.charAt(0).toString()); + } + sort: timestamp + docvalue_fields: [first_letter] + - match: {hits.total.value: 6} + - match: {hits.hits.0.fields.first_letter: [T] } + - match: {hits.hits.1.fields.first_letter: [F] } + - match: {hits.hits.2.fields.first_letter: [S] } + - match: {hits.hits.3.fields.first_letter: [S] } + - match: {hits.hits.4.fields.first_letter: [M] } + - match: {hits.hits.5.fields.first_letter: [T] } + +--- +"runtime defined query": + - do: + search: + index: sensor + body: + runtime_mappings: + first_letter: + type: script + runtime_type: keyword + script: | + for (String node : doc['day_of_week']) { + value(node.charAt(0).toString()); + } + query: + term: + first_letter: M + - match: {hits.total.value: 1} + - match: {hits.hits.0._source.voltage: 5.8}