From 42ba204323d82f94f615a78565c7e70267217e13 Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Thu, 12 Nov 2020 15:56:04 +0100 Subject: [PATCH 1/7] Introduce runtime section in mappings (#62906) The runtime section is at the same level as the existing properties section. Its purpose is to hold runtime fields only. With the introduction of the runtime section, a runtime field can be defined by specifying its type (previously called runtime_type) and script. ``` PUT /my-index/_mappings { "runtime" : { "day_of_week" : { "type" : "keyword", "script" : { "source" : "emit(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))" } } }, "properties" : { "timestamp" : { "type" : "date" } } } ``` Fields defined in the runtime section can be updated at any time as they are not present in the lucene index. They get replaced entirely when they get updated. Thanks to the introduction of the runtime section, runtime fields override existing mapped fields defined with the same name, similarly to runtime fields defined in the search request. Relates to #59332 --- .../TransportGetFieldMappingsIndexAction.java | 2 + .../index/mapper/FieldMapper.java | 22 +- .../index/mapper/FieldTypeLookup.java | 9 +- .../elasticsearch/index/mapper/Mapper.java | 10 +- .../index/mapper/MapperService.java | 4 +- .../index/mapper/MappingLookup.java | 18 +- .../index/mapper/RootObjectMapper.java | 54 ++- .../index/mapper/RuntimeFieldType.java | 96 ++++ .../index/query/QueryShardContext.java | 42 +- .../elasticsearch/indices/IndicesModule.java | 19 +- .../indices/mapper/MapperRegistry.java | 9 +- .../elasticsearch/plugins/MapperPlugin.java | 5 + .../search/builder/SearchSourceBuilder.java | 2 +- .../MetadataRolloverServiceTests.java | 2 +- .../MetadataIndexUpgradeServiceTests.java | 2 +- .../index/IndexSortSettingsTests.java | 61 +++ .../elasticsearch/index/codec/CodecTests.java | 3 +- .../mapper/DocumentFieldMapperTests.java | 1 + .../FieldAliasMapperValidationTests.java | 24 +- .../index/mapper/FieldTypeLookupTests.java | 122 ++++- .../index/mapper/MappingLookupTests.java | 72 +++ .../index/mapper/ParametrizedMapperTests.java | 2 +- .../index/mapper/RootObjectMapperTests.java | 330 ++++++++++---- .../index/mapper/TestRuntimeField.java | 52 +++ .../index/mapper/TypeParsersTests.java | 4 +- .../index/query/QueryShardContextTests.java | 329 +++++++------- .../indices/IndicesModuleTests.java | 14 + .../index/mapper/MapperServiceTestCase.java | 27 +- .../index/mapper/MockFieldMapper.java | 6 - .../aggregations/AggregatorTestCase.java | 2 +- .../mapper/FlattenedFieldLookupTests.java | 10 +- .../ml/integration/DatafeedJobsRestIT.java | 11 +- x-pack/plugin/runtime-fields/qa/build.gradle | 6 +- .../mapped/CoreWithMappedRuntimeFieldsIT.java | 18 +- .../CoreTestsWithSearchRuntimeFieldsIT.java | 29 +- .../test/CoreTestTranslater.java | 45 +- .../xpack/security/PermissionsIT.java | 27 +- .../xpack/runtimefields/RuntimeFields.java | 29 +- .../mapper/AbstractScriptFieldType.java | 159 +++++-- .../mapper/BooleanScriptFieldType.java | 31 +- .../mapper/DateScriptFieldType.java | 75 +++- .../mapper/DoubleScriptFieldType.java | 31 +- .../mapper/GeoPointScriptFieldType.java | 30 +- .../mapper/IpScriptFieldType.java | 37 +- .../mapper/KeywordScriptFieldType.java | 29 +- .../mapper/LongScriptFieldType.java | 31 +- .../mapper/RuntimeFieldMapper.java | 276 ------------ ...bstractNonTextScriptFieldTypeTestCase.java | 18 +- .../AbstractScriptFieldTypeTestCase.java | 194 +++++++- .../mapper/BooleanScriptFieldTypeTests.java | 5 +- .../mapper/DateScriptFieldTypeTests.java | 43 +- .../mapper/DoubleScriptFieldTypeTests.java | 5 +- .../mapper/GeoPointScriptFieldTypeTests.java | 10 +- .../mapper/IpScriptFieldTypeTests.java | 9 +- .../mapper/KeywordScriptFieldTypeTests.java | 5 +- .../mapper/LongScriptFieldTypeTests.java | 5 +- .../mapper/RuntimeFieldMapperTests.java | 417 ------------------ .../test/runtime_fields/100_geo_point.yml | 27 +- .../test/runtime_fields/10_keyword.yml | 89 ++-- .../test/runtime_fields/20_long.yml | 55 ++- .../test/runtime_fields/30_double.yml | 37 +- .../test/runtime_fields/40_date.yml | 52 +-- .../test/runtime_fields/50_ip.yml | 27 +- .../test/runtime_fields/60_boolean.yml | 37 +- .../runtime_fields/80_multiple_indices.yml | 34 +- .../test/runtime_fields/90_loops.yml | 56 +-- 66 files changed, 1855 insertions(+), 1489 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/index/mapper/RuntimeFieldType.java create mode 100644 server/src/test/java/org/elasticsearch/index/mapper/MappingLookupTests.java create mode 100644 server/src/test/java/org/elasticsearch/index/mapper/TestRuntimeField.java delete mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeFieldMapper.java delete mode 100644 x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeFieldMapperTests.java diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java index aa28857e144ee..269749c8d8477 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java @@ -173,6 +173,8 @@ public Boolean paramAsBoolean(String key, Boolean defaultValue) { private static Map findFieldMappingsByType(Predicate fieldPredicate, DocumentMapper documentMapper, GetFieldMappingsIndexRequest request) { + //TODO the logic here needs to be reworked to also include runtime fields. Though matching is against mappers rather + // than field types, and runtime fields are mixed with ordinary fields in FieldTypeLookup Map fieldMappings = new HashMap<>(); final MappingLookup mappingLookup = documentMapper.mappers(); for (String field : request.fields()) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java index ae5b8a942d5c0..7a21de9e90a88 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java @@ -279,15 +279,7 @@ public final FieldMapper merge(Mapper mergeWith) { Conflicts conflicts = new Conflicts(name()); builder.merge((FieldMapper) mergeWith, conflicts); conflicts.check(); - return builder.build(parentPath(name())); - } - - private static ContentPath parentPath(String name) { - int endPos = name.lastIndexOf("."); - if (endPos == -1) { - return new ContentPath(0); - } - return new ContentPath(name.substring(0, endPos)); + return builder.build(Builder.parentPath(name())); } protected void checkIncomingMergeType(FieldMapper mergeWith) { @@ -483,7 +475,7 @@ public List copyToFields() { /** * Serializes a parameter */ - protected interface Serializer { + public interface Serializer { void serialize(XContentBuilder builder, String name, T value) throws IOException; } @@ -936,7 +928,7 @@ protected String buildFullName(ContentPath contentPath) { /** * Writes the current builder parameter values as XContent */ - protected final void toXContent(XContentBuilder builder, boolean includeDefaults) throws IOException { + public final void toXContent(XContentBuilder builder, boolean includeDefaults) throws IOException { for (Parameter parameter : getParameters()) { parameter.toXContent(builder, includeDefaults); } @@ -1010,6 +1002,14 @@ public final void parse(String name, ParserContext parserContext, Map fullNameToFieldType = new HashMap<>(); /** @@ -47,7 +46,8 @@ final class FieldTypeLookup { private final DynamicKeyFieldTypeLookup dynamicKeyLookup; FieldTypeLookup(Collection fieldMappers, - Collection fieldAliasMappers) { + Collection fieldAliasMappers, + Collection runtimeFieldTypes) { Map dynamicKeyMappers = new HashMap<>(); for (FieldMapper fieldMapper : fieldMappers) { @@ -77,6 +77,11 @@ final class FieldTypeLookup { fullNameToFieldType.put(aliasName, fullNameToFieldType.get(path)); } + for (RuntimeFieldType runtimeFieldType : runtimeFieldTypes) { + //this will override concrete fields with runtime fields that have the same name + fullNameToFieldType.put(runtimeFieldType.name(), runtimeFieldType); + } + this.dynamicKeyLookup = new DynamicKeyFieldTypeLookup(dynamicKeyMappers, aliasToConcreteName); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java b/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java index d7cfe22950ff9..2bae141fc9d12 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java @@ -59,6 +59,7 @@ class ParserContext { private final Function similarityLookupService; private final Function typeParsers; + private final Function runtimeTypeParsers; private final Version indexVersionCreated; private final Supplier queryShardContextSupplier; private final DateFormatter dateFormatter; @@ -69,6 +70,7 @@ class ParserContext { public ParserContext(Function similarityLookupService, Function typeParsers, + Function runtimeTypeParsers, Version indexVersionCreated, Supplier queryShardContextSupplier, DateFormatter dateFormatter, @@ -78,6 +80,7 @@ public ParserContext(Function similarityLookupServic BooleanSupplier idFieldDataEnabled) { this.similarityLookupService = similarityLookupService; this.typeParsers = typeParsers; + this.runtimeTypeParsers = runtimeTypeParsers; this.indexVersionCreated = indexVersionCreated; this.queryShardContextSupplier = queryShardContextSupplier; this.dateFormatter = dateFormatter; @@ -132,6 +135,8 @@ public DateFormatter getDateFormatter() { protected Function typeParsers() { return typeParsers; } + protected Function runtimeTypeParsers() { return runtimeTypeParsers; } + protected Function similarityLookupService() { return similarityLookupService; } /** @@ -147,8 +152,9 @@ public ParserContext createMultiFieldContext(ParserContext in) { static class MultiFieldParserContext extends ParserContext { MultiFieldParserContext(ParserContext in) { - super(in.similarityLookupService, in.typeParsers, in.indexVersionCreated, in.queryShardContextSupplier, - in.dateFormatter, in.scriptService, in.indexAnalyzers, in.indexSettings, in.idFieldDataEnabled); + super(in.similarityLookupService, in.typeParsers, in.runtimeTypeParsers, in.indexVersionCreated, + in.queryShardContextSupplier, in.dateFormatter, in.scriptService, in.indexAnalyzers, in.indexSettings, + in.idFieldDataEnabled); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java index 8aab2f4d57de8..eba38b31c00e3 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -149,8 +149,8 @@ public MapperService(IndexSettings indexSettings, IndexAnalyzers indexAnalyzers, this.mapperRegistry = mapperRegistry; Function parserContextFunction = dateFormatter -> new Mapper.TypeParser.ParserContext(similarityService::getSimilarity, mapperRegistry.getMapperParsers()::get, - indexVersionCreated, queryShardContextSupplier, dateFormatter, scriptService, indexAnalyzers, indexSettings, - idFieldDataEnabled); + mapperRegistry.getRuntimeFieldTypeParsers()::get, indexVersionCreated, queryShardContextSupplier, dateFormatter, + scriptService, indexAnalyzers, indexSettings, idFieldDataEnabled); this.documentParser = new DocumentParser(xContentRegistry, parserContextFunction); Map metadataMapperParsers = mapperRegistry.getMetadataMapperParsers(indexSettings.getIndexVersionCreated()); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java b/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java index 481c5497c88e1..7226b06a28661 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java @@ -32,7 +32,6 @@ import java.util.stream.Stream; public final class MappingLookup { - /** Full field name to mapper */ private final Map fieldMappers; private final Map objectMappers; @@ -50,24 +49,24 @@ public static MappingLookup fromMapping(Mapping mapping) { newFieldMappers.add(metadataMapper); } } - collect(mapping.root, newObjectMappers, newFieldMappers, newFieldAliasMappers); - return new MappingLookup(newFieldMappers, newObjectMappers, newFieldAliasMappers, mapping.metadataMappers.length); + for (Mapper child : mapping.root) { + collect(child, newObjectMappers, newFieldMappers, newFieldAliasMappers); + } + return new MappingLookup(newFieldMappers, newObjectMappers, newFieldAliasMappers, + mapping.root.runtimeFieldTypes(), mapping.metadataMappers.length); } private static void collect(Mapper mapper, Collection objectMappers, Collection fieldMappers, Collection fieldAliasMappers) { - if (mapper instanceof RootObjectMapper) { - // root mapper isn't really an object mapper - } else if (mapper instanceof ObjectMapper) { + if (mapper instanceof ObjectMapper) { objectMappers.add((ObjectMapper)mapper); } else if (mapper instanceof FieldMapper) { fieldMappers.add((FieldMapper)mapper); } else if (mapper instanceof FieldAliasMapper) { fieldAliasMappers.add((FieldAliasMapper) mapper); } else { - throw new IllegalStateException("Unrecognized mapper type [" + - mapper.getClass().getSimpleName() + "]."); + throw new IllegalStateException("Unrecognized mapper type [" + mapper.getClass().getSimpleName() + "]."); } for (Mapper child : mapper) { @@ -78,6 +77,7 @@ private static void collect(Mapper mapper, Collection objectMapper public MappingLookup(Collection mappers, Collection objectMappers, Collection aliasMappers, + Collection runtimeFieldTypes, int metadataFieldCount) { Map fieldMappers = new HashMap<>(); Map indexAnalyzers = new HashMap<>(); @@ -114,7 +114,7 @@ public MappingLookup(Collection mappers, } } - this.fieldTypeLookup = new FieldTypeLookup(mappers, aliasMappers); + this.fieldTypeLookup = new FieldTypeLookup(mappers, aliasMappers, runtimeFieldTypes); this.fieldMappers = Collections.unmodifiableMap(fieldMappers); this.indexAnalyzer = new FieldNameAnalyzer(indexAnalyzers); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java index 7fb307dfe2236..2b3f79e42be43 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java @@ -19,6 +19,7 @@ package org.elasticsearch.index.mapper; +import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.Version; import org.elasticsearch.common.Explicit; import org.elasticsearch.common.Strings; @@ -33,11 +34,14 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.stream.Collectors; import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeBooleanValue; import static org.elasticsearch.index.mapper.TypeParsers.parseDateTimeFormatter; @@ -61,6 +65,7 @@ public static class Builder extends ObjectMapper.Builder { protected Explicit dynamicDateTimeFormatters = new Explicit<>(Defaults.DYNAMIC_DATE_TIME_FORMATTERS, false); protected Explicit dateDetection = new Explicit<>(Defaults.DATE_DETECTION, false); protected Explicit numericDetection = new Explicit<>(Defaults.NUMERIC_DETECTION, false); + protected final Map runtimeFieldTypes = new HashMap<>(); public Builder(String name, Version indexCreatedVersion) { super(name, indexCreatedVersion); @@ -82,6 +87,11 @@ public RootObjectMapper.Builder add(Mapper.Builder builder) { return this; } + public RootObjectMapper.Builder addRuntime(RuntimeFieldType runtimeFieldType) { + this.runtimeFieldTypes.put(runtimeFieldType.name(), runtimeFieldType); + return this; + } + @Override public RootObjectMapper build(ContentPath contentPath) { return (RootObjectMapper) super.build(contentPath); @@ -91,7 +101,7 @@ public RootObjectMapper build(ContentPath contentPath) { protected ObjectMapper createMapper(String name, String fullPath, Explicit enabled, Nested nested, Dynamic dynamic, Map mappers, Version indexCreatedVersion) { assert !nested.isNested(); - return new RootObjectMapper(name, enabled, dynamic, mappers, + return new RootObjectMapper(name, enabled, dynamic, mappers, runtimeFieldTypes, dynamicDateTimeFormatters, dynamicTemplates, dateDetection, numericDetection, indexCreatedVersion); @@ -126,7 +136,7 @@ private static void fixRedundantIncludes(ObjectMapper objectMapper, boolean pare } } - public static class TypeParser extends ObjectMapper.TypeParser { + static final class TypeParser extends ObjectMapper.TypeParser { @Override public Mapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { @@ -144,8 +154,8 @@ public Mapper.Builder parse(String name, Map node, ParserContext return builder; } - protected boolean processField(RootObjectMapper.Builder builder, String fieldName, Object fieldNode, - ParserContext parserContext) { + @SuppressWarnings("unchecked") + private boolean processField(RootObjectMapper.Builder builder, String fieldName, Object fieldNode, ParserContext parserContext) { if (fieldName.equals("date_formats") || fieldName.equals("dynamic_date_formats")) { if (fieldNode instanceof List) { List formatters = new ArrayList<>(); @@ -199,6 +209,13 @@ protected boolean processField(RootObjectMapper.Builder builder, String fieldNam } else if (fieldName.equals("numeric_detection")) { builder.numericDetection = new Explicit<>(nodeBooleanValue(fieldNode, "numeric_detection"), true); return true; + } else if (fieldName.equals("runtime")) { + if (fieldNode instanceof Map) { + RuntimeFieldType.parseRuntimeFields((Map) fieldNode, parserContext, builder::addRuntime); + return true; + } else { + throw new ElasticsearchParseException("runtime must be a map type"); + } } return false; } @@ -208,11 +225,14 @@ protected boolean processField(RootObjectMapper.Builder builder, String fieldNam private Explicit dateDetection; private Explicit numericDetection; private Explicit dynamicTemplates; + private final Map runtimeFieldTypes; RootObjectMapper(String name, Explicit enabled, Dynamic dynamic, Map mappers, + Map runtimeFieldTypes, Explicit dynamicDateTimeFormatters, Explicit dynamicTemplates, Explicit dateDetection, Explicit numericDetection, Version indexCreatedVersion) { super(name, name, enabled, Nested.NO, dynamic, mappers, indexCreatedVersion); + this.runtimeFieldTypes = runtimeFieldTypes; this.dynamicTemplates = dynamicTemplates; this.dynamicDateTimeFormatters = dynamicDateTimeFormatters; this.dateDetection = dateDetection; @@ -232,23 +252,26 @@ public ObjectMapper mappingUpdate(Mapper mapper) { return update; } - public boolean dateDetection() { + boolean dateDetection() { return this.dateDetection.value(); } - public boolean numericDetection() { + boolean numericDetection() { return this.numericDetection.value(); } - public DateFormatter[] dynamicDateTimeFormatters() { + DateFormatter[] dynamicDateTimeFormatters() { return dynamicDateTimeFormatters.value(); } - public DynamicTemplate[] dynamicTemplates() { + DynamicTemplate[] dynamicTemplates() { return dynamicTemplates.value(); } - @SuppressWarnings("rawtypes") + Collection runtimeFieldTypes() { + return runtimeFieldTypes.values(); + } + public Mapper.Builder findTemplateBuilder(ParseContext context, String name, XContentFieldType matchType) { return findTemplateBuilder(context, name, matchType, null); } @@ -264,7 +287,6 @@ public Mapper.Builder findTemplateBuilder(ParseContext context, String name, Dat * @param dateFormat a dateformatter to use if the type is a date, null if not a date or is using the default format * @return a mapper builder, or null if there is no template for such a field */ - @SuppressWarnings("rawtypes") private Mapper.Builder findTemplateBuilder(ParseContext context, String name, XContentFieldType matchType, DateFormatter dateFormat) { DynamicTemplate dynamicTemplate = findTemplate(context.path(), name, matchType); if (dynamicTemplate == null) { @@ -327,6 +349,8 @@ protected void doMerge(ObjectMapper mergeWith, MergeReason reason) { this.dynamicTemplates = mergeWithObject.dynamicTemplates; } } + + this.runtimeFieldTypes.putAll(mergeWithObject.runtimeFieldTypes); } @Override @@ -357,6 +381,16 @@ protected void doXContent(XContentBuilder builder, ToXContent.Params params) thr if (numericDetection.explicit() || includeDefaults) { builder.field("numeric_detection", numericDetection.value()); } + + if (runtimeFieldTypes.size() > 0) { + builder.startObject("runtime"); + List sortedRuntimeFieldTypes = runtimeFieldTypes.values().stream().sorted( + Comparator.comparing(RuntimeFieldType::name)).collect(Collectors.toList()); + for (RuntimeFieldType fieldType : sortedRuntimeFieldTypes) { + fieldType.toXContent(builder, params); + } + builder.endObject(); + } } private static void validateDynamicTemplate(Mapper.TypeParser.ParserContext parserContext, diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RuntimeFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/RuntimeFieldType.java new file mode 100644 index 0000000000000..75fd74532a15b --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/mapper/RuntimeFieldType.java @@ -0,0 +1,96 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.mapper; + +import org.elasticsearch.common.xcontent.ToXContentFragment; +import org.elasticsearch.common.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.function.Consumer; + +/** + * Base implementation for a runtime field that can be defined as part of the runtime section of the index mappings + */ +public abstract class RuntimeFieldType extends MappedFieldType implements ToXContentFragment { + + protected RuntimeFieldType(String name, Map meta) { + super(name, false, false, false, TextSearchInfo.SIMPLE_MATCH_WITHOUT_TERMS, meta); + } + + @Override + public final XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(name()); + builder.field("type", typeName()); + boolean includeDefaults = params.paramAsBoolean("include_defaults", false); + doXContentBody(builder, includeDefaults); + builder.endObject(); + return builder; + } + + /** + * Prints out the parameters that subclasses expose + */ + protected abstract void doXContentBody(XContentBuilder builder, boolean includeDefaults) throws IOException; + + /** + * Parser for a runtime field. Creates the appropriate {@link RuntimeFieldType} for a runtime field, + * as defined in the runtime section of the index mappings. + */ + public interface Parser { + RuntimeFieldType parse(String name, Map node, Mapper.TypeParser.ParserContext parserContext) + throws MapperParsingException; + } + + public static void parseRuntimeFields(Map node, + Mapper.TypeParser.ParserContext parserContext, + Consumer runtimeFieldTypeConsumer) { + Iterator> iterator = node.entrySet().iterator(); + while (iterator.hasNext()) { + Map.Entry entry = iterator.next(); + String fieldName = entry.getKey(); + if (entry.getValue() instanceof Map) { + @SuppressWarnings("unchecked") + Map propNode = new HashMap<>(((Map) entry.getValue())); + Object typeNode = propNode.get("type"); + String type; + if (typeNode == null) { + throw new MapperParsingException("No type specified for runtime field [" + fieldName + "]"); + } else { + type = typeNode.toString(); + } + Parser typeParser = parserContext.runtimeTypeParsers().apply(type); + if (typeParser == null) { + throw new MapperParsingException("No handler for type [" + type + + "] declared on runtime field [" + fieldName + "]"); + } + runtimeFieldTypeConsumer.accept(typeParser.parse(fieldName, propNode, parserContext)); + propNode.remove("type"); + DocumentMapperParser.checkNoRemainingFields(fieldName, propNode, parserContext.indexVersionCreated()); + iterator.remove(); + } else { + throw new MapperParsingException("Expected map for runtime field [" + fieldName + "] definition but got a " + + fieldName.getClass().getName()); + } + } + } +} diff --git a/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java b/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java index ee380cc7868a2..243be3f8ca0a3 100644 --- a/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java @@ -26,7 +26,6 @@ import org.apache.lucene.search.join.BitSetProducer; import org.apache.lucene.search.similarities.Similarity; import org.apache.lucene.util.SetOnce; -import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.client.Client; @@ -57,6 +56,7 @@ import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.ObjectMapper; import org.elasticsearch.index.mapper.ParsedDocument; +import org.elasticsearch.index.mapper.RuntimeFieldType; import org.elasticsearch.index.mapper.SourceToParse; import org.elasticsearch.index.mapper.TextFieldMapper; import org.elasticsearch.index.query.support.NestedScope; @@ -195,7 +195,7 @@ public QueryShardContext( ), allowExpensiveQueries, valuesSourceRegistry, - parseRuntimeMappings(runtimeMappings, mapperService, indexSettings) + parseRuntimeMappings(runtimeMappings, mapperService) ); } @@ -396,21 +396,12 @@ public MappedFieldType buildAnonymousFieldType(String type) { "[unmapped_type:string] should be replaced with [unmapped_type:keyword]"); type = "keyword"; } - return buildFieldType(type, "__anonymous_" + type, Collections.emptyMap(), mapperService.parserContext(), indexSettings); - } - - private static MappedFieldType buildFieldType( - String type, - String field, - Map node, - Mapper.TypeParser.ParserContext parserContext, - IndexSettings indexSettings - ) { + Mapper.TypeParser.ParserContext parserContext = mapperService.parserContext(); Mapper.TypeParser typeParser = parserContext.typeParser(type); if (typeParser == null) { throw new IllegalArgumentException("No mapper found for type [" + type + "]"); } - Mapper.Builder builder = typeParser.parse(field, node, parserContext); + Mapper.Builder builder = typeParser.parse("__anonymous_", Collections.emptyMap(), parserContext); Mapper mapper = builder.build(new ContentPath(0)); if (mapper instanceof FieldMapper) { return ((FieldMapper)mapper).fieldType(); @@ -674,25 +665,14 @@ public BigArrays bigArrays() { } private static Map parseRuntimeMappings( - Map mappings, - MapperService mapperService, - IndexSettings indexSettings + Map runtimeMappings, + MapperService mapperService ) { - Map runtimeMappings = new HashMap<>(); - for (Map.Entry entry : mappings.entrySet()) { - String field = entry.getKey(); - if (entry.getValue() instanceof Map == false) { - throw new ElasticsearchParseException("runtime mappings must be a map type"); - } - @SuppressWarnings("unchecked") - Map node = new HashMap<>((Map) entry.getValue()); - // Replace the type until we have native support for the runtime section - Object oldRuntimeType = node.put("runtime_type", node.remove("type")); - if (oldRuntimeType != null) { - throw new ElasticsearchParseException("use [type] in [runtime_mappings] instead of [runtime_type]"); - } - runtimeMappings.put(field, buildFieldType("runtime", field, node, mapperService.parserContext(), indexSettings)); + Map runtimeFieldTypes = new HashMap<>(); + if (runtimeMappings.isEmpty() == false) { + RuntimeFieldType.parseRuntimeFields(new HashMap<>(runtimeMappings), mapperService.parserContext(), + runtimeFieldType -> runtimeFieldTypes.put(runtimeFieldType.name(), runtimeFieldType)); } - return runtimeMappings; + return Collections.unmodifiableMap(runtimeFieldTypes); } } diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java index 57a8f1383329c..53ba191f3ffff 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java @@ -48,6 +48,7 @@ import org.elasticsearch.index.mapper.ObjectMapper; import org.elasticsearch.index.mapper.RangeType; import org.elasticsearch.index.mapper.RoutingFieldMapper; +import org.elasticsearch.index.mapper.RuntimeFieldType; import org.elasticsearch.index.mapper.SeqNoFieldMapper; import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.index.mapper.TextFieldMapper; @@ -83,8 +84,8 @@ public class IndicesModule extends AbstractModule { private final MapperRegistry mapperRegistry; public IndicesModule(List mapperPlugins) { - this.mapperRegistry = new MapperRegistry(getMappers(mapperPlugins), getMetadataMappers(mapperPlugins), - getFieldFilter(mapperPlugins)); + this.mapperRegistry = new MapperRegistry(getMappers(mapperPlugins), getRuntimeFieldTypes(mapperPlugins), + getMetadataMappers(mapperPlugins), getFieldFilter(mapperPlugins)); registerBuiltinWritables(); } @@ -144,9 +145,21 @@ public static Map getMappers(List mappe return Collections.unmodifiableMap(mappers); } + private static Map getRuntimeFieldTypes(List mapperPlugins) { + Map runtimeParsers = new LinkedHashMap<>(); + for (MapperPlugin mapperPlugin : mapperPlugins) { + for (Map.Entry entry : mapperPlugin.getRuntimeFieldTypes().entrySet()) { + if (runtimeParsers.put(entry.getKey(), entry.getValue()) != null) { + throw new IllegalArgumentException("Runtime field type [" + entry.getKey() + "] is already registered"); + } + } + } + return Collections.unmodifiableMap(runtimeParsers); + } + private static final Map builtInMetadataMappers = initBuiltInMetadataMappers(); - private static Set builtInMetadataFields = Collections.unmodifiableSet(builtInMetadataMappers.keySet()); + private static final Set builtInMetadataFields = Collections.unmodifiableSet(builtInMetadataMappers.keySet()); private static Map initBuiltInMetadataMappers() { Map builtInMetadataMappers; diff --git a/server/src/main/java/org/elasticsearch/indices/mapper/MapperRegistry.java b/server/src/main/java/org/elasticsearch/indices/mapper/MapperRegistry.java index 5fa385da449ad..0c43b7139eeac 100644 --- a/server/src/main/java/org/elasticsearch/indices/mapper/MapperRegistry.java +++ b/server/src/main/java/org/elasticsearch/indices/mapper/MapperRegistry.java @@ -23,6 +23,7 @@ import org.elasticsearch.index.mapper.AllFieldMapper; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MetadataFieldMapper; +import org.elasticsearch.index.mapper.RuntimeFieldType; import org.elasticsearch.plugins.MapperPlugin; import java.util.Collections; @@ -37,14 +38,16 @@ public final class MapperRegistry { private final Map mapperParsers; + private final Map runtimeFieldTypeParsers; private final Map metadataMapperParsers; private final Map metadataMapperParsers6x; private final Function> fieldFilter; - public MapperRegistry(Map mapperParsers, + public MapperRegistry(Map mapperParsers, Map runtimeFieldTypeParsers, Map metadataMapperParsers, Function> fieldFilter) { this.mapperParsers = Collections.unmodifiableMap(new LinkedHashMap<>(mapperParsers)); + this.runtimeFieldTypeParsers = runtimeFieldTypeParsers; this.metadataMapperParsers = Collections.unmodifiableMap(new LinkedHashMap<>(metadataMapperParsers)); // add the _all field mapper for indices created in 6x Map metadata6x = new LinkedHashMap<>(); @@ -62,6 +65,10 @@ public Map getMapperParsers() { return mapperParsers; } + public Map getRuntimeFieldTypeParsers() { + return runtimeFieldTypeParsers; + } + /** * Return a map of the meta mappers that have been registered. The * returned map uses the name of the field as a key. diff --git a/server/src/main/java/org/elasticsearch/plugins/MapperPlugin.java b/server/src/main/java/org/elasticsearch/plugins/MapperPlugin.java index 5edf994b32ea4..8544c75a575ee 100644 --- a/server/src/main/java/org/elasticsearch/plugins/MapperPlugin.java +++ b/server/src/main/java/org/elasticsearch/plugins/MapperPlugin.java @@ -21,6 +21,7 @@ import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MetadataFieldMapper; +import org.elasticsearch.index.mapper.RuntimeFieldType; import java.util.Collections; import java.util.Map; @@ -43,6 +44,10 @@ default Map getMappers() { return Collections.emptyMap(); } + default Map getRuntimeFieldTypes() { + return Collections.emptyMap(); + } + /** * Returns additional metadata mapper implementations added by this plugin. *

diff --git a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java index 5f66552afdf19..9c5097b06743c 100644 --- a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java @@ -1030,7 +1030,7 @@ public SearchSourceBuilder pointInTimeBuilder(PointInTimeBuilder builder) { * Mappings specified on this search request that override built in mappings. */ public Map runtimeMappings() { - return runtimeMappings; + return Collections.unmodifiableMap(runtimeMappings); } /** diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java index 8a75d0d28dec2..e6da2484cf870 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java @@ -562,7 +562,7 @@ protected String contentType() { } }; MappingLookup mappingLookup = new MappingLookup(Arrays.asList(mockedTimestampField, dateFieldMapper), - Collections.emptyList(), Collections.emptyList(), 0); + Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), 0); ClusterService clusterService = ClusterServiceUtils.createClusterService(testThreadPool); Environment env = mock(Environment.class); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexUpgradeServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexUpgradeServiceTests.java index 48aac8a713d33..725d2ac71a1a9 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexUpgradeServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexUpgradeServiceTests.java @@ -153,7 +153,7 @@ private MetadataIndexUpgradeService getMetadataIndexUpgradeService() { return new MetadataIndexUpgradeService( Settings.EMPTY, xContentRegistry(), - new MapperRegistry(Collections.emptyMap(), Collections.emptyMap(), MapperPlugin.NOOP_FIELD_FILTER), + new MapperRegistry(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), MapperPlugin.NOOP_FIELD_FILTER), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, new SystemIndices(Collections.singletonMap("system-plugin", Collections.singletonList(new SystemIndexDescriptor(".system", "a system index")))), diff --git a/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java b/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java index ffc08f9e1d689..553349428f90a 100644 --- a/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java +++ b/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java @@ -19,15 +19,30 @@ package org.elasticsearch.index; +import org.apache.lucene.search.Query; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.IndexFieldDataService; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.RuntimeFieldType; +import org.elasticsearch.index.mapper.ValueFetcher; +import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; +import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache; import org.elasticsearch.search.MultiValueMode; +import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.ESTestCase; +import java.util.Collections; +import java.util.function.Supplier; + import static org.elasticsearch.common.settings.Settings.Builder.EMPTY_SETTINGS; import static org.elasticsearch.index.IndexSettingsTests.newIndexMeta; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; public class IndexSortSettingsTests extends ESTestCase { private static IndexSettings indexSettings(Settings settings) { @@ -129,4 +144,50 @@ public void testInvalidMissing() { assertThat(exc.getMessage(), containsString("Illegal missing value:[default]," + " must be one of [_last, _first]")); } + + public void testIndexSorting() { + IndexSettings indexSettings = indexSettings(Settings.builder().put("index.sort.field", "field").build()); + IndexSortConfig config = indexSettings.getIndexSortConfig(); + assertTrue(config.hasIndexSort()); + IndicesFieldDataCache cache = new IndicesFieldDataCache(Settings.EMPTY, null); + NoneCircuitBreakerService circuitBreakerService = new NoneCircuitBreakerService(); + final IndexFieldDataService indexFieldDataService = new IndexFieldDataService(indexSettings, cache, circuitBreakerService, null); + MappedFieldType fieldType = new RuntimeFieldType("field", Collections.emptyMap()) { + @Override + public ValueFetcher valueFetcher(QueryShardContext context, SearchLookup searchLookup, String format) { + throw new UnsupportedOperationException(); + } + + @Override + public String typeName() { + return null; + } + + @Override + public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { + searchLookup.get(); + return null; + } + + @Override + public Query termQuery(Object value, QueryShardContext context) { + throw new UnsupportedOperationException(); + } + + @Override + protected void doXContentBody(XContentBuilder builder, boolean includeDefaults) { + throw new UnsupportedOperationException(); + } + }; + IllegalArgumentException iae = expectThrows( + IllegalArgumentException.class, + () -> config.buildIndexSort( + field -> fieldType, + (ft, searchLookupSupplier) -> indexFieldDataService.getForField(ft, "index", searchLookupSupplier) + ) + ); + assertEquals("docvalues not found for index sort field:[field]", iae.getMessage()); + assertThat(iae.getCause(), instanceOf(UnsupportedOperationException.class)); + assertEquals("index sorting not supported on runtime field [field]", iae.getCause().getMessage()); + } } diff --git a/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java b/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java index aa86888cdefe9..f01bcc169b191 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java @@ -92,7 +92,8 @@ private CodecService createCodecService() throws IOException { IndexSettings settings = IndexSettingsModule.newIndexSettings("_na", nodeSettings); SimilarityService similarityService = new SimilarityService(settings, null, Collections.emptyMap()); IndexAnalyzers indexAnalyzers = createTestAnalysis(settings, nodeSettings).indexAnalyzers; - MapperRegistry mapperRegistry = new MapperRegistry(Collections.emptyMap(), Collections.emptyMap(), MapperPlugin.NOOP_FIELD_FILTER); + MapperRegistry mapperRegistry = new MapperRegistry(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), + MapperPlugin.NOOP_FIELD_FILTER); MapperService service = new MapperService(settings, indexAnalyzers, xContentRegistry(), similarityService, mapperRegistry, () -> null, () -> false, null); return new CodecService(service, LogManager.getLogger("test")); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java index 829550a5439e2..2b386472f8816 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java @@ -119,6 +119,7 @@ public void testAnalyzers() throws IOException { Arrays.asList(fieldMapper1, fieldMapper2), Collections.emptyList(), Collections.emptyList(), + Collections.emptyList(), 0); assertAnalyzes(mappingLookup.indexAnalyzer(), "field1", "index1"); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java b/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java index 975a22872dfb6..16d2d97bb5cee 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java @@ -39,7 +39,9 @@ public void testDuplicateFieldAliasAndObject() { new MappingLookup( Collections.emptyList(), singletonList(objectMapper), - singletonList(aliasMapper), 0)); + singletonList(aliasMapper), + emptyList(), + 0)); assertEquals("Alias [some.path] is defined both as an object and an alias", e.getMessage()); } @@ -52,7 +54,9 @@ public void testDuplicateFieldAliasAndConcreteField() { new MappingLookup( Arrays.asList(field, invalidField), emptyList(), - singletonList(invalidAlias), 0)); + singletonList(invalidAlias), + emptyList(), + 0)); assertEquals("Alias [invalid] is defined both as an alias and a concrete field", e.getMessage()); } @@ -65,7 +69,9 @@ public void testAliasThatRefersToAlias() { MappingLookup mappers = new MappingLookup( singletonList(field), emptyList(), - Arrays.asList(alias, invalidAlias), 0); + Arrays.asList(alias, invalidAlias), + emptyList(), + 0); alias.validate(mappers); MapperParsingException e = expectThrows(MapperParsingException.class, () -> invalidAlias.validate(mappers)); @@ -81,7 +87,9 @@ public void testAliasThatRefersToItself() { MappingLookup mappers = new MappingLookup( emptyList(), emptyList(), - singletonList(invalidAlias), 0); + singletonList(invalidAlias), + emptyList(), + 0); invalidAlias.validate(mappers); }); @@ -96,7 +104,9 @@ public void testAliasWithNonExistentPath() { MappingLookup mappers = new MappingLookup( emptyList(), emptyList(), - singletonList(invalidAlias), 0); + singletonList(invalidAlias), + emptyList(), + 0); invalidAlias.validate(mappers); }); @@ -112,6 +122,7 @@ public void testFieldAliasWithNestedScope() { singletonList(createFieldMapper("nested", "field")), singletonList(objectMapper), singletonList(aliasMapper), + emptyList(), 0); aliasMapper.validate(mappers); } @@ -124,6 +135,7 @@ public void testFieldAliasWithDifferentObjectScopes() { singletonList(createFieldMapper("object1", "field")), Arrays.asList(createObjectMapper("object1"), createObjectMapper("object2")), singletonList(aliasMapper), + emptyList(), 0); aliasMapper.validate(mappers); } @@ -137,6 +149,7 @@ public void testFieldAliasWithNestedTarget() { singletonList(createFieldMapper("nested", "field")), Collections.singletonList(objectMapper), singletonList(aliasMapper), + emptyList(), 0); aliasMapper.validate(mappers); }); @@ -155,6 +168,7 @@ public void testFieldAliasWithDifferentNestedScopes() { singletonList(createFieldMapper("nested1", "field")), Arrays.asList(createNestedObjectMapper("nested1"), createNestedObjectMapper("nested2")), singletonList(aliasMapper), + emptyList(), 0); aliasMapper.validate(mappers); }); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FieldTypeLookupTests.java b/server/src/test/java/org/elasticsearch/index/mapper/FieldTypeLookupTests.java index 5a8543324ea75..d0277665315f1 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/FieldTypeLookupTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/FieldTypeLookupTests.java @@ -34,7 +34,7 @@ public class FieldTypeLookupTests extends ESTestCase { public void testEmpty() { - FieldTypeLookup lookup = new FieldTypeLookup(Collections.emptyList(), Collections.emptyList()); + FieldTypeLookup lookup = new FieldTypeLookup(Collections.emptyList(), Collections.emptyList(), Collections.emptyList()); assertNull(lookup.get("foo")); Collection names = lookup.simpleMatchToFullName("foo"); assertNotNull(names); @@ -45,23 +45,37 @@ public void testEmpty() { public void testFilter() { Collection fieldMappers = List.of(new MockFieldMapper("field"), new MockFieldMapper("test")); Collection fieldAliases = singletonList(new FieldAliasMapper("alias", "alias", "test")); - FieldTypeLookup fieldTypeLookup = new FieldTypeLookup(fieldMappers, fieldAliases); - Iterable allFieldTypes = fieldTypeLookup.filter(ft -> true); - assertEquals(2, size(allFieldTypes)); - for (MappedFieldType allFieldType : allFieldTypes) { - assertThat(allFieldType, instanceOf(MockFieldMapper.FakeFieldType.class)); + Collection runtimeFields = List.of(new TestRuntimeField("runtime"), new TestRuntimeField("field")); + FieldTypeLookup fieldTypeLookup = new FieldTypeLookup(fieldMappers, fieldAliases, runtimeFields); + assertEquals(3, size(fieldTypeLookup.filter(ft -> true))); + for (MappedFieldType fieldType : fieldTypeLookup.filter(ft -> true)) { + if (fieldType.name().equals("test")) { + assertThat(fieldType, instanceOf(MockFieldMapper.FakeFieldType.class)); + } + if (fieldType.name().equals("field") || fieldType.name().equals("runtime")) { + assertThat(fieldType, instanceOf(TestRuntimeField.class)); + } } assertEquals(0, size(fieldTypeLookup.filter(ft -> false))); - Iterable fieldIterable = fieldTypeLookup.filter(ft -> ft.name().equals("field")); - assertEquals(1, size(fieldIterable)); - MappedFieldType field = fieldIterable.iterator().next(); - assertEquals("field", field.name()); - assertThat(field, instanceOf(MockFieldMapper.FakeFieldType.class)); + { + Iterable fieldIterable = fieldTypeLookup.filter(ft -> ft.name().equals("field")); + assertEquals(1, size(fieldIterable)); + MappedFieldType field = fieldIterable.iterator().next(); + assertEquals("field", field.name()); + assertThat(field, instanceOf(TestRuntimeField.class)); + } + { + Iterable fieldIterable = fieldTypeLookup.filter(ft -> ft.name().equals("test")); + assertEquals(1, size(fieldIterable)); + MappedFieldType field = fieldIterable.iterator().next(); + assertEquals("test", field.name()); + assertThat(field, instanceOf(MockFieldMapper.FakeFieldType.class)); + } } public void testAddNewField() { MockFieldMapper f = new MockFieldMapper("foo"); - FieldTypeLookup lookup = new FieldTypeLookup(Collections.singletonList(f), emptyList()); + FieldTypeLookup lookup = new FieldTypeLookup(Collections.singletonList(f), emptyList(), Collections.emptyList()); assertNull(lookup.get("bar")); assertEquals(f.fieldType(), lookup.get("foo")); assertEquals(1, size(lookup.filter(ft -> true))); @@ -71,7 +85,8 @@ public void testAddFieldAlias() { MockFieldMapper field = new MockFieldMapper("foo"); FieldAliasMapper alias = new FieldAliasMapper("alias", "alias", "foo"); - FieldTypeLookup lookup = new FieldTypeLookup(Collections.singletonList(field), Collections.singletonList(alias)); + FieldTypeLookup lookup = new FieldTypeLookup(Collections.singletonList(field), Collections.singletonList(alias), + Collections.emptyList()); MappedFieldType aliasType = lookup.get("alias"); assertEquals(field.fieldType(), aliasType); @@ -84,7 +99,7 @@ public void testSimpleMatchToFullName() { FieldAliasMapper alias1 = new FieldAliasMapper("food", "food", "foo"); FieldAliasMapper alias2 = new FieldAliasMapper("barometer", "barometer", "bar"); - FieldTypeLookup lookup = new FieldTypeLookup(Arrays.asList(field1, field2), Arrays.asList(alias1, alias2)); + FieldTypeLookup lookup = new FieldTypeLookup(Arrays.asList(field1, field2), Arrays.asList(alias1, alias2), Collections.emptyList()); Collection names = lookup.simpleMatchToFullName("b*"); @@ -101,7 +116,7 @@ public void testSourcePathWithMultiFields() { .addMultiField(new MockFieldMapper.Builder("field.subfield2")) .build(new ContentPath()); - FieldTypeLookup lookup = new FieldTypeLookup(singletonList(field), emptyList()); + FieldTypeLookup lookup = new FieldTypeLookup(singletonList(field), emptyList(), emptyList()); assertEquals(Set.of("field"), lookup.sourcePaths("field")); assertEquals(Set.of("field"), lookup.sourcePaths("field.subfield1")); @@ -117,13 +132,86 @@ public void testSourcePathsWithCopyTo() { .copyTo("field") .build(new ContentPath()); - FieldTypeLookup lookup = new FieldTypeLookup(Arrays.asList(field, otherField), emptyList()); + FieldTypeLookup lookup = new FieldTypeLookup(Arrays.asList(field, otherField), emptyList(), emptyList()); assertEquals(Set.of("other_field", "field"), lookup.sourcePaths("field")); assertEquals(Set.of("other_field", "field"), lookup.sourcePaths("field.subfield1")); } - private int size(Iterable iterable) { + public void testRuntimeFieldsLookup() { + MockFieldMapper concrete = new MockFieldMapper("concrete"); + TestRuntimeField runtime = new TestRuntimeField("runtime"); + + FieldTypeLookup fieldTypeLookup = new FieldTypeLookup(List.of(concrete), emptyList(), List.of(runtime)); + assertThat(fieldTypeLookup.get("concrete"), instanceOf(MockFieldMapper.FakeFieldType.class)); + assertThat(fieldTypeLookup.get("runtime"), instanceOf(TestRuntimeField.class)); + assertEquals(2, size(fieldTypeLookup.filter(ft -> true))); + } + + public void testRuntimeFieldOverrides() { + MockFieldMapper field = new MockFieldMapper("field"); + MockFieldMapper subfield = new MockFieldMapper("object.subfield"); + MockFieldMapper concrete = new MockFieldMapper("concrete"); + TestRuntimeField fieldOverride = new TestRuntimeField("field"); + TestRuntimeField subfieldOverride = new TestRuntimeField("object.subfield"); + TestRuntimeField runtime = new TestRuntimeField("runtime"); + + FieldTypeLookup fieldTypeLookup = new FieldTypeLookup(List.of(field, concrete, subfield), emptyList(), + List.of(fieldOverride, runtime, subfieldOverride)); + assertThat(fieldTypeLookup.get("field"), instanceOf(TestRuntimeField.class)); + assertThat(fieldTypeLookup.get("object.subfield"), instanceOf(TestRuntimeField.class)); + assertThat(fieldTypeLookup.get("concrete"), instanceOf(MockFieldMapper.FakeFieldType.class)); + assertThat(fieldTypeLookup.get("runtime"), instanceOf(TestRuntimeField.class)); + assertEquals(4, size(fieldTypeLookup.filter(ft -> true))); + } + + public void testRuntimeFieldsSimpleMatchToFullName() { + MockFieldMapper field1 = new MockFieldMapper("field1"); + MockFieldMapper concrete = new MockFieldMapper("concrete"); + TestRuntimeField field2 = new TestRuntimeField("field2"); + TestRuntimeField subfield = new TestRuntimeField("object.subfield"); + + FieldTypeLookup fieldTypeLookup = new FieldTypeLookup(List.of(field1, concrete), emptyList(), List.of(field2, subfield)); + { + java.util.Set matches = fieldTypeLookup.simpleMatchToFullName("fie*"); + assertEquals(2, matches.size()); + assertTrue(matches.contains("field1")); + assertTrue(matches.contains("field2")); + } + { + java.util.Set matches = fieldTypeLookup.simpleMatchToFullName("object.sub*"); + assertEquals(1, matches.size()); + assertTrue(matches.contains("object.subfield")); + } + } + + public void testRuntimeFieldsSourcePaths() { + //we test that runtime fields are treated like any other field by sourcePaths, although sourcePaths + // should never be called for runtime fields as they are not in _source + MockFieldMapper field1 = new MockFieldMapper("field1"); + MockFieldMapper concrete = new MockFieldMapper("concrete"); + TestRuntimeField field2 = new TestRuntimeField("field2"); + TestRuntimeField subfield = new TestRuntimeField("object.subfield"); + + FieldTypeLookup fieldTypeLookup = new FieldTypeLookup(List.of(field1, concrete), emptyList(), List.of(field2, subfield)); + { + java.util.Set sourcePaths = fieldTypeLookup.sourcePaths("field1"); + assertEquals(1, sourcePaths.size()); + assertTrue(sourcePaths.contains("field1")); + } + { + java.util.Set sourcePaths = fieldTypeLookup.sourcePaths("field2"); + assertEquals(1, sourcePaths.size()); + assertTrue(sourcePaths.contains("field2")); + } + { + java.util.Set sourcePaths = fieldTypeLookup.sourcePaths("object.subfield"); + assertEquals(1, sourcePaths.size()); + assertTrue(sourcePaths.contains("object.subfield")); + } + } + + private static int size(Iterable iterable) { int count = 0; for (MappedFieldType fieldType : iterable) { count++; diff --git a/server/src/test/java/org/elasticsearch/index/mapper/MappingLookupTests.java b/server/src/test/java/org/elasticsearch/index/mapper/MappingLookupTests.java new file mode 100644 index 0000000000000..f8307a7fdd361 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/MappingLookupTests.java @@ -0,0 +1,72 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.mapper; + +import org.elasticsearch.Version; +import org.elasticsearch.common.Explicit; +import org.elasticsearch.test.ESTestCase; + +import java.util.Collections; + +import static org.hamcrest.CoreMatchers.instanceOf; + +public class MappingLookupTests extends ESTestCase { + + public void testOnlyRuntimeField() { + MappingLookup mappingLookup = new MappingLookup(Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), + Collections.singletonList(new TestRuntimeField("test")), 0); + assertEquals(0, size(mappingLookup.fieldMappers())); + assertEquals(0, mappingLookup.objectMappers().size()); + assertNull(mappingLookup.getMapper("test")); + assertThat(mappingLookup.fieldTypes().get("test"), instanceOf(TestRuntimeField.class)); + } + + public void testRuntimeFieldLeafOverride() { + MockFieldMapper fieldMapper = new MockFieldMapper("test"); + MappingLookup mappingLookup = new MappingLookup(Collections.singletonList(fieldMapper), Collections.emptyList(), + Collections.emptyList(), Collections.singletonList(new TestRuntimeField("test")), 0); + assertThat(mappingLookup.getMapper("test"), instanceOf(MockFieldMapper.class)); + assertEquals(1, size(mappingLookup.fieldMappers())); + assertEquals(0, mappingLookup.objectMappers().size()); + assertThat(mappingLookup.fieldTypes().get("test"), instanceOf(TestRuntimeField.class)); + assertEquals(1, size(mappingLookup.fieldTypes().filter(ft -> true))); + } + + public void testSubfieldOverride() { + MockFieldMapper fieldMapper = new MockFieldMapper("object.subfield"); + ObjectMapper objectMapper = new ObjectMapper("object", "object", new Explicit<>(true, true), ObjectMapper.Nested.NO, + ObjectMapper.Dynamic.TRUE, Collections.singletonMap("object.subfield", fieldMapper), Version.CURRENT); + MappingLookup mappingLookup = new MappingLookup(Collections.singletonList(fieldMapper), Collections.singletonList(objectMapper), + Collections.emptyList(), Collections.singletonList(new TestRuntimeField("object.subfield")), 0); + assertThat(mappingLookup.getMapper("object.subfield"), instanceOf(MockFieldMapper.class)); + assertEquals(1, size(mappingLookup.fieldMappers())); + assertEquals(1, mappingLookup.objectMappers().size()); + assertThat(mappingLookup.fieldTypes().get("object.subfield"), instanceOf(TestRuntimeField.class)); + assertEquals(1, size(mappingLookup.fieldTypes().filter(ft -> true))); + } + + private static int size(Iterable iterable) { + int count = 0; + for (Object obj : iterable) { + count++; + } + return count; + } +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java index 8b9efa3b08e9b..ae3ed0e69dca7 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java @@ -214,7 +214,7 @@ private static TestMapper fromMapping(String mapping, Version version) { return BinaryFieldMapper.PARSER; } return null; - }, version, () -> null, null, null, + }, name -> null, version, () -> null, null, null, mapperService.getIndexAnalyzers(), mapperService.getIndexSettings(), () -> { throw new UnsupportedOperationException(); }); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/RootObjectMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/RootObjectMapperTests.java index 5452deda585bc..d1f715f7a967d 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/RootObjectMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/RootObjectMapperTests.java @@ -19,123 +19,127 @@ package org.elasticsearch.index.mapper; +import org.apache.lucene.search.Query; import org.elasticsearch.Version; -import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Strings; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.mapper.MapperService.MergeReason; -import org.elasticsearch.test.ESSingleNodeTestCase; +import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.plugins.MapperPlugin; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Map; import static org.elasticsearch.test.VersionUtils.randomVersionBetween; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.instanceOf; -public class RootObjectMapperTests extends ESSingleNodeTestCase { +public class RootObjectMapperTests extends MapperServiceTestCase { public void testNumericDetection() throws Exception { MergeReason reason = randomFrom(MergeReason.MAPPING_UPDATE, MergeReason.INDEX_TEMPLATE); String mapping = Strings.toString(XContentFactory.jsonBuilder() .startObject() - .startObject("type") + .startObject(MapperService.SINGLE_MAPPING_NAME) .field("numeric_detection", false) .endObject() .endObject()); - MapperService mapperService = createIndex("test").mapperService(); - DocumentMapper mapper = mapperService.merge("type", new CompressedXContent(mapping), reason); - assertEquals(mapping, mapper.mappingSource().toString()); + MapperService mapperService = createMapperService(MapperService.SINGLE_MAPPING_NAME, mapping); + assertEquals(mapping, mapperService.documentMapper().mappingSource().toString()); // update with a different explicit value String mapping2 = Strings.toString(XContentFactory.jsonBuilder() .startObject() - .startObject("type") + .startObject(MapperService.SINGLE_MAPPING_NAME) .field("numeric_detection", true) .endObject() .endObject()); - mapper = mapperService.merge("type", new CompressedXContent(mapping2), reason); - assertEquals(mapping2, mapper.mappingSource().toString()); + merge(mapperService, reason, mapping2); + assertEquals(mapping2, mapperService.documentMapper().mappingSource().toString()); // update with an implicit value: no change String mapping3 = Strings.toString(XContentFactory.jsonBuilder() .startObject() - .startObject("type") + .startObject(MapperService.SINGLE_MAPPING_NAME) .endObject() .endObject()); - mapper = mapperService.merge("type", new CompressedXContent(mapping3), reason); - assertEquals(mapping2, mapper.mappingSource().toString()); + merge(mapperService, reason, mapping3); + assertEquals(mapping2, mapperService.documentMapper().mappingSource().toString()); } public void testDateDetection() throws Exception { MergeReason reason = randomFrom(MergeReason.MAPPING_UPDATE, MergeReason.INDEX_TEMPLATE); String mapping = Strings.toString(XContentFactory.jsonBuilder() .startObject() - .startObject("type") + .startObject(MapperService.SINGLE_MAPPING_NAME) .field("date_detection", true) .endObject() .endObject()); - MapperService mapperService = createIndex("test").mapperService(); - DocumentMapper mapper = mapperService.merge("type", new CompressedXContent(mapping), reason); - assertEquals(mapping, mapper.mappingSource().toString()); + MapperService mapperService = createMapperService(MapperService.SINGLE_MAPPING_NAME, mapping); + assertEquals(mapping, mapperService.documentMapper().mappingSource().toString()); // update with a different explicit value String mapping2 = Strings.toString(XContentFactory.jsonBuilder() .startObject() - .startObject("type") + .startObject(MapperService.SINGLE_MAPPING_NAME) .field("date_detection", false) .endObject() .endObject()); - mapper = mapperService.merge("type", new CompressedXContent(mapping2), reason); - assertEquals(mapping2, mapper.mappingSource().toString()); + merge(mapperService, reason, mapping2); + assertEquals(mapping2, mapperService.documentMapper().mappingSource().toString()); // update with an implicit value: no change String mapping3 = Strings.toString(XContentFactory.jsonBuilder() .startObject() - .startObject("type") + .startObject(MapperService.SINGLE_MAPPING_NAME) .endObject() .endObject()); - mapper = mapperService.merge("type", new CompressedXContent(mapping3), reason); - assertEquals(mapping2, mapper.mappingSource().toString()); + merge(mapperService, reason, mapping3); + assertEquals(mapping2, mapperService.documentMapper().mappingSource().toString()); } public void testDateFormatters() throws Exception { MergeReason reason = randomFrom(MergeReason.MAPPING_UPDATE, MergeReason.INDEX_TEMPLATE); String mapping = Strings.toString(XContentFactory.jsonBuilder() .startObject() - .startObject("type") - .field("dynamic_date_formats", Arrays.asList("yyyy-MM-dd")) + .startObject(MapperService.SINGLE_MAPPING_NAME) + .field("dynamic_date_formats", Collections.singletonList("yyyy-MM-dd")) .endObject() .endObject()); - MapperService mapperService = createIndex("test").mapperService(); - DocumentMapper mapper = mapperService.merge("type", new CompressedXContent(mapping), reason); - assertEquals(mapping, mapper.mappingSource().toString()); + MapperService mapperService = createMapperService(MapperService.SINGLE_MAPPING_NAME, mapping); + assertEquals(mapping, mapperService.documentMapper().mappingSource().toString()); // no update if formatters are not set explicitly String mapping2 = Strings.toString(XContentFactory.jsonBuilder() .startObject() - .startObject("type") + .startObject(MapperService.SINGLE_MAPPING_NAME) .endObject() .endObject()); - mapper = mapperService.merge("type", new CompressedXContent(mapping2), reason); - assertEquals(mapping, mapper.mappingSource().toString()); + merge(mapperService, reason, mapping2); + assertEquals(mapping, mapperService.documentMapper().mappingSource().toString()); String mapping3 = Strings.toString(XContentFactory.jsonBuilder() .startObject() - .startObject("type") - .field("dynamic_date_formats", Arrays.asList()) + .startObject(MapperService.SINGLE_MAPPING_NAME) + .field("dynamic_date_formats", Collections.emptyList()) .endObject() .endObject()); - mapper = mapperService.merge("type", new CompressedXContent(mapping3), reason); - assertEquals(mapping3, mapper.mappingSource().toString()); + merge(mapperService, reason, mapping3); + assertEquals(mapping3, mapperService.documentMapper().mappingSource().toString()); } public void testDynamicTemplates() throws Exception { String mapping = Strings.toString(XContentFactory.jsonBuilder() .startObject() - .startObject("type") + .startObject(MapperService.SINGLE_MAPPING_NAME) .startArray("dynamic_templates") .startObject() .startObject("my_template") @@ -148,27 +152,26 @@ public void testDynamicTemplates() throws Exception { .endArray() .endObject() .endObject()); - MapperService mapperService = createIndex("test").mapperService(); - DocumentMapper mapper = mapperService.merge("type", new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE); - assertEquals(mapping, mapper.mappingSource().toString()); + MapperService mapperService = createMapperService(MapperService.SINGLE_MAPPING_NAME, mapping); + assertEquals(mapping, mapperService.documentMapper().mappingSource().toString()); // no update if templates are not set explicitly String mapping2 = Strings.toString(XContentFactory.jsonBuilder() .startObject() - .startObject("type") + .startObject(MapperService.SINGLE_MAPPING_NAME) .endObject() .endObject()); - mapper = mapperService.merge("type", new CompressedXContent(mapping2), MergeReason.MAPPING_UPDATE); - assertEquals(mapping, mapper.mappingSource().toString()); + merge(MapperService.SINGLE_MAPPING_NAME, mapperService, mapping2); + assertEquals(mapping, mapperService.documentMapper().mappingSource().toString()); String mapping3 = Strings.toString(XContentFactory.jsonBuilder() .startObject() - .startObject("type") - .field("dynamic_templates", Arrays.asList()) + .startObject(MapperService.SINGLE_MAPPING_NAME) + .field("dynamic_templates", Collections.emptyList()) .endObject() .endObject()); - mapper = mapperService.merge("type", new CompressedXContent(mapping3), MergeReason.MAPPING_UPDATE); - assertEquals(mapping3, mapper.mappingSource().toString()); + merge(MapperService.SINGLE_MAPPING_NAME, mapperService, mapping3); + assertEquals(mapping3, mapperService.documentMapper().mappingSource().toString()); } public void testDynamicTemplatesForIndexTemplate() throws IOException { @@ -192,7 +195,7 @@ public void testDynamicTemplatesForIndexTemplate() throws IOException { .endObject() .endArray() .endObject()); - MapperService mapperService = createIndex("test").mapperService(); + MapperService mapperService = createMapperService(Version.CURRENT, Settings.EMPTY, () -> true); mapperService.merge(MapperService.SINGLE_MAPPING_NAME, new CompressedXContent(mapping), MergeReason.INDEX_TEMPLATE); // There should be no update if templates are not set. @@ -249,7 +252,7 @@ public void testDynamicTemplatesForIndexTemplate() throws IOException { public void testIllegalFormatField() throws Exception { String dynamicMapping = Strings.toString(XContentFactory.jsonBuilder() .startObject() - .startObject("type") + .startObject(MapperService.SINGLE_MAPPING_NAME) .startArray("dynamic_date_formats") .startArray().value("test_format").endArray() .endArray() @@ -257,41 +260,39 @@ public void testIllegalFormatField() throws Exception { .endObject()); String mapping = Strings.toString(XContentFactory.jsonBuilder() .startObject() - .startObject("type") + .startObject(MapperService.SINGLE_MAPPING_NAME) .startArray("date_formats") .startArray().value("test_format").endArray() .endArray() .endObject() .endObject()); - - MapperService mapperService = createIndex("test").mapperService(); for (String m : Arrays.asList(mapping, dynamicMapping)) { - IllegalArgumentException e = expectThrows(IllegalArgumentException.class, - () -> mapperService.parse("type", new CompressedXContent(m), false)); - assertEquals("Invalid format: [[test_format]]: expected string value", e.getMessage()); + MapperParsingException e = expectThrows(MapperParsingException.class, + () -> createMapperService(MapperService.SINGLE_MAPPING_NAME, m)); + assertEquals("Failed to parse mapping [_doc]: Invalid format: [[test_format]]: expected string value", e.getMessage()); } } public void testIllegalDynamicTemplates() throws Exception { String mapping = Strings.toString(XContentFactory.jsonBuilder() .startObject() - .startObject("type") + .startObject(MapperService.SINGLE_MAPPING_NAME) .startObject("dynamic_templates") .endObject() .endObject() .endObject()); - MapperService mapperService = createIndex("test").mapperService(); MapperParsingException e = expectThrows(MapperParsingException.class, - () -> mapperService.parse("type", new CompressedXContent(mapping), false)); - assertEquals("Dynamic template syntax error. An array of named objects is expected.", e.getMessage()); + () -> createMapperService(MapperService.SINGLE_MAPPING_NAME, mapping)); + assertEquals("Failed to parse mapping [_doc]: Dynamic template syntax error. An array of named objects is expected.", + e.getMessage()); } public void testIllegalDynamicTemplateUnknownFieldType() throws Exception { XContentBuilder mapping = XContentFactory.jsonBuilder(); mapping.startObject(); { - mapping.startObject("type"); + mapping.startObject(MapperService.SINGLE_MAPPING_NAME); mapping.startArray("dynamic_templates"); { mapping.startObject(); @@ -307,9 +308,8 @@ public void testIllegalDynamicTemplateUnknownFieldType() throws Exception { mapping.endObject(); } mapping.endObject(); - MapperService mapperService = createIndex("test").mapperService(); - DocumentMapper mapper = mapperService.merge("type", new CompressedXContent(Strings.toString(mapping)), MergeReason.MAPPING_UPDATE); - assertThat(mapper.mappingSource().toString(), containsString("\"type\":\"string\"")); + MapperService mapperService = createMapperService(mapping); + assertThat(mapperService.documentMapper().mappingSource().toString(), containsString("\"type\":\"string\"")); assertWarnings("dynamic template [my_template] has invalid content [{\"match_mapping_type\":\"string\",\"mapping\":{\"type\":" + "\"string\"}}], caused by [No mapper found for type [string]]"); } @@ -318,7 +318,7 @@ public void testIllegalDynamicTemplateUnknownAttribute() throws Exception { XContentBuilder mapping = XContentFactory.jsonBuilder(); mapping.startObject(); { - mapping.startObject("type"); + mapping.startObject(MapperService.SINGLE_MAPPING_NAME); mapping.startArray("dynamic_templates"); { mapping.startObject(); @@ -335,9 +335,9 @@ public void testIllegalDynamicTemplateUnknownAttribute() throws Exception { mapping.endObject(); } mapping.endObject(); - MapperService mapperService = createIndex("test").mapperService(); - DocumentMapper mapper = mapperService.merge("type", new CompressedXContent(Strings.toString(mapping)), MergeReason.MAPPING_UPDATE); - assertThat(mapper.mappingSource().toString(), containsString("\"foo\":\"bar\"")); + + MapperService mapperService = createMapperService(mapping); + assertThat(mapperService.documentMapper().mappingSource().toString(), containsString("\"foo\":\"bar\"")); assertWarnings("dynamic template [my_template] has invalid content [{\"match_mapping_type\":\"string\",\"mapping\":{" + "\"foo\":\"bar\",\"type\":\"keyword\"}}], " + "caused by [unknown parameter [foo] on mapper [__dynamic__my_template] of type [keyword]]"); @@ -347,7 +347,7 @@ public void testIllegalDynamicTemplateInvalidAttribute() throws Exception { XContentBuilder mapping = XContentFactory.jsonBuilder(); mapping.startObject(); { - mapping.startObject("type"); + mapping.startObject(MapperService.SINGLE_MAPPING_NAME); mapping.startArray("dynamic_templates"); { mapping.startObject(); @@ -364,21 +364,20 @@ public void testIllegalDynamicTemplateInvalidAttribute() throws Exception { mapping.endObject(); } mapping.endObject(); - MapperService mapperService = createIndex("test").mapperService(); - DocumentMapper mapper = mapperService.merge("type", new CompressedXContent(Strings.toString(mapping)), MergeReason.MAPPING_UPDATE); - assertThat(mapper.mappingSource().toString(), containsString("\"analyzer\":\"foobar\"")); + + MapperService mapperService = createMapperService(mapping); + assertThat(mapperService.documentMapper().mappingSource().toString(), containsString("\"analyzer\":\"foobar\"")); assertWarnings("dynamic template [my_template] has invalid content [{\"match_mapping_type\":\"string\",\"mapping\":{" + "\"analyzer\":\"foobar\",\"type\":\"text\"}}], caused by [analyzer [foobar] has not been configured in mappings]"); } public void testIllegalDynamicTemplateNoMappingType() throws Exception { MapperService mapperService; - { XContentBuilder mapping = XContentFactory.jsonBuilder(); mapping.startObject(); { - mapping.startObject("type"); + mapping.startObject(MapperService.SINGLE_MAPPING_NAME); mapping.startArray("dynamic_templates"); { mapping.startObject(); @@ -399,17 +398,15 @@ public void testIllegalDynamicTemplateNoMappingType() throws Exception { mapping.endObject(); } mapping.endObject(); - mapperService = createIndex("test").mapperService(); - DocumentMapper mapper = - mapperService.merge("type", new CompressedXContent(Strings.toString(mapping)), MergeReason.MAPPING_UPDATE); - assertThat(mapper.mappingSource().toString(), containsString("\"index_phrases\":true")); + mapperService = createMapperService(mapping); + assertThat(mapperService.documentMapper().mappingSource().toString(), containsString("\"index_phrases\":true")); } { boolean useMatchMappingType = randomBoolean(); XContentBuilder mapping = XContentFactory.jsonBuilder(); mapping.startObject(); { - mapping.startObject("type"); + mapping.startObject(MapperService.SINGLE_MAPPING_NAME); mapping.startArray("dynamic_templates"); { mapping.startObject(); @@ -431,9 +428,8 @@ public void testIllegalDynamicTemplateNoMappingType() throws Exception { } mapping.endObject(); - DocumentMapper mapper = - mapperService.merge("type", new CompressedXContent(Strings.toString(mapping)), MergeReason.MAPPING_UPDATE); - assertThat(mapper.mappingSource().toString(), containsString("\"foo\":\"bar\"")); + merge(mapperService, mapping); + assertThat(mapperService.documentMapper().mappingSource().toString(), containsString("\"foo\":\"bar\"")); if (useMatchMappingType) { assertWarnings("dynamic template [my_template] has invalid content [{\"match_mapping_type\":\"*\",\"mapping\":{" + "\"foo\":\"bar\",\"type\":\"{dynamic_type}\"}}], " + @@ -446,16 +442,11 @@ public void testIllegalDynamicTemplateNoMappingType() throws Exception { } } - @Override - protected boolean forbidPrivateIndexSettings() { - return false; - } - public void testIllegalDynamicTemplatePre7Dot7Index() throws Exception { XContentBuilder mapping = XContentFactory.jsonBuilder(); mapping.startObject(); { - mapping.startObject("type"); + mapping.startObject(MapperService.SINGLE_MAPPING_NAME); mapping.startArray("dynamic_templates"); { mapping.startObject(); @@ -471,12 +462,167 @@ public void testIllegalDynamicTemplatePre7Dot7Index() throws Exception { mapping.endObject(); } mapping.endObject(); + Version createdVersion = randomVersionBetween(random(), Version.V_7_0_0, Version.V_7_6_0); - Settings indexSettings = Settings.builder() - .put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), createdVersion) - .build(); - MapperService mapperService = createIndex("test", indexSettings).mapperService(); - DocumentMapper mapper = mapperService.merge("type", new CompressedXContent(Strings.toString(mapping)), MergeReason.MAPPING_UPDATE); - assertThat(mapper.mappingSource().toString(), containsString("\"type\":\"string\"")); + MapperService mapperService = createMapperService(createdVersion, mapping); + assertThat(mapperService.documentMapper().mappingSource().toString(), containsString("\"type\":\"string\"")); + } + + @Override + protected Collection getPlugins() { + return Collections.singletonList(new RuntimeFieldPlugin()); + } + + public void testRuntimeSection() throws IOException { + String mapping = Strings.toString(runtimeMapping(builder -> { + builder.startObject("field1").field("type", "test").field("prop1", "value1").endObject(); + builder.startObject("field2").field("type", "test").field("prop2", "value2").endObject(); + builder.startObject("field3").field("type", "test").endObject(); + })); + MapperService mapperService = createMapperService(MapperService.SINGLE_MAPPING_NAME, mapping); + assertEquals(mapping, mapperService.documentMapper().mappingSource().toString()); + } + + public void testRuntimeSectionMerge() throws IOException { + MapperService mapperService; + { + String mapping = Strings.toString(fieldMapping(b -> b.field("type", "keyword"))); + mapperService = createMapperService(MapperService.SINGLE_MAPPING_NAME, mapping); + assertEquals(mapping, mapperService.documentMapper().mappingSource().toString()); + MappedFieldType field = mapperService.fieldType("field"); + assertThat(field, instanceOf(KeywordFieldMapper.KeywordFieldType.class)); + } + { + String mapping = Strings.toString(runtimeMapping(builder -> { + builder.startObject("field").field("type", "test").field("prop1", "first version").endObject(); + builder.startObject("field2").field("type", "test").endObject(); + })); + merge(MapperService.SINGLE_MAPPING_NAME, mapperService, mapping); + //field overrides now the concrete field already defined + RuntimeField field = (RuntimeField)mapperService.fieldType("field"); + assertEquals("first version", field.prop1); + assertNull(field.prop2); + RuntimeField field2 = (RuntimeField)mapperService.fieldType("field2"); + assertNull(field2.prop1); + assertNull(field2.prop2); + } + { + String mapping = Strings.toString(runtimeMapping( + //the existing runtime field gets updated + builder -> builder.startObject("field").field("type", "test").field("prop2", "second version").endObject())); + merge(MapperService.SINGLE_MAPPING_NAME, mapperService, mapping); + RuntimeField field = (RuntimeField)mapperService.fieldType("field"); + assertNull(field.prop1); + assertEquals("second version", field.prop2); + RuntimeField field2 = (RuntimeField)mapperService.fieldType("field2"); + assertNull(field2.prop1); + assertNull(field2.prop2); + } + { + String mapping = Strings.toString(mapping(builder -> builder.startObject("concrete").field("type", "keyword").endObject())); + merge(MapperService.SINGLE_MAPPING_NAME, mapperService, mapping); + RuntimeField field = (RuntimeField)mapperService.fieldType("field"); + assertNull(field.prop1); + assertEquals("second version", field.prop2); + RuntimeField field2 = (RuntimeField)mapperService.fieldType("field2"); + assertNull(field2.prop1); + assertNull(field2.prop2); + MappedFieldType concrete = mapperService.fieldType("concrete"); + assertThat(concrete, instanceOf(KeywordFieldMapper.KeywordFieldType.class)); + } + { + String mapping = Strings.toString(runtimeMapping( + builder -> builder.startObject("field3").field("type", "test").field("prop1", "value").endObject())); + merge(MapperService.SINGLE_MAPPING_NAME, mapperService, mapping); + assertEquals("{\"_doc\":" + + "{\"runtime\":{" + + "\"field\":{\"type\":\"test\",\"prop2\":\"second version\"}," + + "\"field2\":{\"type\":\"test\"}," + + "\"field3\":{\"type\":\"test\",\"prop1\":\"value\"}}," + + "\"properties\":{" + + "\"concrete\":{\"type\":\"keyword\"}," + + "\"field\":{\"type\":\"keyword\"}}}}", + mapperService.documentMapper().mappingSource().toString()); + } + } + + public void testRuntimeSectionNonRuntimeType() throws IOException { + XContentBuilder mapping = runtimeFieldMapping(builder -> builder.field("type", "keyword")); + MapperParsingException e = expectThrows(MapperParsingException.class, () -> createMapperService(mapping)); + assertEquals("Failed to parse mapping [_doc]: No handler for type [keyword] declared on runtime field [field]", e.getMessage()); + } + + public void testRuntimeSectionHandlerNotFound() throws IOException { + XContentBuilder mapping = runtimeFieldMapping(builder -> builder.field("type", "unknown")); + MapperParsingException e = expectThrows(MapperParsingException.class, () -> createMapperService(mapping)); + assertEquals("Failed to parse mapping [_doc]: No handler for type [unknown] declared on runtime field [field]", e.getMessage()); + } + + public void testRuntimeSectionMissingType() throws IOException { + XContentBuilder mapping = runtimeFieldMapping(builder -> {}); + MapperParsingException e = expectThrows(MapperParsingException.class, () -> createMapperService(mapping)); + assertEquals("Failed to parse mapping [_doc]: No type specified for runtime field [field]", e.getMessage()); + } + + public void testRuntimeSectionWrongFormat() throws IOException { + XContentBuilder mapping = runtimeMapping(builder -> builder.field("field", "value")); + MapperParsingException e = expectThrows(MapperParsingException.class, () -> createMapperService(mapping)); + assertEquals("Failed to parse mapping [_doc]: Expected map for runtime field [field] definition but got a java.lang.String", + e.getMessage()); + } + + public void testRuntimeSectionRemainingField() throws IOException { + XContentBuilder mapping = runtimeFieldMapping(builder -> builder.field("type", "test").field("unsupported", "value")); + MapperParsingException e = expectThrows(MapperParsingException.class, () -> createMapperService(mapping)); + assertEquals("Failed to parse mapping [_doc]: Mapping definition for [field] has unsupported parameters: " + + "[unsupported : value]", e.getMessage()); + } + + private static class RuntimeFieldPlugin extends Plugin implements MapperPlugin { + @Override + public Map getRuntimeFieldTypes() { + return Collections.singletonMap("test", (name, node, parserContext) -> { + Object prop1 = node.remove("prop1"); + Object prop2 = node.remove("prop2"); + return new RuntimeField(name, prop1 == null ? null : prop1.toString(), prop2 == null ? null : prop2.toString()); + }); + } + } + + private static final class RuntimeField extends RuntimeFieldType { + private final String prop1; + private final String prop2; + + protected RuntimeField(String name, String prop1, String prop2) { + super(name, Collections.emptyMap()); + this.prop1 = prop1; + this.prop2 = prop2; + } + + @Override + public ValueFetcher valueFetcher(QueryShardContext context, SearchLookup searchLookup, String format) { + return null; + } + + @Override + public String typeName() { + return "test"; + } + + @Override + public Query termQuery(Object value, QueryShardContext context) { + return null; + } + + @Override + protected void doXContentBody(XContentBuilder builder, boolean includeDefaults) throws IOException { + if (prop1 != null) { + builder.field("prop1", prop1); + } + if (prop2 != null) { + builder.field("prop2", prop2); + } + } + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TestRuntimeField.java b/server/src/test/java/org/elasticsearch/index/mapper/TestRuntimeField.java new file mode 100644 index 0000000000000..15a03e4de6dc7 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/TestRuntimeField.java @@ -0,0 +1,52 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.mapper; + +import org.apache.lucene.search.Query; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.lookup.SearchLookup; + +import java.util.Collections; + +public class TestRuntimeField extends RuntimeFieldType { + public TestRuntimeField(String name) { + super(name, Collections.emptyMap()); + } + + @Override + protected void doXContentBody(XContentBuilder builder, boolean includeDefaults) { + } + + @Override + public ValueFetcher valueFetcher(QueryShardContext context, SearchLookup searchLookup, String format) { + return null; + } + + @Override + public String typeName() { + return null; + } + + @Override + public Query termQuery(Object value, QueryShardContext context) { + return null; + } +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TypeParsersTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TypeParsersTests.java index 83383f6343ded..00a058c9c0255 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TypeParsersTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TypeParsersTests.java @@ -79,8 +79,8 @@ public void testMultiFieldWithinMultiField() throws IOException { IndexAnalyzers indexAnalyzers = new IndexAnalyzers(defaultAnalyzers(), Collections.emptyMap(), Collections.emptyMap()); MapperService mapperService = mock(MapperService.class); when(mapperService.getIndexAnalyzers()).thenReturn(indexAnalyzers); - Mapper.TypeParser.ParserContext olderContext = new Mapper.TypeParser.ParserContext(null, type -> typeParser, Version.CURRENT, null, - null, null, mapperService.getIndexAnalyzers(), mapperService.getIndexSettings(), () -> { + Mapper.TypeParser.ParserContext olderContext = new Mapper.TypeParser.ParserContext(null, type -> typeParser, type -> null, + Version.CURRENT, null, null, null, mapperService.getIndexAnalyzers(), mapperService.getIndexSettings(), () -> { throw new UnsupportedOperationException(); }); diff --git a/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java b/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java index eaed67f424abb..75c9827fe018d 100644 --- a/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java @@ -31,6 +31,7 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.Scorable; import org.apache.lucene.search.ScoreMode; +import org.apache.lucene.search.SortField; import org.apache.lucene.search.TermQuery; import org.apache.lucene.store.Directory; import org.elasticsearch.Version; @@ -38,6 +39,7 @@ import org.elasticsearch.common.TriFunction; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.xcontent.NamedXContentRegistry; @@ -49,34 +51,41 @@ import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.LeafFieldData; import org.elasticsearch.index.fielddata.ScriptDocValues; +import org.elasticsearch.index.fielddata.SortedBinaryDocValues; import org.elasticsearch.index.fielddata.plain.AbstractLeafOrdinalsFieldData; -import org.elasticsearch.index.mapper.ContentPath; -import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.IndexFieldMapper; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.mapper.Mapper; -import org.elasticsearch.index.mapper.Mapper.TypeParser; import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.mapper.MockFieldMapper; import org.elasticsearch.index.mapper.NumberFieldMapper; -import org.elasticsearch.index.mapper.ParseContext; +import org.elasticsearch.index.mapper.RuntimeFieldType; +import org.elasticsearch.index.mapper.TestRuntimeField; import org.elasticsearch.index.mapper.TextFieldMapper; -import org.elasticsearch.index.mapper.TextSearchInfo; -import org.elasticsearch.index.mapper.ValueFetcher; +import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.indices.IndicesModule; +import org.elasticsearch.indices.mapper.MapperRegistry; import org.elasticsearch.plugins.MapperPlugin; +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.MultiValueMode; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import org.elasticsearch.search.lookup.LeafDocLookup; import org.elasticsearch.search.lookup.LeafSearchLookup; import org.elasticsearch.search.lookup.SearchLookup; +import org.elasticsearch.search.sort.BucketedSort; +import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.ESTestCase; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.function.BiFunction; +import java.util.Set; +import java.util.function.Function; import java.util.function.Supplier; import static org.hamcrest.Matchers.equalTo; @@ -84,9 +93,6 @@ import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; import static org.hamcrest.Matchers.sameInstance; -import static org.mockito.Matchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; public class QueryShardContextTests extends ESTestCase { @@ -216,40 +222,40 @@ public void testIndexSortedOnField() { } public void testFielddataLookupSelfReference() { - QueryShardContext queryShardContext = createQueryShardContext("uuid", null, (field, leafLookup, docId) -> { + QueryShardContext queryShardContext = createQueryShardContext("uuid", null, fieldTypeLookup((field, leafLookup, docId) -> { //simulate a runtime field that depends on itself e.g. field: doc['field'] return leafLookup.doc().get(field).toString(); - }); + })); IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, () -> collect("field", queryShardContext)); assertEquals("Cyclic dependency detected while resolving runtime fields: field -> field", iae.getMessage()); } public void testFielddataLookupLooseLoop() { - QueryShardContext queryShardContext = createQueryShardContext("uuid", null, (field, leafLookup, docId) -> { + QueryShardContext queryShardContext = createQueryShardContext("uuid", null, fieldTypeLookup((field, leafLookup, docId) -> { //simulate a runtime field cycle: 1: doc['2'] 2: doc['3'] 3: doc['4'] 4: doc['1'] if (field.equals("4")) { return leafLookup.doc().get("1").toString(); } return leafLookup.doc().get(Integer.toString(Integer.parseInt(field) + 1)).toString(); - }); + })); IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, () -> collect("1", queryShardContext)); assertEquals("Cyclic dependency detected while resolving runtime fields: 1 -> 2 -> 3 -> 4 -> 1", iae.getMessage()); } public void testFielddataLookupTerminatesInLoop() { - QueryShardContext queryShardContext = createQueryShardContext("uuid", null, (field, leafLookup, docId) -> { + QueryShardContext queryShardContext = createQueryShardContext("uuid", null, fieldTypeLookup((field, leafLookup, docId) -> { //simulate a runtime field cycle: 1: doc['2'] 2: doc['3'] 3: doc['4'] 4: doc['4'] if (field.equals("4")) { return leafLookup.doc().get("4").toString(); } return leafLookup.doc().get(Integer.toString(Integer.parseInt(field) + 1)).toString(); - }); + })); IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, () -> collect("1", queryShardContext)); assertEquals("Cyclic dependency detected while resolving runtime fields: 1 -> 2 -> 3 -> 4 -> 4", iae.getMessage()); } public void testFielddataLookupSometimesLoop() throws IOException { - QueryShardContext queryShardContext = createQueryShardContext("uuid", null, (field, leafLookup, docId) -> { + QueryShardContext queryShardContext = createQueryShardContext("uuid", null, fieldTypeLookup((field, leafLookup, docId) -> { if (docId == 0) { return field + "_" + docId; } else { @@ -260,7 +266,7 @@ public void testFielddataLookupSometimesLoop() throws IOException { int i = Integer.parseInt(field.substring(field.length() - 1)); return leafLookup.doc().get("field" + (i + 1)).toString(); } - }); + })); List values = collect("field1", queryShardContext, new TermQuery(new Term("indexed_field", "first"))); assertEquals(Collections.singletonList("field1_0"), values); IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, () -> collect("field1", queryShardContext)); @@ -269,16 +275,16 @@ public void testFielddataLookupSometimesLoop() throws IOException { } public void testFielddataLookupBeyondMaxDepth() { - QueryShardContext queryShardContext = createQueryShardContext("uuid", null, (field, leafLookup, docId) -> { + QueryShardContext queryShardContext = createQueryShardContext("uuid", null, fieldTypeLookup((field, leafLookup, docId) -> { int i = Integer.parseInt(field); return leafLookup.doc().get(Integer.toString(i + 1)).toString(); - }); + })); IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, () -> collect("1", queryShardContext)); assertEquals("Field requires resolving too many dependent fields: 1 -> 2 -> 3 -> 4 -> 5 -> 6", iae.getMessage()); } public void testFielddataLookupReferencesBelowMaxDepth() throws IOException { - QueryShardContext queryShardContext = createQueryShardContext("uuid", null, (field, leafLookup, docId) -> { + QueryShardContext queryShardContext = createQueryShardContext("uuid", null, fieldTypeLookup((field, leafLookup, docId) -> { int i = Integer.parseInt(field.substring(field.length() - 1)); if (i == 5) { return "test"; @@ -286,13 +292,13 @@ public void testFielddataLookupReferencesBelowMaxDepth() throws IOException { ScriptDocValues scriptDocValues = leafLookup.doc().get("field" + (i + 1)); return scriptDocValues.get(0).toString() + docId; } - }); + })); assertEquals(Arrays.asList("test0000", "test1111"), collect("field1", queryShardContext)); } public void testFielddataLookupOneFieldManyReferences() throws IOException { int numFields = randomIntBetween(5, 20); - QueryShardContext queryShardContext = createQueryShardContext("uuid", null, (field, leafLookup, docId) -> { + QueryShardContext queryShardContext = createQueryShardContext("uuid", null, fieldTypeLookup((field, leafLookup, docId) -> { if (field.equals("field")) { StringBuilder value = new StringBuilder(); for (int i = 0; i < numFields; i++) { @@ -302,7 +308,7 @@ public void testFielddataLookupOneFieldManyReferences() throws IOException { } else { return "test" + docId; } - }); + })); StringBuilder expectedFirstDoc = new StringBuilder(); StringBuilder expectedSecondDoc = new StringBuilder(); for (int i = 0; i < numFields; i++) { @@ -312,86 +318,80 @@ public void testFielddataLookupOneFieldManyReferences() throws IOException { assertEquals(Arrays.asList(expectedFirstDoc.toString(), expectedSecondDoc.toString()), collect("field", queryShardContext)); } - public void testRuntimeFields() throws IOException { - MapperService mapperService = mockMapperService("test", org.elasticsearch.common.collect.List.of(new MapperPlugin() { - @Override - public Map getMappers() { - return org.elasticsearch.common.collect.Map.of("runtime", (name, node, parserContext) -> new Mapper.Builder(name) { - @Override - public Mapper build(ContentPath path) { - return new DummyMapper(name, new DummyMappedFieldType(name)); - } - }); - } - })); - when(mapperService.fieldType("pig")).thenReturn(new DummyMappedFieldType("pig")); - when(mapperService.simpleMatchToFullName("*")).thenReturn(org.elasticsearch.common.collect.Set.of("pig")); + public void testSearchRequestRuntimeFields() { /* * Making these immutable here test that we don't modify them. * Modifying them would cause all kinds of problems if two * shards are parsed on the same node. */ Map runtimeMappings = org.elasticsearch.common.collect.Map.ofEntries( - org.elasticsearch.common.collect.Map.entry("cat", org.elasticsearch.common.collect.Map.of("type", "keyword")), - org.elasticsearch.common.collect.Map.entry("dog", org.elasticsearch.common.collect.Map.of("type", "keyword")) + org.elasticsearch.common.collect.Map.entry("cat", org.elasticsearch.common.collect.Map.of("type", "test")), + org.elasticsearch.common.collect.Map.entry("dog", org.elasticsearch.common.collect.Map.of("type", "test")) ); - QueryShardContext qsc = new QueryShardContext( - 0, - mapperService.getIndexSettings(), - BigArrays.NON_RECYCLING_INSTANCE, - null, - (mappedFieldType, idxName, searchLookup) -> mappedFieldType.fielddataBuilder(idxName, searchLookup).build(null, null), - mapperService, - null, - null, - NamedXContentRegistry.EMPTY, - new NamedWriteableRegistry(org.elasticsearch.common.collect.List.of()), + QueryShardContext qsc = createQueryShardContext( + "uuid", null, - null, - () -> 0, - "test", - null, - () -> true, - null, - runtimeMappings - ); + org.elasticsearch.common.collect.Map.of( + "pig", new MockFieldMapper.FakeFieldType("pig"), + "cat", new MockFieldMapper.FakeFieldType("cat")), + runtimeMappings, + Collections.singletonList(new MapperPlugin() { + @Override + public Map getRuntimeFieldTypes() { + return org.elasticsearch.common.collect.Map.of( + "test", (name, node, parserContext) -> new TestRuntimeField(name)); + } + })); assertTrue(qsc.isFieldMapped("cat")); - assertThat(qsc.getFieldType("cat"), instanceOf(DummyMappedFieldType.class)); + assertThat(qsc.getFieldType("cat"), instanceOf(TestRuntimeField.class)); assertThat(qsc.simpleMatchToIndexNames("cat"), equalTo(org.elasticsearch.common.collect.Set.of("cat"))); assertTrue(qsc.isFieldMapped("dog")); - assertThat(qsc.getFieldType("dog"), instanceOf(DummyMappedFieldType.class)); + assertThat(qsc.getFieldType("dog"), instanceOf(TestRuntimeField.class)); assertThat(qsc.simpleMatchToIndexNames("dog"), equalTo(org.elasticsearch.common.collect.Set.of("dog"))); assertTrue(qsc.isFieldMapped("pig")); - assertThat(qsc.getFieldType("pig"), instanceOf(DummyMappedFieldType.class)); + assertThat(qsc.getFieldType("pig"), instanceOf(MockFieldMapper.FakeFieldType.class)); assertThat(qsc.simpleMatchToIndexNames("pig"), equalTo(org.elasticsearch.common.collect.Set.of("pig"))); assertThat(qsc.simpleMatchToIndexNames("*"), equalTo(org.elasticsearch.common.collect.Set.of("cat", "dog", "pig"))); } public static QueryShardContext createQueryShardContext(String indexUuid, String clusterAlias) { - return createQueryShardContext(indexUuid, clusterAlias, null); + return createQueryShardContext(indexUuid, clusterAlias, name -> { + throw new UnsupportedOperationException(); + }); } private static QueryShardContext createQueryShardContext( String indexUuid, String clusterAlias, - TriFunction runtimeDocValues + Function fieldTypeLookup ) { - MapperService mapperService = mockMapperService(indexUuid, org.elasticsearch.common.collect.List.of()); - if (runtimeDocValues != null) { - when(mapperService.fieldType(any())).thenAnswer(fieldTypeInv -> { - String fieldName = (String)fieldTypeInv.getArguments()[0]; - return mockFieldType(fieldName, (leafSearchLookup, docId) -> runtimeDocValues.apply(fieldName, leafSearchLookup, docId)); - }); - } + return createQueryShardContext(indexUuid, clusterAlias, new HashMap() { + @Override + public MappedFieldType get(Object key) { + return fieldTypeLookup.apply(key.toString()); + } + }, Collections.emptyMap(), Collections.emptyList()); + } + + private static QueryShardContext createQueryShardContext( + String indexUuid, + String clusterAlias, + Map fieldTypeLookup, + Map runtimeMappings, + List mapperPlugins + ) { + MapperService mapperService = createMapperService(indexUuid, fieldTypeLookup, mapperPlugins); final long nowInMillis = randomNonNegativeLong(); return new QueryShardContext( 0, mapperService.getIndexSettings(), BigArrays.NON_RECYCLING_INSTANCE, null, (mappedFieldType, idxName, searchLookup) -> mappedFieldType.fielddataBuilder(idxName, searchLookup).build(null, null), mapperService, null, null, NamedXContentRegistry.EMPTY, new NamedWriteableRegistry(Collections.emptyList()), - null, null, () -> nowInMillis, clusterAlias, null, () -> true, null); + null, null, () -> nowInMillis, clusterAlias, null, () -> true, null, runtimeMappings); } - private static MapperService mockMapperService(String indexUuid, List mapperPlugins) { + private static MapperService createMapperService(String indexUuid, + Map fieldTypeLookup, + List mapperPlugins) { IndexMetadata.Builder indexMetadataBuilder = new IndexMetadata.Builder("index"); indexMetadataBuilder.settings(Settings.builder().put("index.version.created", Version.CURRENT) .put("index.number_of_shards", 1) @@ -404,60 +404,119 @@ private static MapperService mockMapperService(String indexUuid, List { + throw new UnsupportedOperationException(); + }, () -> true, null) { + @Override + public MappedFieldType fieldType(String name) { + return fieldTypeLookup.get(name); + } - MapperService mapperService = mock(MapperService.class); - when(mapperService.getIndexSettings()).thenReturn(indexSettings); - when(mapperService.index()).thenReturn(indexMetadata.getIndex()); - when(mapperService.getIndexAnalyzers()).thenReturn(indexAnalyzers); - Map typeParserMap = IndicesModule.getMappers(mapperPlugins); - Mapper.TypeParser.ParserContext parserContext = new Mapper.TypeParser.ParserContext(name -> null, typeParserMap::get, - Version.CURRENT, () -> null, null, null, mapperService.getIndexAnalyzers(), mapperService.getIndexSettings(), - () -> { + @Override + public Set simpleMatchToFullName(String pattern) { + if (Regex.isMatchAllPattern(pattern)) { + return Collections.unmodifiableSet(fieldTypeLookup.keySet()); + } throw new UnsupportedOperationException(); - }); - when(mapperService.parserContext()).thenReturn(parserContext); - return mapperService; + } + }; } - private static MappedFieldType mockFieldType(String fieldName, BiFunction runtimeDocValues) { - MappedFieldType fieldType = mock(MappedFieldType.class); - when(fieldType.name()).thenReturn(fieldName); - when(fieldType.fielddataBuilder(any(), any())).thenAnswer(builderInv -> { - @SuppressWarnings("unchecked") - Supplier searchLookup = ((Supplier) builderInv.getArguments()[1]); - IndexFieldData indexFieldData = mock(IndexFieldData.class); - when(indexFieldData.load(any())).thenAnswer(loadArgs -> { - LeafReaderContext leafReaderContext = (LeafReaderContext) loadArgs.getArguments()[0]; - LeafFieldData leafFieldData = mock(LeafFieldData.class); - when(leafFieldData.getScriptValues()).thenAnswer(scriptValuesArgs -> new ScriptDocValues() { - String value; + private static Function fieldTypeLookup( + TriFunction runtimeDocValues) { + return name -> new TestRuntimeField(name) { + @Override + public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, + Supplier searchLookup) { + return (cache, breakerService) -> new IndexFieldData() { + @Override + public String getFieldName() { + return name; + } + + @Override + public ValuesSourceType getValuesSourceType() { + throw new UnsupportedOperationException(); + } + + @Override + public LeafFieldData load(LeafReaderContext context) { + return new LeafFieldData() { + @Override + public ScriptDocValues getScriptValues() { + return new ScriptDocValues() { + String value; + + @Override + public int size() { + return 1; + } + + @Override + public String get(int index) { + assert index == 0; + return value; + } + + @Override + public void setNextDocId(int docId) { + assert docId >= 0; + LeafSearchLookup leafLookup = searchLookup.get() + .getLeafSearchLookup(context); + leafLookup.setDocument(docId); + value = runtimeDocValues.apply(name, leafLookup, docId); + } + }; + } + + @Override + public SortedBinaryDocValues getBytesValues() { + throw new UnsupportedOperationException(); + } + + @Override + public long ramBytesUsed() { + throw new UnsupportedOperationException(); + } + + @Override + public void close() { + throw new UnsupportedOperationException(); + } + }; + } @Override - public int size() { - return 1; + public LeafFieldData loadDirect(LeafReaderContext context) { + throw new UnsupportedOperationException(); } @Override - public String get(int index) { - assert index == 0; - return value; + public SortField sortField(Object missingValue, + MultiValueMode sortMode, + XFieldComparatorSource.Nested nested, + boolean reverse) { + throw new UnsupportedOperationException(); } @Override - public void setNextDocId(int docId) { - assert docId >= 0; - LeafSearchLookup leafLookup = searchLookup.get().getLeafSearchLookup(leafReaderContext); - leafLookup.setDocument(docId); - value = runtimeDocValues.apply(leafLookup, docId); + public BucketedSort newBucketedSort(BigArrays bigArrays, + Object missingValue, + MultiValueMode sortMode, + XFieldComparatorSource.Nested nested, + SortOrder sortOrder, + DocValueFormat format, + int bucketSize, + BucketedSort.ExtraData extra) { + throw new UnsupportedOperationException(); } - }); - return leafFieldData; - }); - IndexFieldData.Builder builder = mock(IndexFieldData.Builder.class); - when(builder.build(any(), any())).thenAnswer(buildInv -> indexFieldData); - return builder; - }); - return fieldType; + }; + } + }; } private static List collect(String field, QueryShardContext queryShardContext) throws IOException { @@ -513,46 +572,4 @@ public void collect(int doc) throws IOException { return result; } } - - private static class DummyMapper extends FieldMapper { - protected DummyMapper(String simpleName, MappedFieldType mappedFieldType) { - super(simpleName, mappedFieldType, org.elasticsearch.common.collect.Map.of(), MultiFields.empty(), CopyTo.empty()); - } - - @Override - protected void parseCreateField(ParseContext context) throws IOException { - throw new UnsupportedOperationException(); - } - - @Override - public Builder getMergeBuilder() { - throw new UnsupportedOperationException(); - } - - @Override - protected String contentType() { - throw new UnsupportedOperationException(); - } - } - - private static class DummyMappedFieldType extends MappedFieldType { - DummyMappedFieldType(String name) { - super(name, true, false, true, TextSearchInfo.SIMPLE_MATCH_ONLY, null); - } - - @Override - public ValueFetcher valueFetcher(QueryShardContext context, SearchLookup searchLookup, String format) { - throw new UnsupportedOperationException(); - } - - @Override - public String typeName() { - return "runtime"; - } - - @Override - public Query termQuery(Object value, QueryShardContext context) { - throw new UnsupportedOperationException(); - } - } } diff --git a/server/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java b/server/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java index ccfdd34aa81f2..e95d401eff356 100644 --- a/server/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java +++ b/server/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java @@ -30,6 +30,7 @@ import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.MetadataFieldMapper; import org.elasticsearch.index.mapper.RoutingFieldMapper; +import org.elasticsearch.index.mapper.RuntimeFieldType; import org.elasticsearch.index.mapper.SeqNoFieldMapper; import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.index.mapper.TextFieldMapper; @@ -193,6 +194,19 @@ public Map getMetadataMappers() { assertThat(e.getMessage(), containsString("already registered")); } + public void testDuplicateRuntimeFieldPlugin() { + MapperPlugin plugin = new MapperPlugin() { + @Override + public Map getRuntimeFieldTypes() { + return Collections.singletonMap("test", (name, node, parserContext) -> null); + } + }; + List plugins = Arrays.asList(plugin, plugin); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> new IndicesModule(plugins)); + assertThat(e.getMessage(), containsString("already registered")); + } + public void testDuplicateFieldNamesMapper() { List plugins = Arrays.asList(new MapperPlugin() { @Override diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java index b8ac57de07d90..df78993331d6f 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java @@ -172,6 +172,15 @@ protected final MapperService createMapperService(Version version, Settings settings, BooleanSupplier idFieldDataEnabled, XContentBuilder mapping) throws IOException { + + MapperService mapperService = createMapperService(version, settings, idFieldDataEnabled); + merge(mapperService, mapping); + return mapperService; + } + + protected final MapperService createMapperService(Version version, + Settings settings, + BooleanSupplier idFieldDataEnabled) { settings = Settings.builder() .put("index.number_of_replicas", 0) .put("index.number_of_shards", 1) @@ -191,7 +200,7 @@ protected final MapperService createMapperService(Version version, ); ScriptService scriptService = new ScriptService(getIndexSettings(), scriptModule.engines, scriptModule.contexts); SimilarityService similarityService = new SimilarityService(indexSettings, scriptService, emptyMap()); - MapperService mapperService = new MapperService( + return new MapperService( indexSettings, createIndexAnalyzers(indexSettings), xContentRegistry(), @@ -201,8 +210,6 @@ protected final MapperService createMapperService(Version version, idFieldDataEnabled, scriptService ); - merge(mapperService, mapping); - return mapperService; } protected final void withLuceneIndex( @@ -290,6 +297,20 @@ protected final XContentBuilder fieldMapping(CheckedConsumer buildField) throws IOException { + return runtimeMapping(b -> { + b.startObject("field"); + buildField.accept(b); + b.endObject(); + }); + } + + protected final XContentBuilder runtimeMapping(CheckedConsumer buildFields) throws IOException { + XContentBuilder builder = XContentFactory.jsonBuilder().startObject().startObject("_doc").startObject("runtime"); + buildFields.accept(builder); + return builder.endObject().endObject().endObject(); + } + private AggregationContext aggregationContext( ValuesSourceRegistry valuesSourceRegistry, MapperService mapperService, diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java index 78c3330b23aa1..08bfae61d0cca 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java @@ -19,9 +19,6 @@ package org.elasticsearch.index.mapper; -import org.elasticsearch.Version; -import org.elasticsearch.cluster.metadata.IndexMetadata; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.lookup.SearchLookup; @@ -30,9 +27,6 @@ // this sucks how much must be overridden just do get a dummy field mapper... public class MockFieldMapper extends FieldMapper { - static Settings DEFAULT_SETTINGS = Settings.builder() - .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id) - .build(); public MockFieldMapper(String fullName) { this(new FakeFieldType(fullName)); diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java index a87df96e4556a..2b4d6a1e16941 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java @@ -789,7 +789,7 @@ private void writeTestDoc(MappedFieldType fieldType, String fieldName, RandomInd private static class MockParserContext extends Mapper.TypeParser.ParserContext { MockParserContext() { - super(null, null, Version.CURRENT, null, null, null, null, null, null); + super(null, null, null, null, null, null, null, null, null, null); } @Override diff --git a/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/index/mapper/FlattenedFieldLookupTests.java b/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/index/mapper/FlattenedFieldLookupTests.java index 1083f4d246ac4..3279ca35b3581 100644 --- a/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/index/mapper/FlattenedFieldLookupTests.java +++ b/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/index/mapper/FlattenedFieldLookupTests.java @@ -39,7 +39,7 @@ public void testFieldTypeLookup() { String fieldName = "object1.object2.field"; FlattenedFieldMapper mapper = createFlattenedMapper(fieldName); - FieldTypeLookup lookup = new FieldTypeLookup(singletonList(mapper), emptyList()); + FieldTypeLookup lookup = new FieldTypeLookup(singletonList(mapper), emptyList(), emptyList()); assertEquals(mapper.fieldType(), lookup.get(fieldName)); String objectKey = "key1.key2"; @@ -60,7 +60,7 @@ public void testFieldTypeLookupWithAlias() { String aliasName = "alias"; FieldAliasMapper alias = new FieldAliasMapper(aliasName, aliasName, fieldName); - FieldTypeLookup lookup = new FieldTypeLookup(singletonList(mapper), singletonList(alias)); + FieldTypeLookup lookup = new FieldTypeLookup(singletonList(mapper), singletonList(alias), emptyList()); assertEquals(mapper.fieldType(), lookup.get(aliasName)); String objectKey = "key1.key2"; @@ -83,11 +83,11 @@ public void testFieldTypeLookupWithMultipleFields() { FlattenedFieldMapper mapper2 = createFlattenedMapper(field2); FlattenedFieldMapper mapper3 = createFlattenedMapper(field3); - FieldTypeLookup lookup = new FieldTypeLookup(Arrays.asList(mapper1, mapper2), emptyList()); + FieldTypeLookup lookup = new FieldTypeLookup(Arrays.asList(mapper1, mapper2), emptyList(), emptyList()); assertNotNull(lookup.get(field1 + ".some.key")); assertNotNull(lookup.get(field2 + ".some.key")); - lookup = new FieldTypeLookup(Arrays.asList(mapper1, mapper2, mapper3), emptyList()); + lookup = new FieldTypeLookup(Arrays.asList(mapper1, mapper2, mapper3), emptyList(), emptyList()); assertNotNull(lookup.get(field1 + ".some.key")); assertNotNull(lookup.get(field2 + ".some.key")); assertNotNull(lookup.get(field3 + ".some.key")); @@ -124,7 +124,7 @@ public void testFieldLookupIterator() { MockFieldMapper mapper = new MockFieldMapper("foo"); FlattenedFieldMapper flattenedMapper = createFlattenedMapper("object1.object2.field"); - FieldTypeLookup lookup = new FieldTypeLookup(Arrays.asList(mapper, flattenedMapper), emptyList()); + FieldTypeLookup lookup = new FieldTypeLookup(Arrays.asList(mapper, flattenedMapper), emptyList(), emptyList()); Set fieldNames = new HashSet<>(); lookup.filter(ft -> true).forEach(ft -> fieldNames.add(ft.name())); diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsRestIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsRestIT.java index 09d430222c871..24d985ff120a1 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsRestIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsRestIT.java @@ -118,6 +118,12 @@ private void addAirlineData() throws IOException { Request createAirlineDataRequest = new Request("PUT", "/airline-data"); createAirlineDataRequest.setJsonEntity("{" + " \"mappings\": {" + + " \"runtime\": {" + + " \"airline_lowercase_rt\": { " + + " \"type\":\"keyword\"," + + " \"script\" : { \"source\": \"emit(params._source.airline.toLowerCase())\" }" + + " }" + + " }," + " \"properties\": {" + " \"time stamp\": { \"type\":\"date\"}," // space in 'time stamp' is intentional + " \"airline\": {" @@ -127,11 +133,6 @@ private void addAirlineData() throws IOException { + " \"keyword\":{\"type\":\"keyword\"}" + " }" + " }," - + " \"airline_lowercase_rt\": { " - + " \"type\":\"runtime\"," - + " \"runtime_type\": \"keyword\"," - + " \"script\" : { \"source\": \"emit(params._source.airline.toLowerCase())\" }" - + " }," + " \"responsetime\": { \"type\":\"float\"}" + " }" + " }" diff --git a/x-pack/plugin/runtime-fields/qa/build.gradle b/x-pack/plugin/runtime-fields/qa/build.gradle index 1ab8b6efe9250..9fd5e0e62ecf7 100644 --- a/x-pack/plugin/runtime-fields/qa/build.gradle +++ b/x-pack/plugin/runtime-fields/qa/build.gradle @@ -1,4 +1,4 @@ -// Shared infratructure +// Shared infrastructure apply plugin: 'elasticsearch.build' @@ -43,6 +43,8 @@ subprojects { 'suggest', ] if (project.name.equals('core-with-mapped')) { + //disabled until runtime fields can be created from a dynamic template + yamlRestTest.enabled = false suites += [ // These two don't support runtime fields on the request. Should they? 'field_caps', @@ -76,4 +78,4 @@ subprojects { ].join(',') } } -} \ No newline at end of file +} diff --git a/x-pack/plugin/runtime-fields/qa/core-with-mapped/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/mapped/CoreWithMappedRuntimeFieldsIT.java b/x-pack/plugin/runtime-fields/qa/core-with-mapped/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/mapped/CoreWithMappedRuntimeFieldsIT.java index 0c19187c088d0..131e124bac48d 100644 --- a/x-pack/plugin/runtime-fields/qa/core-with-mapped/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/mapped/CoreWithMappedRuntimeFieldsIT.java +++ b/x-pack/plugin/runtime-fields/qa/core-with-mapped/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/mapped/CoreWithMappedRuntimeFieldsIT.java @@ -8,14 +8,12 @@ import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; - import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; import org.elasticsearch.test.rest.yaml.section.ApiCallSection; import org.elasticsearch.xpack.runtimefields.test.CoreTestTranslater; -import java.util.HashMap; import java.util.Map; /** @@ -43,20 +41,8 @@ protected Map dynamicTemplateFor(String type) { protected Suite suite(ClientYamlTestCandidate candidate) { return new Suite(candidate) { @Override - protected boolean modifyMappingProperties(String index, Map properties) { - Map newProperties = new HashMap<>(properties.size()); - Map> runtimeProperties = new HashMap<>(properties.size()); - if (false == runtimeifyMappingProperties(properties, newProperties, runtimeProperties)) { - return false; - } - for (Map.Entry> runtimeProperty : runtimeProperties.entrySet()) { - runtimeProperty.getValue().put("runtime_type", runtimeProperty.getValue().get("type")); - runtimeProperty.getValue().put("type", "runtime"); - newProperties.put(runtimeProperty.getKey(), runtimeProperty.getValue()); - } - properties.clear(); - properties.putAll(newProperties); - return true; + protected boolean modifyMappingProperties(String index, Map properties, Map runtimeFields) { + return runtimeifyMappingProperties(properties, runtimeFields); } @Override diff --git a/x-pack/plugin/runtime-fields/qa/core-with-search/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/search/CoreTestsWithSearchRuntimeFieldsIT.java b/x-pack/plugin/runtime-fields/qa/core-with-search/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/search/CoreTestsWithSearchRuntimeFieldsIT.java index 69325a5729dd0..4539e436bb9e8 100644 --- a/x-pack/plugin/runtime-fields/qa/core-with-search/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/search/CoreTestsWithSearchRuntimeFieldsIT.java +++ b/x-pack/plugin/runtime-fields/qa/core-with-search/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/search/CoreTestsWithSearchRuntimeFieldsIT.java @@ -8,7 +8,6 @@ import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; - import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.xcontent.XContentHelper; @@ -57,9 +56,9 @@ protected Map dynamicTemplateFor(String type) { @Override protected Suite suite(ClientYamlTestCandidate candidate) { return new Suite(candidate) { - private Map>> runtimeMappingsAfterSetup; + private Map> runtimeMappingsAfterSetup; private Map> mappedFieldsAfterSetup; - private Map>> runtimeMappings; + private Map> runtimeMappings; private Map> mappedFields; @Override @@ -83,16 +82,12 @@ public boolean modifySections(List executables) { } @Override - protected boolean modifyMappingProperties(String index, Map properties) { - Map untouchedMapping = new HashMap<>(); - Map> runtimeMapping = new HashMap<>(); - if (false == runtimeifyMappingProperties(properties, untouchedMapping, runtimeMapping)) { + protected boolean modifyMappingProperties(String index, Map properties, Map runtimeFields) { + if (false == runtimeifyMappingProperties(properties, runtimeFields)) { return false; } - properties.clear(); - properties.putAll(untouchedMapping); - mappedFields.put(index, untouchedMapping.keySet()); - runtimeMappings.put(index, runtimeMapping); + mappedFields.put(index, properties.keySet()); + runtimeMappings.put(index, runtimeFields); return true; } @@ -116,6 +111,7 @@ protected boolean modifySearch(ApiCallSection search) { return mergeMappings(new String[] { "*" }); } String[] patterns = Arrays.stream(index.split(",")).map(m -> m.equals("_all") ? "*" : m).toArray(String[]::new); + // TODO this is always false? if (patterns.length == 0 && Regex.isSimpleMatchPattern(patterns[0])) { return runtimeMappings.get(patterns[0]); } @@ -123,13 +119,14 @@ protected boolean modifySearch(ApiCallSection search) { } private Map mergeMappings(String[] patterns) { - Map> merged = new HashMap<>(); - for (Map.Entry>> indexEntry : runtimeMappings.entrySet()) { + Map merged = new HashMap<>(); + for (Map.Entry> indexEntry : runtimeMappings.entrySet()) { if (false == Regex.simpleMatch(patterns, indexEntry.getKey())) { continue; } - for (Map.Entry> field : indexEntry.getValue().entrySet()) { - Map mergedConfig = merged.get(field.getKey()); + for (Map.Entry field : indexEntry.getValue().entrySet()) { + @SuppressWarnings("unchecked") + Map mergedConfig = (Map) merged.get(field.getKey()); if (mergedConfig == null) { merged.put(field.getKey(), field.getValue()); } else if (false == mergedConfig.equals(field.getValue())) { @@ -164,7 +161,7 @@ protected boolean handleIndex(IndexRequest index) { return false; } Map map = XContentHelper.convertToMap(index.source(), false, index.getContentType()).v2(); - Map> indexRuntimeMappings = runtimeMappings.computeIfAbsent( + Map indexRuntimeMappings = runtimeMappings.computeIfAbsent( index.index(), i -> new HashMap<>() ); diff --git a/x-pack/plugin/runtime-fields/qa/src/main/java/org/elasticsearch/xpack/runtimefields/test/CoreTestTranslater.java b/x-pack/plugin/runtime-fields/qa/src/main/java/org/elasticsearch/xpack/runtimefields/test/CoreTestTranslater.java index b034b572c7e6e..6fc52cb49510d 100644 --- a/x-pack/plugin/runtime-fields/qa/src/main/java/org/elasticsearch/xpack/runtimefields/test/CoreTestTranslater.java +++ b/x-pack/plugin/runtime-fields/qa/src/main/java/org/elasticsearch/xpack/runtimefields/test/CoreTestTranslater.java @@ -116,6 +116,7 @@ protected static Map dynamicTemplateToDisableRuntimeCompatibleFi return org.elasticsearch.common.collect.Map.of("type", type, "index", false, "doc_values", false); } + // TODO there isn't yet a way to create fields in the runtime section from a dynamic template protected static Map dynamicTemplateToAddRuntimeFields(String type) { return org.elasticsearch.common.collect.Map.ofEntries( org.elasticsearch.common.collect.Map.entry("type", "runtime"), @@ -280,25 +281,25 @@ private boolean modifyCreateIndex(ApiCallSection createIndex) { Object settings = body.get("settings"); if (settings instanceof Map && ((Map) settings).containsKey("sort.field")) { /* - * You can't sort the index on a runtime_keyword and it is - * hard to figure out if the sort was a runtime_keyword so - * let's just skip this test. + * You can't sort the index on a runtime field */ continue; } - Object mapping = body.get("mappings"); - if (false == (mapping instanceof Map)) { + @SuppressWarnings("unchecked") + Map mapping = (Map) body.get("mappings"); + if (mapping == null) { continue; } - Object properties = ((Map) mapping).get("properties"); - if (false == (properties instanceof Map)) { + @SuppressWarnings("unchecked") + Map propertiesMap = (Map) ((Map) mapping).get("properties"); + if (propertiesMap == null) { continue; } - @SuppressWarnings("unchecked") - Map propertiesMap = (Map) properties; - if (false == modifyMappingProperties(index, propertiesMap)) { + Map runtimeFields = new HashMap<>(); + if (false == modifyMappingProperties(index, propertiesMap, runtimeFields)) { return false; } + mapping.put("runtime", runtimeFields); } return true; } @@ -306,26 +307,21 @@ private boolean modifyCreateIndex(ApiCallSection createIndex) { /** * Modify the mapping defined in the test. */ - protected abstract boolean modifyMappingProperties(String index, Map properties); + protected abstract boolean modifyMappingProperties(String index, Map mappings, Map runtimeFields); /** * Modify the provided map in place, translating all fields into * runtime fields that load from source. * @return true if this mapping supports runtime fields, false otherwise */ - protected final boolean runtimeifyMappingProperties( - Map properties, - Map untouchedProperties, - Map> runtimeProperties - ) { + protected final boolean runtimeifyMappingProperties(Map properties, Map runtimeFields) { for (Map.Entry property : properties.entrySet()) { if (false == property.getValue() instanceof Map) { - untouchedProperties.put(property.getKey(), property.getValue()); continue; } @SuppressWarnings("unchecked") Map propertyMap = (Map) property.getValue(); - String name = property.getKey().toString(); + String name = property.getKey(); String type = Objects.toString(propertyMap.get("type")); if ("nested".equals(type)) { // Our loading scripts can't be made to manage nested fields so we have to skip those tests. @@ -333,40 +329,39 @@ protected final boolean runtimeifyMappingProperties( } if ("false".equals(Objects.toString(propertyMap.get("doc_values")))) { // If doc_values is false we can't emulate with scripts. So we keep the old definition. `null` and `true` are fine. - untouchedProperties.put(property.getKey(), property.getValue()); continue; } if ("false".equals(Objects.toString(propertyMap.get("index")))) { // If index is false we can't emulate with scripts - untouchedProperties.put(property.getKey(), property.getValue()); continue; } if ("true".equals(Objects.toString(propertyMap.get("store")))) { // If store is true we can't emulate with scripts - untouchedProperties.put(property.getKey(), property.getValue()); continue; } if (propertyMap.containsKey("ignore_above")) { // Scripts don't support ignore_above so we skip those fields - untouchedProperties.put(property.getKey(), property.getValue()); continue; } if (propertyMap.containsKey("ignore_malformed")) { // Our source reading script doesn't emulate ignore_malformed - untouchedProperties.put(property.getKey(), property.getValue()); continue; } String toLoad = painlessToLoadFromSource(name, type); if (toLoad == null) { - untouchedProperties.put(property.getKey(), property.getValue()); continue; } Map runtimeConfig = new HashMap<>(propertyMap); + runtimeConfig.put("type", type); runtimeConfig.put("script", toLoad); runtimeConfig.remove("store"); runtimeConfig.remove("index"); runtimeConfig.remove("doc_values"); - runtimeProperties.put(property.getKey(), runtimeConfig); + runtimeFields.put(name, runtimeConfig); + + // we disable the mapped fields and shadow them with their corresponding runtime field + propertyMap.put("doc_values", false); + propertyMap.put("index", false); } /* * Its tempting to return false here if we didn't make any runtime diff --git a/x-pack/plugin/runtime-fields/qa/with-security/src/javaRestTest/java/org/elasticsearch/xpack/security/PermissionsIT.java b/x-pack/plugin/runtime-fields/qa/with-security/src/javaRestTest/java/org/elasticsearch/xpack/security/PermissionsIT.java index 6c26fb4730929..a12439f9925f1 100644 --- a/x-pack/plugin/runtime-fields/qa/with-security/src/javaRestTest/java/org/elasticsearch/xpack/security/PermissionsIT.java +++ b/x-pack/plugin/runtime-fields/qa/with-security/src/javaRestTest/java/org/elasticsearch/xpack/security/PermissionsIT.java @@ -68,13 +68,14 @@ public void testDLS() throws IOException { createIndex.setJsonEntity( "{\n" + " \"mappings\" : {\n" - + " \"properties\" : {\n" - + " \"date\" : {\"type\" : \"keyword\"},\n" + + " \"runtime\" : {\n" + " \"year\" : {\n" - + " \"type\" : \"runtime\", \n" - + " \"runtime_type\" : \"keyword\",\n" + + " \"type\" : \"keyword\", \n" + " \"script\" : \"emit(doc['date'].value.substring(0,4))\"\n" + " }\n" + + " },\n" + + " \"properties\" : {\n" + + " \"date\" : {\"type\" : \"keyword\"}\n" + " }\n" + " }\n" + "}\n" @@ -110,13 +111,14 @@ public void testFLSProtectsData() throws IOException { createIndex.setJsonEntity( "{\n" + " \"mappings\" : {\n" - + " \"properties\" : {\n" - + " \"hidden\" : {\"type\" : \"keyword\"},\n" + + " \"runtime\" : {\n" + " \"hidden_values_count\" : {\n" - + " \"type\" : \"runtime\", \n" - + " \"runtime_type\" : \"long\",\n" + + " \"type\" : \"long\", \n" + " \"script\" : \"emit(doc['hidden'].size())\"\n" + " }\n" + + " },\n" + + " \"properties\" : {\n" + + " \"hidden\" : {\"type\" : \"keyword\"}\n" + " }\n" + " }\n" + "}\n" @@ -159,13 +161,14 @@ public void testFLSOnRuntimeField() throws IOException { createIndex.setJsonEntity( "{\n" + " \"mappings\" : {\n" - + " \"properties\" : {\n" - + " \"date\" : {\"type\" : \"keyword\"},\n" + + " \"runtime\" : {\n" + " \"year\" : {\n" - + " \"type\" : \"runtime\", \n" - + " \"runtime_type\" : \"keyword\",\n" + + " \"type\" : \"keyword\", \n" + " \"script\" : \"emit(doc['date'].value.substring(0,4))\"\n" + " }\n" + + " },\n" + + " \"properties\" : {\n" + + " \"date\" : {\"type\" : \"keyword\"}\n" + " }\n" + " }\n" + "}\n" diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java index 45c08120c400a..e074badf47611 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java @@ -6,29 +6,48 @@ package org.elasticsearch.xpack.runtimefields; -import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.index.mapper.BooleanFieldMapper; +import org.elasticsearch.index.mapper.DateFieldMapper; +import org.elasticsearch.index.mapper.GeoPointFieldMapper; +import org.elasticsearch.index.mapper.IpFieldMapper; +import org.elasticsearch.index.mapper.KeywordFieldMapper; +import org.elasticsearch.index.mapper.NumberFieldMapper; +import org.elasticsearch.index.mapper.RuntimeFieldType; import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.ScriptPlugin; import org.elasticsearch.script.ScriptContext; import org.elasticsearch.xpack.runtimefields.mapper.BooleanFieldScript; +import org.elasticsearch.xpack.runtimefields.mapper.BooleanScriptFieldType; import org.elasticsearch.xpack.runtimefields.mapper.DateFieldScript; +import org.elasticsearch.xpack.runtimefields.mapper.DateScriptFieldType; import org.elasticsearch.xpack.runtimefields.mapper.DoubleFieldScript; +import org.elasticsearch.xpack.runtimefields.mapper.DoubleScriptFieldType; import org.elasticsearch.xpack.runtimefields.mapper.GeoPointFieldScript; +import org.elasticsearch.xpack.runtimefields.mapper.GeoPointScriptFieldType; import org.elasticsearch.xpack.runtimefields.mapper.IpFieldScript; +import org.elasticsearch.xpack.runtimefields.mapper.IpScriptFieldType; +import org.elasticsearch.xpack.runtimefields.mapper.KeywordScriptFieldType; import org.elasticsearch.xpack.runtimefields.mapper.LongFieldScript; -import org.elasticsearch.xpack.runtimefields.mapper.RuntimeFieldMapper; +import org.elasticsearch.xpack.runtimefields.mapper.LongScriptFieldType; import org.elasticsearch.xpack.runtimefields.mapper.StringFieldScript; -import java.util.Collections; import java.util.List; import java.util.Map; public final class RuntimeFields extends Plugin implements MapperPlugin, ScriptPlugin { @Override - public Map getMappers() { - return Collections.singletonMap(RuntimeFieldMapper.CONTENT_TYPE, RuntimeFieldMapper.PARSER); + public Map getRuntimeFieldTypes() { + return Map.ofEntries( + Map.entry(BooleanFieldMapper.CONTENT_TYPE, BooleanScriptFieldType.PARSER), + Map.entry(NumberFieldMapper.NumberType.LONG.typeName(), LongScriptFieldType.PARSER), + Map.entry(NumberFieldMapper.NumberType.DOUBLE.typeName(), DoubleScriptFieldType.PARSER), + Map.entry(IpFieldMapper.CONTENT_TYPE, IpScriptFieldType.PARSER), + Map.entry(DateFieldMapper.CONTENT_TYPE, DateScriptFieldType.PARSER), + Map.entry(KeywordFieldMapper.CONTENT_TYPE, KeywordScriptFieldType.PARSER), + Map.entry(GeoPointFieldMapper.CONTENT_TYPE, GeoPointScriptFieldType.PARSER) + ); } @Override diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/AbstractScriptFieldType.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/AbstractScriptFieldType.java index 0feb612f82018..8d3a0671e065f 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/AbstractScriptFieldType.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/AbstractScriptFieldType.java @@ -12,52 +12,58 @@ import org.apache.lucene.search.spans.SpanMultiTermQueryWrapper; import org.apache.lucene.search.spans.SpanQuery; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.common.CheckedBiConsumer; import org.elasticsearch.common.TriFunction; import org.elasticsearch.common.geo.ShapeRelation; import org.elasticsearch.common.time.DateMathParser; import org.elasticsearch.common.unit.Fuzziness; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.index.mapper.ContentPath; import org.elasticsearch.index.mapper.DocValueFetcher; +import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.mapper.TextSearchInfo; +import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.index.mapper.MapperParsingException; +import org.elasticsearch.index.mapper.RuntimeFieldType; import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.script.Script; +import org.elasticsearch.script.ScriptType; import org.elasticsearch.search.lookup.SearchLookup; +import java.io.IOException; import java.time.ZoneId; +import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.function.BiFunction; +import java.util.function.Function; import static org.elasticsearch.search.SearchService.ALLOW_EXPENSIVE_QUERIES; /** * Abstract base {@linkplain MappedFieldType} for scripted fields. */ -abstract class AbstractScriptFieldType extends MappedFieldType { +abstract class AbstractScriptFieldType extends RuntimeFieldType { protected final Script script; private final TriFunction, SearchLookup, LeafFactory> factory; + private final CheckedBiConsumer toXContent; + + AbstractScriptFieldType(String name, TriFunction, SearchLookup, LeafFactory> factory, Builder builder) { + this(name, factory, builder.script.getValue(), builder.meta.getValue(), builder::toXContent); + } AbstractScriptFieldType( String name, - Script script, TriFunction, SearchLookup, LeafFactory> factory, - Map meta + Script script, + Map meta, + CheckedBiConsumer toXContent ) { - super(name, false, false, false, TextSearchInfo.SIMPLE_MATCH_WITHOUT_TERMS, meta); - this.script = script; + super(name, meta); this.factory = factory; - } - - protected abstract String runtimeType(); - - @Override - public final String typeName() { - return RuntimeFieldMapper.CONTENT_TYPE; - } - - @Override - public final String familyTypeName() { - return runtimeType(); + this.script = script; + this.toXContent = toXContent; } @Override @@ -101,8 +107,8 @@ public final Query rangeQuery( QueryShardContext context ) { if (relation == ShapeRelation.DISJOINT) { - String message = "Field [%s] of type [%s] with runtime type [%s] does not support DISJOINT ranges"; - throw new IllegalArgumentException(String.format(Locale.ROOT, message, name(), typeName(), runtimeType())); + String message = "Runtime field [%s] of type [%s] does not support DISJOINT ranges"; + throw new IllegalArgumentException(String.format(Locale.ROOT, message, name(), typeName())); } return rangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, timeZone, parser, context); } @@ -174,46 +180,115 @@ public SpanQuery spanPrefixQuery(String value, SpanMultiTermQueryWrapper.SpanRew private String unsupported(String query, String supported) { return String.format( Locale.ROOT, - "Can only use %s queries on %s fields - not on [%s] which is of type [%s] with runtime_type [%s]", + "Can only use %s queries on %s fields - not on [%s] which is a runtime field of type [%s]", query, supported, name(), - RuntimeFieldMapper.CONTENT_TYPE, - runtimeType() + typeName() ); } protected final void checkAllowExpensiveQueries(QueryShardContext context) { if (context.allowExpensiveQueries() == false) { throw new ElasticsearchException( - "queries cannot be executed against [" - + RuntimeFieldMapper.CONTENT_TYPE - + "] fields while [" - + ALLOW_EXPENSIVE_QUERIES.getKey() - + "] is set to [false]." + "queries cannot be executed against runtime fields while [" + ALLOW_EXPENSIVE_QUERIES.getKey() + "] is set to [false]." ); } } - /** - * The format that this field should use. The default implementation is - * {@code null} because most fields don't support formats. - */ - protected String format() { - return null; + @Override + public ValueFetcher valueFetcher(QueryShardContext context, SearchLookup lookup, String format) { + return new DocValueFetcher(docValueFormat(format, null), lookup.doc().getForField(this)); + } + + @Override + protected final void doXContentBody(XContentBuilder builder, boolean includeDefaults) throws IOException { + toXContent.accept(builder, includeDefaults); } /** - * The locale that this field's format should use. The default - * implementation is {@code null} because most fields don't - * support formats. + * For runtime fields the {@link RuntimeFieldType.Parser} returns directly the {@link MappedFieldType}. + * Internally we still create a {@link Builder} so we reuse the {@link FieldMapper.Parameter} infrastructure, + * but {@link Builder#init(FieldMapper)} and {@link Builder#build(ContentPath)} are never called as + * {@link RuntimeFieldTypeParser#parse(String, Map, Mapper.TypeParser.ParserContext)} calls + * {@link Builder#parse(String, Mapper.TypeParser.ParserContext, Map)} and returns the corresponding + * {@link MappedFieldType}. */ - protected Locale formatLocale() { - return null; + abstract static class Builder extends FieldMapper.Builder { + final FieldMapper.Parameter> meta = FieldMapper.Parameter.metaParam(); + final FieldMapper.Parameter