diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java b/server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java index 0b66f69cfebf0..c6d77d7b7d26e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java @@ -29,6 +29,7 @@ import java.io.IOException; import java.time.format.DateTimeParseException; +import java.util.Map; /** * Encapsulates the logic for dynamically creating fields as part of document parsing. @@ -144,7 +145,7 @@ Mapper createDynamicObjectMapper(ParseContext context, String name) { * Note that objects are always mapped under properties. */ Mapper createObjectMapperFromTemplate(ParseContext context, String name) { - Mapper.Builder templateBuilder = findTemplateBuilder(context, name, DynamicTemplate.XContentFieldType.OBJECT, null); + Mapper.Builder templateBuilder = findTemplateBuilderForObject(context, name); return templateBuilder == null ? null : templateBuilder.build(context.path()); } @@ -176,38 +177,70 @@ private static void createDynamicField(ParseContext context, DynamicTemplate.XContentFieldType matchType, DateFormatter dateFormatter, CheckedRunnable dynamicFieldStrategy) throws IOException { - Mapper.Builder templateBuilder = findTemplateBuilder(context, name, matchType, dateFormatter); - if (templateBuilder == null) { + if (applyMatchingTemplate(context, name, matchType, dateFormatter) == false) { dynamicFieldStrategy.run(); - } else { - CONCRETE.createDynamicField(templateBuilder, context); } } /** - * Find a template. Returns {@code null} if no template could be found. + * Find and apply a matching dynamic template. Returns {@code true} if a template could be found, {@code false} otherwise. * @param context the parse context for this document * @param name the current field name * @param matchType the type of the field in the json document or null if unknown * @param dateFormatter a date formatter to use if the type is a date, null if not a date or is using the default format - * @return a mapper builder, or null if there is no template for such a field + * @return true if a template was found and applied, false otherwise */ - private static Mapper.Builder findTemplateBuilder(ParseContext context, - String name, - DynamicTemplate.XContentFieldType matchType, - DateFormatter dateFormatter) { + private static boolean applyMatchingTemplate(ParseContext context, + String name, + DynamicTemplate.XContentFieldType matchType, + DateFormatter dateFormatter) throws IOException { + DynamicTemplate dynamicTemplate = context.root().findTemplate(context.path(), name, matchType); + if (dynamicTemplate == null) { + return false; + } + String dynamicType = dynamicTemplate.isRuntimeMapping() ? matchType.defaultRuntimeMappingType() : matchType.defaultMappingType(); + + String mappingType = dynamicTemplate.mappingType(dynamicType); + Map mapping = dynamicTemplate.mappingForName(name, dynamicType); + if (dynamicTemplate.isRuntimeMapping()) { + Mapper.TypeParser.ParserContext parserContext = context.parserContext(dateFormatter); + RuntimeFieldType.Parser parser = parserContext.runtimeFieldTypeParser(mappingType); + String fullName = context.path().pathAsText(name); + if (parser == null) { + throw new MapperParsingException("failed to find type parsed [" + mappingType + "] for [" + fullName + "]"); + } + RuntimeFieldType runtimeFieldType = parser.parse(fullName, mapping, parserContext); + Runtime.createDynamicField(runtimeFieldType, context); + } else { + Mapper.Builder builder = parseMapping(name, mappingType, mapping, dateFormatter, context); + CONCRETE.createDynamicField(builder, context); + } + return true; + } + + private static Mapper.Builder findTemplateBuilderForObject(ParseContext context, String name) { + DynamicTemplate.XContentFieldType matchType = DynamicTemplate.XContentFieldType.OBJECT; DynamicTemplate dynamicTemplate = context.root().findTemplate(context.path(), name, matchType); if (dynamicTemplate == null) { return null; } String dynamicType = matchType.defaultMappingType(); - Mapper.TypeParser.ParserContext parserContext = context.parserContext(dateFormatter); String mappingType = dynamicTemplate.mappingType(dynamicType); + Map mapping = dynamicTemplate.mappingForName(name, dynamicType); + return parseMapping(name, mappingType, mapping, null, context); + } + + private static Mapper.Builder parseMapping(String name, + String mappingType, + Map mapping, + DateFormatter dateFormatter, + ParseContext context) { + Mapper.TypeParser.ParserContext parserContext = context.parserContext(dateFormatter); Mapper.TypeParser typeParser = parserContext.typeParser(mappingType); if (typeParser == null) { throw new MapperParsingException("failed to find type parsed [" + mappingType + "] for [" + name + "]"); } - return typeParser.parse(name, dynamicTemplate.mappingForName(name, dynamicType), parserContext); + return typeParser.parse(name, mapping, parserContext); } /** @@ -284,39 +317,43 @@ void newDynamicBinaryField(ParseContext context, String name) throws IOException * @see Dynamic */ private static final class Runtime implements Strategy { + static void createDynamicField(RuntimeFieldType runtimeFieldType, ParseContext context) { + context.addDynamicRuntimeField(runtimeFieldType); + } + @Override public void newDynamicStringField(ParseContext context, String name) { String fullName = context.path().pathAsText(name); RuntimeFieldType runtimeFieldType = context.getDynamicRuntimeFieldsBuilder().newDynamicStringField(fullName); - context.addDynamicRuntimeField(runtimeFieldType); + createDynamicField(runtimeFieldType, context); } @Override public void newDynamicLongField(ParseContext context, String name) { String fullName = context.path().pathAsText(name); RuntimeFieldType runtimeFieldType = context.getDynamicRuntimeFieldsBuilder().newDynamicLongField(fullName); - context.addDynamicRuntimeField(runtimeFieldType); + createDynamicField(runtimeFieldType, context); } @Override public void newDynamicDoubleField(ParseContext context, String name) { String fullName = context.path().pathAsText(name); RuntimeFieldType runtimeFieldType = context.getDynamicRuntimeFieldsBuilder().newDynamicDoubleField(fullName); - context.addDynamicRuntimeField(runtimeFieldType); + createDynamicField(runtimeFieldType, context); } @Override public void newDynamicBooleanField(ParseContext context, String name) { String fullName = context.path().pathAsText(name); RuntimeFieldType runtimeFieldType = context.getDynamicRuntimeFieldsBuilder().newDynamicBooleanField(fullName); - context.addDynamicRuntimeField(runtimeFieldType); + createDynamicField(runtimeFieldType, context); } @Override public void newDynamicDateField(ParseContext context, String name, DateFormatter dateFormatter) { String fullName = context.path().pathAsText(name); RuntimeFieldType runtimeFieldType = context.getDynamicRuntimeFieldsBuilder().newDynamicDateField(fullName, dateFormatter); - context.addDynamicRuntimeField(runtimeFieldType); + createDynamicField(runtimeFieldType, context); } } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DynamicTemplate.java b/server/src/main/java/org/elasticsearch/index/mapper/DynamicTemplate.java index 79f33da17bd04..17c4d9db4787c 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DynamicTemplate.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DynamicTemplate.java @@ -30,6 +30,7 @@ import java.util.Arrays; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.TreeMap; @@ -76,12 +77,8 @@ public static MatchType fromString(String value) { public enum XContentFieldType { OBJECT { @Override - public String defaultMappingType() { - return ObjectMapper.CONTENT_TYPE; - } - @Override - public String toString() { - return "object"; + boolean supportsRuntimeField() { + return false; } }, STRING { @@ -90,58 +87,23 @@ public String defaultMappingType() { return TextFieldMapper.CONTENT_TYPE; } @Override - public String toString() { - return "string"; - } - }, - LONG { - @Override - public String defaultMappingType() { - return NumberFieldMapper.NumberType.LONG.typeName(); - } - @Override - public String toString() { - return "long"; + String defaultRuntimeMappingType() { + return KeywordFieldMapper.CONTENT_TYPE; } }, + LONG, DOUBLE { @Override public String defaultMappingType() { return NumberFieldMapper.NumberType.FLOAT.typeName(); } - @Override - public String toString() { - return "double"; - } - }, - BOOLEAN { - @Override - public String defaultMappingType() { - return BooleanFieldMapper.CONTENT_TYPE; - } - @Override - public String toString() { - return "boolean"; - } - }, - DATE { - @Override - public String defaultMappingType() { - return DateFieldMapper.CONTENT_TYPE; - } - @Override - public String toString() { - return "date"; - } }, + BOOLEAN, + DATE, BINARY { @Override - public String defaultMappingType() { - return BinaryFieldMapper.CONTENT_TYPE; - } - @Override - public String toString() { - return "binary"; + boolean supportsRuntimeField() { + return false; } }; @@ -155,17 +117,47 @@ public static XContentFieldType fromString(String value) { + Arrays.toString(values())); } - /** The default mapping type to use for fields of this {@link XContentFieldType}. */ - public abstract String defaultMappingType(); + /** + * The default mapping type to use for fields of this {@link XContentFieldType}. + * By default, the lowercase field type is used. + */ + String defaultMappingType() { + return toString(); + } + + /** + * The default mapping type to use for fields of this {@link XContentFieldType} when defined as runtime fields + * By default, the lowercase field type is used. + */ + String defaultRuntimeMappingType() { + return toString(); + } + + /** + * Returns true if the field type supported as runtime field, false otherwise. + * Whenever a match_mapping_type has not been defined in a dynamic template, if a runtime mapping has been specified only + * field types that are supported as runtime field will match the template. + * Also, it is not possible to define a dynamic template that defines a runtime field and explicitly matches a type that + * is not supported as runtime field. + */ + boolean supportsRuntimeField() { + return true; + } + + @Override + public final String toString() { + return name().toLowerCase(Locale.ROOT); + } } - public static DynamicTemplate parse(String name, Map conf, - Version indexVersionCreated) throws MapperParsingException { + @SuppressWarnings("unchecked") + static DynamicTemplate parse(String name, Map conf, Version indexVersionCreated) throws MapperParsingException { String match = null; String pathMatch = null; String unmatch = null; String pathUnmatch = null; Map mapping = null; + boolean runtime = false; String matchMappingType = null; String matchPattern = MatchType.SIMPLE.toString(); @@ -184,7 +176,20 @@ public static DynamicTemplate parse(String name, Map conf, } else if ("match_pattern".equals(propName)) { matchPattern = entry.getValue().toString(); } else if ("mapping".equals(propName)) { + if (mapping != null) { + throw new MapperParsingException("mapping and runtime cannot be both specified in the same dynamic template [" + + name + "]"); + } + mapping = (Map) entry.getValue(); + runtime = false; + } else if ("runtime".equals(propName)) { + if (mapping != null) { + throw new MapperParsingException("mapping and runtime cannot be both specified in the same dynamic template [" + + name + "]"); + + } mapping = (Map) entry.getValue(); + runtime = true; } else { // unknown parameters were ignored before but still carried through serialization // so we need to ignore them at parsing time for old indices @@ -196,13 +201,17 @@ public static DynamicTemplate parse(String name, Map conf, throw new MapperParsingException("template must have match, path_match or match_mapping_type set " + conf.toString()); } if (mapping == null) { - throw new MapperParsingException("template must have mapping set"); + throw new MapperParsingException("template [" + name + "] must have either mapping or runtime set"); } XContentFieldType xcontentFieldType = null; if (matchMappingType != null && matchMappingType.equals("*") == false) { try { xcontentFieldType = XContentFieldType.fromString(matchMappingType); + if (runtime && xcontentFieldType.supportsRuntimeField() == false) { + throw new MapperParsingException("Dynamic template [" + name + "] defines a runtime field but type [" + + xcontentFieldType + "] is not supported as runtime field"); + } } catch (IllegalArgumentException e) { if (indexVersionCreated.onOrAfter(Version.V_6_0_0_alpha1)) { throw e; @@ -234,27 +243,21 @@ public static DynamicTemplate parse(String name, Map conf, } } - return new DynamicTemplate(name, pathMatch, pathUnmatch, match, unmatch, xcontentFieldType, matchType, mapping); + return new DynamicTemplate(name, pathMatch, pathUnmatch, match, unmatch, xcontentFieldType, matchType, mapping, runtime); } private final String name; - private final String pathMatch; - private final String pathUnmatch; - private final String match; - private final String unmatch; - private final MatchType matchType; - private final XContentFieldType xcontentFieldType; - private final Map mapping; + private final boolean runtimeMapping; private DynamicTemplate(String name, String pathMatch, String pathUnmatch, String match, String unmatch, - XContentFieldType xcontentFieldType, MatchType matchType, Map mapping) { + XContentFieldType xcontentFieldType, MatchType matchType, Map mapping, boolean runtimeMapping) { this.name = name; this.pathMatch = pathMatch; this.pathUnmatch = pathUnmatch; @@ -263,6 +266,7 @@ private DynamicTemplate(String name, String pathMatch, String pathUnmatch, Strin this.matchType = matchType; this.xcontentFieldType = xcontentFieldType; this.mapping = mapping; + this.runtimeMapping = runtimeMapping; } public String name() { @@ -289,6 +293,9 @@ public boolean match(String path, String name, XContentFieldType xcontentFieldTy if (this.xcontentFieldType != null && this.xcontentFieldType != xcontentFieldType) { return false; } + if (runtimeMapping && xcontentFieldType.supportsRuntimeField() == false) { + return false; + } return true; } @@ -302,7 +309,7 @@ public String mappingType(String dynamicType) { type = dynamicType; } if (type.equals(mapping.get("type")) == false // either the type was not set, or we updated it through replacements - && "text".equals(type)) { // and the result is "text" + && TextFieldMapper.CONTENT_TYPE.equals(type)) { // and the result is "text" // now that string has been splitted into text and keyword, we use text for // dynamic mappings. However before it used to be possible to index as a keyword // by setting index=not_analyzed, so for now we will use a keyword field rather @@ -312,52 +319,52 @@ public String mappingType(String dynamicType) { // TODO: how to do it in the future? final Object index = mapping.get("index"); if ("not_analyzed".equals(index) || "no".equals(index)) { - type = "keyword"; + return KeywordFieldMapper.CONTENT_TYPE; } } return type; } + public boolean isRuntimeMapping() { + return runtimeMapping; + } + public Map mappingForName(String name, String dynamicType) { return processMap(mapping, name, dynamicType); } - private Map processMap(Map map, String name, String dynamicType) { + private static Map processMap(Map map, String name, String dynamicType) { Map processedMap = new HashMap<>(); for (Map.Entry entry : map.entrySet()) { String key = entry.getKey().replace("{name}", name).replace("{dynamic_type}", dynamicType) .replace("{dynamicType}", dynamicType); - Object value = entry.getValue(); - if (value instanceof Map) { - value = processMap((Map) value, name, dynamicType); - } else if (value instanceof List) { - value = processList((List) value, name, dynamicType); - } else if (value instanceof String) { - value = value.toString().replace("{name}", name).replace("{dynamic_type}", dynamicType) - .replace("{dynamicType}", dynamicType); - } - processedMap.put(key, value); + processedMap.put(key, extractValue(entry.getValue(), name, dynamicType)); } return processedMap; } - private List processList(List list, String name, String dynamicType) { - List processedList = new ArrayList(list.size()); + private static List processList(List list, String name, String dynamicType) { + List processedList = new ArrayList<>(list.size()); for (Object value : list) { - if (value instanceof Map) { - value = processMap((Map) value, name, dynamicType); - } else if (value instanceof List) { - value = processList((List) value, name, dynamicType); - } else if (value instanceof String) { - value = value.toString().replace("{name}", name) - .replace("{dynamic_type}", dynamicType) - .replace("{dynamicType}", dynamicType); - } - processedList.add(value); + processedList.add(extractValue(value, name, dynamicType)); } return processedList; } + @SuppressWarnings("unchecked") + private static Object extractValue(Object value, String name, String dynamicType) { + if (value instanceof Map) { + return processMap((Map) value, name, dynamicType); + } else if (value instanceof List) { + return processList((List) value, name, dynamicType); + } else if (value instanceof String) { + return value.toString().replace("{name}", name) + .replace("{dynamic_type}", dynamicType) + .replace("{dynamicType}", dynamicType); + } + return value; + } + String getName() { return name; } @@ -394,7 +401,11 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field("match_pattern", matchType); } // use a sorted map for consistent serialization - builder.field("mapping", new TreeMap<>(mapping)); + if (runtimeMapping) { + builder.field("runtime", new TreeMap<>(mapping)); + } else { + builder.field("mapping", new TreeMap<>(mapping)); + } builder.endObject(); return builder; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java index 6e54f77b49245..dfe4b9891bd4c 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java @@ -42,6 +42,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.function.BiConsumer; import java.util.stream.Collectors; import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeBooleanValue; @@ -384,16 +385,19 @@ protected void doXContent(XContentBuilder builder, ToXContent.Params params) thr } private static void validateDynamicTemplate(Mapper.TypeParser.ParserContext parserContext, - DynamicTemplate dynamicTemplate) { + DynamicTemplate template) { - if (containsSnippet(dynamicTemplate.getMapping(), "{name}")) { + if (containsSnippet(template.getMapping(), "{name}")) { // Can't validate template, because field names can't be guessed up front. return; } final XContentFieldType[] types; - if (dynamicTemplate.getXContentFieldType() != null) { - types = new XContentFieldType[]{dynamicTemplate.getXContentFieldType()}; + if (template.getXContentFieldType() != null) { + types = new XContentFieldType[]{template.getXContentFieldType()}; + } else if (template.isRuntimeMapping()) { + types = Arrays.stream(XContentFieldType.values()).filter(XContentFieldType::supportsRuntimeField) + .toArray(XContentFieldType[]::new); } else { types = XContentFieldType.values(); } @@ -401,28 +405,29 @@ private static void validateDynamicTemplate(Mapper.TypeParser.ParserContext pars Exception lastError = null; boolean dynamicTemplateInvalid = true; - for (XContentFieldType contentFieldType : types) { - String defaultDynamicType = contentFieldType.defaultMappingType(); - String mappingType = dynamicTemplate.mappingType(defaultDynamicType); - Mapper.TypeParser typeParser = parserContext.typeParser(mappingType); - if (typeParser == null) { - lastError = new IllegalArgumentException("No mapper found for type [" + mappingType + "]"); - continue; - } - - String templateName = "__dynamic__" + dynamicTemplate.name(); - Map fieldTypeConfig = dynamicTemplate.mappingForName(templateName, defaultDynamicType); + for (XContentFieldType fieldType : types) { + String dynamicType = template.isRuntimeMapping() ? fieldType.defaultRuntimeMappingType() : fieldType.defaultMappingType(); + String mappingType = template.mappingType(dynamicType); try { - Mapper.Builder dummyBuilder = typeParser.parse(templateName, fieldTypeConfig, parserContext); - fieldTypeConfig.remove("type"); - if (fieldTypeConfig.isEmpty()) { - dummyBuilder.build(new ContentPath(1)); - dynamicTemplateInvalid = false; - break; + if (template.isRuntimeMapping()) { + RuntimeFieldType.Parser parser = parserContext.runtimeFieldTypeParser(mappingType); + if (parser == null) { + lastError = new IllegalArgumentException("No runtime field found for type [" + mappingType + "]"); + continue; + } + validate(template, dynamicType, (name, mapping) -> parser.parse(name, mapping, parserContext)); } else { - lastError = new IllegalArgumentException("Unused mapping attributes [" + fieldTypeConfig + "]"); + Mapper.TypeParser typeParser = parserContext.typeParser(mappingType); + if (typeParser == null) { + lastError = new IllegalArgumentException("No mapper found for type [" + mappingType + "]"); + continue; + } + validate(template, dynamicType, + (name, mapping) -> typeParser.parse(name, mapping, parserContext).build(new ContentPath(1))); } - } catch (Exception e) { + dynamicTemplateInvalid = false; + break; + } catch(Exception e) { lastError = e; } } @@ -430,8 +435,8 @@ private static void validateDynamicTemplate(Mapper.TypeParser.ParserContext pars final boolean shouldEmitDeprecationWarning = parserContext.indexVersionCreated().onOrAfter(Version.V_7_7_0); if (dynamicTemplateInvalid && shouldEmitDeprecationWarning) { String format = "dynamic template [%s] has invalid content [%s], " + - "attempted to validate it with the following match_mapping_type: [%s]"; - String message = String.format(Locale.ROOT, format, dynamicTemplate.getName(), Strings.toString(dynamicTemplate), + "attempted to validate it with the following match_mapping_type: %s"; + String message = String.format(Locale.ROOT, format, template.getName(), Strings.toString(template), Arrays.toString(types)); final String deprecationMessage; @@ -440,7 +445,19 @@ private static void validateDynamicTemplate(Mapper.TypeParser.ParserContext pars } else { deprecationMessage = message; } - DEPRECATION_LOGGER.deprecate("invalid_dynamic_template", deprecationMessage); + DEPRECATION_LOGGER.deprecate("invalid_dynamic_template", deprecationMessage); + } + } + + private static void validate(DynamicTemplate template, + String dynamicType, + BiConsumer> mappingConsumer) { + String templateName = "__dynamic__" + template.name(); + Map fieldTypeConfig = template.mappingForName(templateName, dynamicType); + mappingConsumer.accept(templateName, fieldTypeConfig); + fieldTypeConfig.remove("type"); + if (fieldTypeConfig.isEmpty() == false) { + throw new IllegalArgumentException("Unknown mapping attributes [" + fieldTypeConfig + "]"); } } @@ -450,21 +467,9 @@ private static boolean containsSnippet(Map map, String snippet) { if (key.contains(snippet)) { return true; } - Object value = entry.getValue(); - if (value instanceof Map) { - if (containsSnippet((Map) value, snippet)) { - return true; - } - } else if (value instanceof List) { - if (containsSnippet((List) value, snippet)) { - return true; - } - } else if (value instanceof String) { - String valueString = (String) value; - if (valueString.contains(snippet)) { - return true; - } + if (containsSnippet(value, snippet)) { + return true; } } @@ -473,21 +478,21 @@ private static boolean containsSnippet(Map map, String snippet) { private static boolean containsSnippet(List list, String snippet) { for (Object value : list) { - if (value instanceof Map) { - if (containsSnippet((Map) value, snippet)) { - return true; - } - } else if (value instanceof List) { - if (containsSnippet((List) value, snippet)) { - return true; - } - } else if (value instanceof String) { - String valueString = (String) value; - if (valueString.contains(snippet)) { - return true; - } + if (containsSnippet(value, snippet)) { + return true; } } return false; } + + private static boolean containsSnippet(Object value, String snippet) { + if (value instanceof Map) { + return containsSnippet((Map) value, snippet); + } else if (value instanceof List) { + return containsSnippet((List) value, snippet); + } else if (value instanceof String) { + return ((String) value).contains(snippet); + } + return false; + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java index 75532e1d86eb5..f6f5e0e490450 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java @@ -61,7 +61,7 @@ protected Collection getPlugins() { } public void testParseWithRuntimeField() throws Exception { - DocumentMapper mapper = createDocumentMapper(runtimeFieldMapping(b -> b.field("type", "string"))); + DocumentMapper mapper = createDocumentMapper(runtimeFieldMapping(b -> b.field("type", "keyword"))); ParsedDocument doc = mapper.parse(source(b -> b.field("field", "value"))); //field defined as runtime field but not under properties: no dynamic updates, the field does not get indexed assertNull(doc.dynamicMappingsUpdate()); @@ -69,7 +69,7 @@ public void testParseWithRuntimeField() throws Exception { } public void testParseWithRuntimeFieldArray() throws Exception { - DocumentMapper mapper = createDocumentMapper(runtimeFieldMapping(b -> b.field("type", "string"))); + DocumentMapper mapper = createDocumentMapper(runtimeFieldMapping(b -> b.field("type", "keyword"))); ParsedDocument doc = mapper.parse(source(b -> b.array("field", "value1", "value2"))); //field defined as runtime field but not under properties: no dynamic updates, the field does not get indexed assertNull(doc.dynamicMappingsUpdate()); @@ -79,7 +79,7 @@ public void testParseWithRuntimeFieldArray() throws Exception { public void testParseWithShadowedField() throws Exception { XContentBuilder builder = XContentFactory.jsonBuilder().startObject().startObject("_doc"); builder.startObject("runtime"); - builder.startObject("field").field("type", "string").endObject(); + builder.startObject("field").field("type", "keyword").endObject(); builder.endObject(); builder.startObject("properties"); builder.startObject("field").field("type", "keyword").endObject(); @@ -95,7 +95,7 @@ public void testParseWithShadowedField() throws Exception { public void testParseWithRuntimeFieldDottedNameDisabledObject() throws Exception { XContentBuilder builder = XContentFactory.jsonBuilder().startObject().startObject("_doc"); builder.startObject("runtime"); - builder.startObject("path1.path2.path3.field").field("type", "string").endObject(); + builder.startObject("path1.path2.path3.field").field("type", "keyword").endObject(); builder.endObject(); builder.startObject("properties"); builder.startObject("path1").field("type", "object").field("enabled", false).endObject(); @@ -113,7 +113,7 @@ public void testParseWithRuntimeFieldDottedNameDisabledObject() throws Exception public void testParseWithShadowedSubField() throws Exception { XContentBuilder builder = XContentFactory.jsonBuilder().startObject().startObject("_doc"); builder.startObject("runtime"); - builder.startObject("field.keyword").field("type", "string").endObject(); + builder.startObject("field.keyword").field("type", "keyword").endObject(); builder.endObject(); builder.startObject("properties"); builder.startObject("field").field("type", "text"); @@ -131,7 +131,7 @@ public void testParseWithShadowedSubField() throws Exception { public void testParseWithShadowedMultiField() throws Exception { XContentBuilder builder = XContentFactory.jsonBuilder().startObject().startObject("_doc"); builder.startObject("runtime"); - builder.startObject("field").field("type", "string").endObject(); + builder.startObject("field").field("type", "keyword").endObject(); builder.endObject(); builder.startObject("properties"); builder.startObject("field").field("type", "text"); @@ -151,7 +151,7 @@ public void testRuntimeFieldAndArrayChildren() throws IOException { b.field("dynamic", "true"); b.startObject("runtime"); { - b.startObject("object").field("type", "string").endObject(); + b.startObject("object").field("type", "keyword").endObject(); } b.endObject(); })); @@ -186,8 +186,8 @@ public void testRuntimeFieldDoesNotShadowObjectChildren() throws IOException { b.field("dynamic", "true"); b.startObject("runtime"); { - b.startObject("location").field("type", "string").endObject(); - b.startObject("country").field("type", "string").endObject(); + b.startObject("location").field("type", "keyword").endObject(); + b.startObject("country").field("type", "keyword").endObject(); } b.endObject(); b.startObject("properties"); @@ -526,7 +526,7 @@ public void testPropagateDynamicRuntimeWithDynamicMapper() throws Exception { })); assertNull(doc.rootDoc().getField("foo.bar.baz")); assertEquals("{\"_doc\":{\"dynamic\":\"false\"," + - "\"runtime\":{\"foo.bar.baz\":{\"type\":\"string\"},\"foo.baz\":{\"type\":\"string\"}}," + + "\"runtime\":{\"foo.bar.baz\":{\"type\":\"keyword\"},\"foo.baz\":{\"type\":\"keyword\"}}," + "\"properties\":{\"foo\":{\"dynamic\":\"runtime\",\"properties\":{\"bar\":{\"type\":\"object\"}}}}}}", Strings.toString(doc.dynamicMappingsUpdate())); } @@ -794,7 +794,7 @@ public void testDynamicRuntimeStringArray() throws Exception { ParsedDocument doc = mapper.parse(source(b -> b.startArray("foo").value("test1").value("test2").endArray())); assertEquals(0, doc.rootDoc().getFields("foo").length); RuntimeFieldType foo = doc.dynamicMappingsUpdate().root.getRuntimeFieldType("foo"); - assertEquals("string", foo.typeName()); + assertEquals("keyword", foo.typeName()); } public void testDynamicRuntimeBooleanArray() throws Exception { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java index a313e3c36f29e..cc02c4fc8c28f 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java @@ -83,7 +83,7 @@ public void testDynamicRuntime() throws IOException { assertNull(doc.rootDoc().get("field2")); assertEquals("{\"_doc\":{\"dynamic\":\"runtime\"," + - "\"runtime\":{\"field2\":{\"type\":\"string\"}}}}", + "\"runtime\":{\"field2\":{\"type\":\"keyword\"}}}}", Strings.toString(doc.dynamicMappingsUpdate())); } @@ -208,7 +208,7 @@ public void testField() throws Exception { } public void testDynamicUpdateWithRuntimeField() throws Exception { - MapperService mapperService = createMapperService(runtimeFieldMapping(b -> b.field("type", "string"))); + MapperService mapperService = createMapperService(runtimeFieldMapping(b -> b.field("type", "keyword"))); ParsedDocument doc = mapperService.documentMapper().parse(source(b -> b.field("test", "value"))); assertEquals("{\"_doc\":{\"properties\":{" + "\"test\":{\"type\":\"text\",\"fields\":{\"keyword\":{\"type\":\"keyword\",\"ignore_above\":256}}}}}}", @@ -222,7 +222,7 @@ public void testDynamicUpdateWithRuntimeField() throws Exception { public void testDynamicUpdateWithRuntimeFieldDottedName() throws Exception { MapperService mapperService = createMapperService(runtimeMapping( - b -> b.startObject("path1.path2.path3.field").field("type", "string").endObject())); + b -> b.startObject("path1.path2.path3.field").field("type", "keyword").endObject())); ParsedDocument doc = mapperService.documentMapper().parse(source(b -> { b.startObject("path1").startObject("path2").startObject("path3"); b.field("field", "value"); @@ -356,7 +356,7 @@ public void testDynamicMappingDynamicRuntimeObject() throws Exception { })); assertEquals("{\"_doc\":{\"dynamic\":\"true\",\"" + - "runtime\":{\"runtime_object.foo.bar.baz\":{\"type\":\"string\"}}," + + "runtime\":{\"runtime_object.foo.bar.baz\":{\"type\":\"keyword\"}}," + "\"properties\":{\"object\":{\"properties\":{\"foo\":{\"properties\":{\"bar\":{\"properties\":{" + "\"baz\":{\"type\":\"text\",\"fields\":{\"keyword\":{\"type\":\"keyword\",\"ignore_above\":256}}}}}}}}}," + "\"runtime_object\":{\"dynamic\":\"runtime\",\"properties\":{\"foo\":{\"properties\":{\"bar\":{\"type\":\"object\"}}}}}}}}", @@ -552,8 +552,8 @@ public void testNumericDetectionDefaultDynamicRuntime() throws Exception { assertNotNull(doc.dynamicMappingsUpdate()); merge(mapperService, dynamicMapping(doc.dynamicMappingsUpdate())); - assertThat(mapperService.fieldType("s_long").typeName(), equalTo("string")); - assertThat(mapperService.fieldType("s_double").typeName(), equalTo("string")); + assertThat(mapperService.fieldType("s_long").typeName(), equalTo("keyword")); + assertThat(mapperService.fieldType("s_double").typeName(), equalTo("keyword")); } public void testDateDetectionInheritsFormat() throws Exception { @@ -641,4 +641,147 @@ public void testDynamicTemplateOrder() throws IOException { merge(mapperService, dynamicMapping(doc.dynamicMappingsUpdate())); assertThat(mapperService.fieldType("foo"), instanceOf(KeywordFieldMapper.KeywordFieldType.class)); } + + public void testDynamicTemplateRuntimeMatchMappingType() throws Exception { + MapperService mapperService = createMapperService(topMapping(b -> { + b.startArray("dynamic_templates"); + { + b.startObject(); + { + b.startObject("test"); + { + b.field("match_mapping_type", "string"); + b.startObject("runtime").field("type", "long").endObject(); + } + b.endObject(); + } + b.endObject(); + } + b.endArray(); + })); + DocumentMapper docMapper = mapperService.documentMapper(); + ParsedDocument parsedDoc = docMapper.parse(source(b -> { + b.field("s", "hello"); + b.field("l", 1); + })); + assertEquals("{\"_doc\":{\"runtime\":{\"s\":{\"type\":\"long\"}},\"properties\":{\"l\":{\"type\":\"long\"}}}}", + Strings.toString(parsedDoc.dynamicMappingsUpdate())); + } + + public void testDynamicTemplateRuntimeMatch() throws Exception { + MapperService mapperService = createMapperService(topMapping(b -> { + b.startArray("dynamic_templates"); + { + b.startObject(); + { + b.startObject("test"); + { + b.field("match", "field*"); + b.startObject("runtime").endObject(); + } + b.endObject(); + } + b.endObject(); + } + b.endArray(); + })); + DocumentMapper docMapper = mapperService.documentMapper(); + ParsedDocument parsedDoc = docMapper.parse(source(b -> { + b.field("field_string", "hello"); + b.field("field_long", 1); + b.field("field_boolean", true); + b.field("concrete_string", "text"); + b.startObject("field_object"); + b.field("field_date", "2020-12-15"); + b.field("concrete_date", "2020-12-15"); + b.endObject(); + b.startArray("field_array"); + b.startObject(); + b.field("field_double", 1.25); + b.field("concrete_double", 1.25); + b.endObject(); + b.endArray(); + })); + assertEquals("{\"_doc\":{\"runtime\":{" + + "\"field_array.field_double\":{\"type\":\"double\"}," + + "\"field_boolean\":{\"type\":\"boolean\"}," + + "\"field_long\":{\"type\":\"long\"}," + + "\"field_object.field_date\":{\"type\":\"date\"}," + + "\"field_string\":{\"type\":\"keyword\"}}," + + "\"properties\":" + + "{\"concrete_string\":{\"type\":\"text\",\"fields\":{\"keyword\":{\"type\":\"keyword\",\"ignore_above\":256}}}," + + "\"field_array\":{\"properties\":{\"concrete_double\":{\"type\":\"float\"}}}," + + "\"field_object\":{\"properties\":{\"concrete_date\":{\"type\":\"date\"}}}}}}", + Strings.toString(parsedDoc.dynamicMappingsUpdate())); + } + + public void testDynamicTemplateRuntimePathMatch() throws Exception { + MapperService mapperService = createMapperService(topMapping(b -> { + b.startArray("dynamic_templates"); + { + b.startObject(); + { + b.startObject("test"); + { + b.field("path_match", "object.*"); + b.field("path_unmatch", "*.concrete*"); + b.startObject("runtime").endObject(); + } + b.endObject(); + } + b.endObject(); + } + b.endArray(); + })); + DocumentMapper docMapper = mapperService.documentMapper(); + ParsedDocument parsedDoc = docMapper.parse(source(b -> { + b.field("double", 1.23); + b.startObject("object"); + { + b.field("date", "2020-12-15"); + b.field("long", 1); + b.startObject("object").field("string", "hello").field("concrete", false).endObject(); + } + b.endObject(); + b.startObject("concrete").field("boolean", true).endObject(); + })); + assertEquals("{\"_doc\":{\"runtime\":{" + + "\"object.date\":{\"type\":\"date\"}," + + "\"object.long\":{\"type\":\"long\"}," + + "\"object.object.string\":{\"type\":\"keyword\"}}," + + "\"properties\":" + "{" + + "\"concrete\":{\"properties\":{\"boolean\":{\"type\":\"boolean\"}}}," + + "\"double\":{\"type\":\"float\"}," + + "\"object\":{\"properties\":{\"object\":{\"properties\":{\"concrete\":{\"type\":\"boolean\"}}}}}}}}", + Strings.toString(parsedDoc.dynamicMappingsUpdate())); + } + + public void testDynamicRuntimeWithDynamicTemplate() throws IOException { + MapperService mapperService = createMapperService(topMapping(b -> { + b.field("dynamic", "runtime"); + b.startArray("dynamic_templates"); + { + b.startObject(); + { + b.startObject("concrete"); + { + b.field("match", "concrete*"); + b.startObject("mapping").endObject(); + } + b.endObject(); + } + b.endObject(); + } + b.endArray(); + })); + DocumentMapper docMapper = mapperService.documentMapper(); + ParsedDocument parsedDoc = docMapper.parse(source(b -> { + b.field("double", 1.23); + b.field("concrete_double", 1.23); + })); + assertEquals("{\"_doc\":{\"dynamic\":\"runtime\"," + + "\"runtime\":{" + "\"double\":{\"type\":\"double\"}}," + + "\"properties\":{\"concrete_double\":{\"type\":\"float\"}}}}", + Strings.toString(parsedDoc.dynamicMappingsUpdate())); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplateTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplateTests.java index a910c2c86bab8..c124dd7b07937 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplateTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplateTests.java @@ -27,13 +27,92 @@ import org.elasticsearch.index.mapper.DynamicTemplate.XContentFieldType; import org.elasticsearch.test.ESTestCase; +import java.io.IOException; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; public class DynamicTemplateTests extends ESTestCase { - public void testParseUnknownParam() throws Exception { + public void testMappingTypeTypeNotSet() { + Map templateDef = new HashMap<>(); + templateDef.put("match_mapping_type", "string"); + templateDef.put("mapping", Collections.emptyMap()); + DynamicTemplate template = DynamicTemplate.parse("my_template", templateDef, Version.CURRENT); + //when type is not set, the provided dynamic type is returned + assertEquals("input", template.mappingType("input")); + } + + public void testMappingTypeTypeNotSetRuntime() { + Map templateDef = new HashMap<>(); + templateDef.put("match_mapping_type", "string"); + templateDef.put("runtime", Collections.emptyMap()); + DynamicTemplate template = DynamicTemplate.parse("my_template", templateDef, Version.CURRENT); + //when type is not set, the provided dynamic type is returned + assertEquals("input", template.mappingType("input")); + } + + public void testMappingType() { + Map templateDef = new HashMap<>(); + templateDef.put("match_mapping_type", "string"); + templateDef.put("mapping", Collections.singletonMap("type", "type_set")); + DynamicTemplate template = DynamicTemplate.parse("my_template", templateDef, Version.CURRENT); + //when type is set, the set type is returned + assertEquals("type_set", template.mappingType("input")); + } + + public void testMappingTypeRuntime() { + Map templateDef = new HashMap<>(); + templateDef.put("match_mapping_type", "string"); + templateDef.put("runtime", Collections.singletonMap("type", "type_set")); + DynamicTemplate template = DynamicTemplate.parse("my_template", templateDef, Version.CURRENT); + //when type is set, the set type is returned + assertEquals("type_set", template.mappingType("input")); + } + + public void testMappingTypeDynamicTypeReplace() { + Map templateDef = new HashMap<>(); + templateDef.put("match_mapping_type", "string"); + templateDef.put("mapping", Collections.singletonMap("type", "type_set_{dynamic_type}_{dynamicType}")); + DynamicTemplate template = DynamicTemplate.parse("my_template", templateDef, Version.CURRENT); + //when type is set, the set type is returned + assertEquals("type_set_input_input", template.mappingType("input")); + } + + public void testMappingTypeDynamicTypeReplaceRuntime() { + Map templateDef = new HashMap<>(); + templateDef.put("match_mapping_type", "string"); + templateDef.put("runtime", Collections.singletonMap("type", "type_set_{dynamic_type}_{dynamicType}")); + DynamicTemplate template = DynamicTemplate.parse("my_template", templateDef, Version.CURRENT); + //when type is set, the set type is returned + assertEquals("type_set_input_input", template.mappingType("input")); + } + + public void testMappingForName() throws IOException { + Map templateDef = new HashMap<>(); + templateDef.put("match_mapping_type", "string"); + templateDef.put("mapping", org.elasticsearch.common.collect.Map.of( + "field1_{name}", "{dynamic_type}", "test", Collections.singletonList("field2_{name}_{dynamicType}"))); + DynamicTemplate template = DynamicTemplate.parse("my_template", templateDef, Version.CURRENT); + Map stringObjectMap = template.mappingForName("my_name", "my_type"); + assertEquals("{\"field1_my_name\":\"my_type\",\"test\":[\"field2_my_name_my_type\"]}", + Strings.toString(JsonXContent.contentBuilder().map(stringObjectMap))); + } + + public void testMappingForNameRuntime() throws IOException { + Map templateDef = new HashMap<>(); + templateDef.put("match_mapping_type", "string"); + templateDef.put("runtime", org.elasticsearch.common.collect.Map.of( + "field1_{name}", "{dynamic_type}", "test", Collections.singletonList("field2_{name}_{dynamicType}"))); + DynamicTemplate template = DynamicTemplate.parse("my_template", templateDef, Version.CURRENT); + Map stringObjectMap = template.mappingForName("my_name", "my_type"); + assertEquals("{\"field1_my_name\":\"my_type\",\"test\":[\"field2_my_name_my_type\"]}", + Strings.toString(JsonXContent.contentBuilder().map(stringObjectMap))); + } + + public void testParseUnknownParam() { Map templateDef = new HashMap<>(); templateDef.put("match_mapping_type", "string"); templateDef.put("mapping", Collections.singletonMap("store", true)); @@ -68,12 +147,69 @@ public void testParseInvalidRegex() { } } + public void testParseMappingAndRuntime() { + for (String param : new String[] { "path_match", "match", "path_unmatch", "unmatch" }) { + Map templateDef = new HashMap<>(); + templateDef.put("match", "foo"); + templateDef.put(param, "*a"); + templateDef.put("match_pattern", "regex"); + templateDef.put("mapping", Collections.emptyMap()); + templateDef.put("runtime", Collections.emptyMap()); + MapperParsingException e = expectThrows(MapperParsingException.class, + () -> DynamicTemplate.parse("my_template", templateDef, Version.CURRENT)); + assertEquals("mapping and runtime cannot be both specified in the same dynamic template [my_template]", e.getMessage()); + } + } + + public void testParseMissingMapping() { + for (String param : new String[] { "path_match", "match", "path_unmatch", "unmatch" }) { + Map templateDef = new HashMap<>(); + templateDef.put("match", "foo"); + templateDef.put(param, "*a"); + templateDef.put("match_pattern", "regex"); + MapperParsingException e = expectThrows(MapperParsingException.class, + () -> DynamicTemplate.parse("my_template", templateDef, Version.CURRENT)); + assertEquals("template [my_template] must have either mapping or runtime set", e.getMessage()); + } + } + public void testMatchAllTemplate() { Map templateDef = new HashMap<>(); templateDef.put("match_mapping_type", "*"); templateDef.put("mapping", Collections.singletonMap("store", true)); DynamicTemplate template = DynamicTemplate.parse("my_template", templateDef, Version.V_6_0_0_alpha1); assertTrue(template.match("a.b", "b", randomFrom(XContentFieldType.values()))); + assertFalse(template.isRuntimeMapping()); + } + + public void testMatchAllTemplateRuntime() { + Map templateDef = new HashMap<>(); + templateDef.put("match_mapping_type", "*"); + templateDef.put("runtime", Collections.emptyMap()); + DynamicTemplate template = DynamicTemplate.parse("my_template", templateDef, Version.CURRENT); + assertTrue(template.isRuntimeMapping()); + assertTrue(template.match("a.b", "b", XContentFieldType.BOOLEAN)); + assertTrue(template.match("a.b", "b", XContentFieldType.DATE)); + assertTrue(template.match("a.b", "b", XContentFieldType.STRING)); + assertTrue(template.match("a.b", "b", XContentFieldType.DOUBLE)); + assertTrue(template.match("a.b", "b", XContentFieldType.LONG)); + assertFalse(template.match("a.b", "b", XContentFieldType.OBJECT)); + assertFalse(template.match("a.b", "b", XContentFieldType.BINARY)); + } + + public void testMatchAllTypesTemplateRuntime() { + Map templateDef = new HashMap<>(); + templateDef.put("match", "b"); + templateDef.put("runtime", Collections.emptyMap()); + DynamicTemplate template = DynamicTemplate.parse("my_template", templateDef, Version.CURRENT); + assertTrue(template.isRuntimeMapping()); + assertTrue(template.match("a.b", "b", XContentFieldType.BOOLEAN)); + assertTrue(template.match("a.b", "b", XContentFieldType.DATE)); + assertTrue(template.match("a.b", "b", XContentFieldType.STRING)); + assertTrue(template.match("a.b", "b", XContentFieldType.DOUBLE)); + assertTrue(template.match("a.b", "b", XContentFieldType.LONG)); + assertFalse(template.match("a.b", "b", XContentFieldType.OBJECT)); + assertFalse(template.match("a.b", "b", XContentFieldType.BINARY)); } public void testMatchTypeTemplate() { @@ -83,6 +219,39 @@ public void testMatchTypeTemplate() { DynamicTemplate template = DynamicTemplate.parse("my_template", templateDef, Version.V_6_0_0_alpha1); assertTrue(template.match("a.b", "b", XContentFieldType.STRING)); assertFalse(template.match("a.b", "b", XContentFieldType.BOOLEAN)); + assertFalse(template.isRuntimeMapping()); + } + + public void testMatchTypeTemplateRuntime() { + Map templateDef = new HashMap<>(); + templateDef.put("match_mapping_type", "string"); + templateDef.put("runtime", Collections.emptyMap()); + DynamicTemplate template = DynamicTemplate.parse("my_template", templateDef, Version.CURRENT); + assertTrue(template.match("a.b", "b", XContentFieldType.STRING)); + assertFalse(template.match("a.b", "b", XContentFieldType.BOOLEAN)); + assertTrue(template.isRuntimeMapping()); + } + + public void testSupportedMatchMappingTypesRuntime() { + //binary and object are not supported as runtime fields + List nonSupported = Arrays.asList("binary", "object"); + for (String type : nonSupported) { + Map templateDef = new HashMap<>(); + templateDef.put("match_mapping_type", type); + templateDef.put("runtime", Collections.emptyMap()); + MapperParsingException e = expectThrows(MapperParsingException.class, + () -> DynamicTemplate.parse("my_template", templateDef, Version.CURRENT)); + assertEquals("Dynamic template [my_template] defines a runtime field but type [" + type + "] is not supported as runtime field", + e.getMessage()); + } + XContentFieldType[] supported = Arrays.stream(XContentFieldType.values()) + .filter(XContentFieldType::supportsRuntimeField).toArray(XContentFieldType[]::new); + for (XContentFieldType type : supported) { + Map templateDef = new HashMap<>(); + templateDef.put("match_mapping_type", type); + templateDef.put("runtime", Collections.emptyMap()); + assertNotNull(DynamicTemplate.parse("my_template", templateDef, Version.CURRENT)); + } } public void testSerialization() throws Exception { @@ -126,4 +295,46 @@ public void testSerialization() throws Exception { template.toXContent(builder, ToXContent.EMPTY_PARAMS); assertEquals("{\"match\":\"^a$\",\"match_pattern\":\"regex\",\"mapping\":{\"store\":true}}", Strings.toString(builder)); } + + public void testSerializationRuntimeMappings() throws Exception { + // type-based template + Map templateDef = new HashMap<>(); + templateDef.put("match_mapping_type", "string"); + templateDef.put("runtime", Collections.emptyMap()); + DynamicTemplate template = DynamicTemplate.parse("my_template", templateDef, Version.CURRENT); + XContentBuilder builder = JsonXContent.contentBuilder(); + template.toXContent(builder, ToXContent.EMPTY_PARAMS); + assertEquals("{\"match_mapping_type\":\"string\",\"runtime\":{}}", Strings.toString(builder)); + + // name-based template + templateDef = new HashMap<>(); + templateDef.put("match", "*name"); + templateDef.put("unmatch", "first_name"); + templateDef.put("runtime", Collections.singletonMap("type", "new_type")); + template = DynamicTemplate.parse("my_template", templateDef, Version.CURRENT); + builder = JsonXContent.contentBuilder(); + template.toXContent(builder, ToXContent.EMPTY_PARAMS); + assertEquals("{\"match\":\"*name\",\"unmatch\":\"first_name\",\"runtime\":{\"type\":\"new_type\"}}", Strings.toString(builder)); + + // path-based template + templateDef = new HashMap<>(); + templateDef.put("path_match", "*name"); + templateDef.put("path_unmatch", "first_name"); + templateDef.put("runtime", Collections.emptyMap()); + template = DynamicTemplate.parse("my_template", templateDef, Version.CURRENT); + builder = JsonXContent.contentBuilder(); + template.toXContent(builder, ToXContent.EMPTY_PARAMS); + assertEquals("{\"path_match\":\"*name\",\"path_unmatch\":\"first_name\",\"runtime\":{}}", + Strings.toString(builder)); + + // regex matching + templateDef = new HashMap<>(); + templateDef.put("match", "^a$"); + templateDef.put("match_pattern", "regex"); + templateDef.put("runtime", Collections.emptyMap()); + template = DynamicTemplate.parse("my_template", templateDef, Version.CURRENT); + builder = JsonXContent.contentBuilder(); + template.toXContent(builder, ToXContent.EMPTY_PARAMS); + assertEquals("{\"match\":\"^a$\",\"match_pattern\":\"regex\",\"runtime\":{}}", Strings.toString(builder)); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplatesTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplatesTests.java index 4ba6aa01cbd2d..69f8b213d4421 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplatesTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplatesTests.java @@ -22,6 +22,7 @@ import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexableField; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.mapper.ParseContext.Document; import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath; @@ -61,6 +62,59 @@ public void testMatchTypeOnly() throws Exception { assertTrue(mapperService.fieldType("l").isSearchable()); } + public void testMatchTypeRuntimeNoPlugin() throws Exception { + XContentBuilder topMapping = topMapping(b -> { + b.startArray("dynamic_templates"); + { + b.startObject(); + { + b.startObject("test"); + { + b.field("match_mapping_type", "string"); + b.startObject("runtime").endObject(); + } + b.endObject(); + } + b.endObject(); + } + b.endArray(); + }); + + createMapperService(topMapping); + assertWarnings("dynamic template [test] has invalid content [{\"match_mapping_type\":\"string\",\"runtime\":{}}], " + + "attempted to validate it with the following match_mapping_type: [string], " + + "caused by [No runtime field found for type [keyword]]"); + } + + public void testMatchAllTypesRuntimeNoPlugin() throws Exception { + boolean matchMappingType = randomBoolean(); + XContentBuilder topMapping = topMapping(b -> { + b.startArray("dynamic_templates"); + { + b.startObject(); + { + b.startObject("test"); + { + if (matchMappingType) { + b.field("match_mapping_type", "*"); + } else { + b.field("match", "field"); + } + b.startObject("runtime").endObject(); + } + b.endObject(); + } + b.endObject(); + } + b.endArray(); + }); + String matchError = matchMappingType ? "\"match_mapping_type\":\"*\"" : "\"match\":\"field\""; + createMapperService(topMapping); + assertWarnings("dynamic template [test] has invalid content [" + "{" + matchError + ",\"runtime\":{}}], " + + "attempted to validate it with the following match_mapping_type: [string, long, double, boolean, date], " + + "caused by [No runtime field found for type [date]]"); + } + public void testSimple() throws Exception { String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/dynamictemplate/simple/test-mapping.json"); MapperService mapperService = createMapperService("person", mapping); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/RootObjectMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/RootObjectMapperTests.java index ff8ef5c667595..4e0400a19d27f 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/RootObjectMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/RootObjectMapperTests.java @@ -310,10 +310,37 @@ public void testIllegalDynamicTemplateUnknownFieldType() throws Exception { MapperService mapperService = createMapperService(mapping); assertThat(mapperService.documentMapper().mappingSource().toString(), containsString("\"type\":\"string\"")); assertWarnings("dynamic template [my_template] has invalid content [{\"match_mapping_type\":\"string\",\"mapping\":{\"type\":" + - "\"string\"}}], attempted to validate it with the following match_mapping_type: [[string]], " + + "\"string\"}}], attempted to validate it with the following match_mapping_type: [string], " + "caused by [No mapper found for type [string]]"); } + public void testIllegalDynamicTemplateUnknownRuntimeFieldType() throws Exception { + XContentBuilder mapping = XContentFactory.jsonBuilder(); + mapping.startObject(); + { + mapping.startObject(MapperService.SINGLE_MAPPING_NAME); + mapping.startArray("dynamic_templates"); + { + mapping.startObject(); + mapping.startObject("my_template"); + mapping.field("match_mapping_type", "string"); + mapping.startObject("runtime"); + mapping.field("type", "unknown"); + mapping.endObject(); + mapping.endObject(); + mapping.endObject(); + } + mapping.endArray(); + mapping.endObject(); + } + mapping.endObject(); + createMapperService(mapping); + assertWarnings("dynamic template [my_template] has invalid content [" + + "{\"match_mapping_type\":\"string\",\"runtime\":{\"type\":\"unknown\"}}], " + + "attempted to validate it with the following match_mapping_type: [string], " + + "caused by [No runtime field found for type [unknown]]"); + } + public void testIllegalDynamicTemplateUnknownAttribute() throws Exception { XContentBuilder mapping = XContentFactory.jsonBuilder(); mapping.startObject(); @@ -340,10 +367,38 @@ public void testIllegalDynamicTemplateUnknownAttribute() throws Exception { assertThat(mapperService.documentMapper().mappingSource().toString(), containsString("\"foo\":\"bar\"")); assertWarnings("dynamic template [my_template] has invalid content [{\"match_mapping_type\":\"string\",\"mapping\":{" + "\"foo\":\"bar\",\"type\":\"keyword\"}}], " + - "attempted to validate it with the following match_mapping_type: [[string]], " + + "attempted to validate it with the following match_mapping_type: [string], " + "caused by [unknown parameter [foo] on mapper [__dynamic__my_template] of type [keyword]]"); } + public void testIllegalDynamicTemplateUnknownAttributeRuntime() throws Exception { + XContentBuilder mapping = XContentFactory.jsonBuilder(); + mapping.startObject(); + { + mapping.startObject(MapperService.SINGLE_MAPPING_NAME); + mapping.startArray("dynamic_templates"); + { + mapping.startObject(); + mapping.startObject("my_template"); + mapping.field("match_mapping_type", "string"); + mapping.startObject("runtime"); + mapping.field("type", "test"); + mapping.field("foo", "bar"); + mapping.endObject(); + mapping.endObject(); + mapping.endObject(); + } + mapping.endArray(); + mapping.endObject(); + } + mapping.endObject(); + + createMapperService(mapping); + assertWarnings("dynamic template [my_template] has invalid content [" + + "{\"match_mapping_type\":\"string\",\"runtime\":{\"foo\":\"bar\",\"type\":\"test\"}}], " + + "attempted to validate it with the following match_mapping_type: [string], caused by [Unknown mapping attributes [{foo=bar}]]"); + } + public void testIllegalDynamicTemplateInvalidAttribute() throws Exception { XContentBuilder mapping = XContentFactory.jsonBuilder(); mapping.startObject(); @@ -369,7 +424,7 @@ public void testIllegalDynamicTemplateInvalidAttribute() throws Exception { MapperService mapperService = createMapperService(mapping); assertThat(mapperService.documentMapper().mappingSource().toString(), containsString("\"analyzer\":\"foobar\"")); assertWarnings("dynamic template [my_template] has invalid content [{\"match_mapping_type\":\"string\",\"mapping\":{" + - "\"analyzer\":\"foobar\",\"type\":\"text\"}}], attempted to validate it with the following match_mapping_type: [[string]], " + + "\"analyzer\":\"foobar\",\"type\":\"text\"}}], attempted to validate it with the following match_mapping_type: [string], " + "caused by [analyzer [foobar] has not been configured in mappings]"); } @@ -436,18 +491,55 @@ public void testIllegalDynamicTemplateNoMappingType() throws Exception { assertWarnings("dynamic template [my_template] has invalid content [{\"match_mapping_type\":\"*\",\"mapping\":{" + "\"foo\":\"bar\",\"type\":\"{dynamic_type}\"}}], " + "attempted to validate it with the following match_mapping_type: " + - "[[object, string, long, double, boolean, date, binary]], " + + "[object, string, long, double, boolean, date, binary], " + "caused by [unknown parameter [foo] on mapper [__dynamic__my_template] of type [binary]]"); } else { assertWarnings("dynamic template [my_template] has invalid content [{\"match\":\"string_*\",\"mapping\":{" + "\"foo\":\"bar\",\"type\":\"{dynamic_type}\"}}], " + "attempted to validate it with the following match_mapping_type: " + - "[[object, string, long, double, boolean, date, binary]], " + + "[object, string, long, double, boolean, date, binary], " + "caused by [unknown parameter [foo] on mapper [__dynamic__my_template] of type [binary]]"); } } } + public void testIllegalDynamicTemplateNoMappingTypeRuntime() throws Exception { + XContentBuilder mapping = XContentFactory.jsonBuilder(); + String matchError; + mapping.startObject(); + { + mapping.startObject(MapperService.SINGLE_MAPPING_NAME); + mapping.startArray("dynamic_templates"); + { + mapping.startObject(); + mapping.startObject("my_template"); + if (randomBoolean()) { + mapping.field("match_mapping_type", "*"); + matchError = "\"match_mapping_type\":\"*\""; + } else { + mapping.field("match", "string_*"); + matchError = "\"match\":\"string_*\""; + } + mapping.startObject("runtime"); + mapping.field("type", "{dynamic_type}"); + mapping.field("foo", "bar"); + mapping.endObject(); + mapping.endObject(); + mapping.endObject(); + } + mapping.endArray(); + mapping.endObject(); + } + mapping.endObject(); + + createMapperService(mapping); + String expected = "dynamic template [my_template] has invalid content [{" + matchError + + ",\"runtime\":{\"foo\":\"bar\",\"type\":\"{dynamic_type}\"}}], " + + "attempted to validate it with the following match_mapping_type: [string, long, double, boolean, date], " + + "caused by [Unknown mapping attributes [{foo=bar}]]"; + assertWarnings(expected); + } + public void testIllegalDynamicTemplatePre7Dot7Index() throws Exception { XContentBuilder mapping = XContentFactory.jsonBuilder(); mapping.startObject(); @@ -595,9 +687,9 @@ public void testRuntimeSectionMerge() throws IOException { } public void testRuntimeSectionNonRuntimeType() throws IOException { - XContentBuilder mapping = runtimeFieldMapping(builder -> builder.field("type", "keyword")); + XContentBuilder mapping = runtimeFieldMapping(builder -> builder.field("type", "unknown")); MapperParsingException e = expectThrows(MapperParsingException.class, () -> createMapperService(mapping)); - assertEquals("Failed to parse mapping [_doc]: No handler for type [keyword] declared on runtime field [field]", e.getMessage()); + assertEquals("Failed to parse mapping [_doc]: No handler for type [unknown] declared on runtime field [field]", e.getMessage()); } public void testRuntimeSectionHandlerNotFound() throws IOException { @@ -650,11 +742,16 @@ public void testDynamicRuntimeNotSupported() { private static class RuntimeFieldPlugin extends Plugin implements MapperPlugin { @Override public Map getRuntimeFieldTypes() { - return Collections.singletonMap("test", (name, node, parserContext) -> { + return org.elasticsearch.common.collect.Map.of("test", (name, node, parserContext) -> { Object prop1 = node.remove("prop1"); Object prop2 = node.remove("prop2"); return new RuntimeField(name, prop1 == null ? null : prop1.toString(), prop2 == null ? null : prop2.toString()); - }); + }, + "keyword", (name, node, parserContext) -> new TestRuntimeField(name, "keyword"), + "boolean", (name, node, parserContext) -> new TestRuntimeField(name, "boolean"), + "long", (name, node, parserContext) -> new TestRuntimeField(name, "long"), + "double", (name, node, parserContext) -> new TestRuntimeField(name, "double"), + "date", (name, node, parserContext) -> new TestRuntimeField(name, "date")); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TestRuntimeField.java b/server/src/test/java/org/elasticsearch/index/mapper/TestRuntimeField.java index dd8cd8d95bb5f..7a1fd13ed8d5c 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TestRuntimeField.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TestRuntimeField.java @@ -61,7 +61,7 @@ public static class Plugin extends org.elasticsearch.plugins.Plugin implements M @Override public Map getRuntimeFieldTypes() { return org.elasticsearch.common.collect.Map.of( - "string", (name, node, parserContext) -> new TestRuntimeField(name, "string"), + "keyword", (name, node, parserContext) -> new TestRuntimeField(name, "keyword"), "double", (name, node, parserContext) -> new TestRuntimeField(name, "double"), "long", (name, node, parserContext) -> new TestRuntimeField(name, "long"), "boolean", (name, node, parserContext) -> new TestRuntimeField(name, "boolean"), @@ -73,7 +73,7 @@ public DynamicRuntimeFieldsBuilder getDynamicRuntimeFieldsBuilder() { return new DynamicRuntimeFieldsBuilder() { @Override public RuntimeFieldType newDynamicStringField(String name) { - return new TestRuntimeField(name, "string"); + return new TestRuntimeField(name, "keyword"); } @Override diff --git a/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java b/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java index 74ea568ea89f0..dd41933e32388 100644 --- a/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java @@ -325,7 +325,7 @@ public void testSearchRequestRuntimeFields() { * shards are parsed on the same node. */ Map runtimeMappings = org.elasticsearch.common.collect.Map.ofEntries( - org.elasticsearch.common.collect.Map.entry("cat", org.elasticsearch.common.collect.Map.of("type", "string")), + org.elasticsearch.common.collect.Map.entry("cat", org.elasticsearch.common.collect.Map.of("type", "keyword")), org.elasticsearch.common.collect.Map.entry("dog", org.elasticsearch.common.collect.Map.of("type", "long")) ); QueryShardContext qsc = createQueryShardContext(