Emit multiple fields from the same runtime field script

modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/org.elasticsearch.script.object_field.txt

@@ -0,0 +1,21 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License
+# 2.0 and the Server Side Public License, v 1; you may not use this file except
+# in compliance with, at your election, the Elastic License 2.0 or the Server
+# Side Public License, v 1.
+#
+
+# The whitelist for object runtime fields
+
+# These two whitelists are required for painless to find the classes
+class org.elasticsearch.script.ObjectFieldScript @no_import {
+}
+class org.elasticsearch.script.ObjectFieldScript$Factory @no_import {
+}
+
+static_import {
+    # The `emit` callback to collect values for the fields
+    void emit(org.elasticsearch.script.ObjectFieldScript, String, Object) bound_to org.elasticsearch.script.ObjectFieldScript$EmitField
+    void emit(org.elasticsearch.script.ObjectFieldScript, Map) bound_to org.elasticsearch.script.ObjectFieldScript$EmitMap
+}

server/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldType.java

@@ -18,6 +18,7 @@
 import org.elasticsearch.common.time.DateMathParser;
 import org.elasticsearch.common.unit.Fuzziness;
 import org.elasticsearch.index.query.SearchExecutionContext;
+import org.elasticsearch.script.ObjectFieldScript;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptContext;
 import org.elasticsearch.script.ScriptType;
@@ -194,7 +195,6 @@ protected final LeafFactory leafFactory(SearchExecutionContext context) {
 
     abstract static class Builder<Factory> extends RuntimeField.Builder {
         private final ScriptContext<Factory> scriptContext;
-        private final Factory parseFromSourceFactory;
 
         final FieldMapper.Parameter<Script> script = new FieldMapper.Parameter<>(
             "script",
@@ -204,23 +204,33 @@ abstract static class Builder<Factory> extends RuntimeField.Builder {
             initializerNotSupported()
         ).setSerializerCheck((id, ic, v) -> ic);
 
-        Builder(String name, ScriptContext<Factory> scriptContext, Factory parseFromSourceFactory) {
+        Builder(String name, ScriptContext<Factory> scriptContext) {
             super(name);
             this.scriptContext = scriptContext;
-            this.parseFromSourceFactory = parseFromSourceFactory;
         }
 
+        abstract Factory getParseFromSourceFactory();
+
+        abstract Factory getObjectSubfieldFactory(Function<SearchLookup, ObjectFieldScript.LeafFactory> parentScriptFactory);
+
         @Override
-        protected final RuntimeField createRuntimeField(MappingParserContext parserContext) {
+        protected final RuntimeField createRuntimeField(MappingParserContext parserContext,
+                                                        String parent,
+                                                        Function<SearchLookup, ObjectFieldScript.LeafFactory> parentScriptFactory) {
             if (script.get() == null) {
-                return createRuntimeField(parseFromSourceFactory);
+                if (parentScriptFactory == null) {
+                    return createRuntimeField(parent, getParseFromSourceFactory());
+                }
+                return createRuntimeField(parent, getObjectSubfieldFactory(parentScriptFactory));
             }
+            assert parent == null && parentScriptFactory == null;
             Factory factory = parserContext.scriptCompiler().compile(script.getValue(), scriptContext);
-            return createRuntimeField(factory);
+            return createRuntimeField(null, factory);
         }
 
-        final RuntimeField createRuntimeField(Factory scriptFactory) {
-            AbstractScriptFieldType<?> fieldType = createFieldType(name, scriptFactory, getScript(), meta());
+        final RuntimeField createRuntimeField(String parent, Factory scriptFactory) {
+            String fullName = parent == null ? name : parent + "." + name;
+            AbstractScriptFieldType<?> fieldType = createFieldType(fullName, scriptFactory, getScript(), meta());
             return new LeafRuntimeField(name, fieldType, getParameters());
         }
 
@@ -240,7 +250,7 @@ protected final Script getScript() {
             return script.get();
         }
 
-        private static Script parseScript(String name, MappingParserContext parserContext, Object scriptObject) {
+        static Script parseScript(String name, MappingParserContext parserContext, Object scriptObject) {
             Script script = Script.parse(scriptObject);
             if (script.getType() == ScriptType.STORED) {
                 throw new IllegalArgumentException("stored scripts are not supported for runtime field [" + name + "]");

server/src/main/java/org/elasticsearch/index/mapper/BooleanScriptFieldType.java

@@ -18,6 +18,7 @@
 import org.elasticsearch.index.fielddata.BooleanScriptFieldData;
 import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.script.BooleanFieldScript;
+import org.elasticsearch.script.ObjectFieldScript;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.lookup.SearchLookup;
@@ -27,6 +28,7 @@
 import java.time.ZoneId;
 import java.util.Collection;
 import java.util.Map;
+import java.util.function.Function;
 import java.util.function.Supplier;
 
 public final class BooleanScriptFieldType extends AbstractScriptFieldType<BooleanFieldScript.LeafFactory> {
@@ -35,7 +37,7 @@ public final class BooleanScriptFieldType extends AbstractScriptFieldType<Boolea
 
     private static class Builder extends AbstractScriptFieldType.Builder<BooleanFieldScript.Factory> {
         Builder(String name) {
-            super(name, BooleanFieldScript.CONTEXT, BooleanFieldScript.PARSE_FROM_SOURCE);
+            super(name, BooleanFieldScript.CONTEXT);
         }
 
         @Override
@@ -45,10 +47,21 @@ AbstractScriptFieldType<?> createFieldType(String name,
                                                    Map<String, String> meta) {
             return new BooleanScriptFieldType(name, factory, script, meta);
         }
+
+        @Override
+        BooleanFieldScript.Factory getParseFromSourceFactory() {
+            return BooleanFieldScript.PARSE_FROM_SOURCE;
+        }
+
+        @Override
+        BooleanFieldScript.Factory getObjectSubfieldFactory(Function<SearchLookup, ObjectFieldScript.LeafFactory> parentScriptFactory) {
+            return BooleanFieldScript.objectAdapter(parentScriptFactory);
+        }
+
     }
 
     public static RuntimeField sourceOnly(String name) {
-        return new Builder(name).createRuntimeField(BooleanFieldScript.PARSE_FROM_SOURCE);
+        return new Builder(name).createRuntimeField(null, BooleanFieldScript.PARSE_FROM_SOURCE);
     }
 
     BooleanScriptFieldType(

server/src/main/java/org/elasticsearch/index/mapper/DateScriptFieldType.java

@@ -23,6 +23,7 @@
 import org.elasticsearch.index.mapper.DateFieldMapper.Resolution;
 import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.script.DateFieldScript;
+import org.elasticsearch.script.ObjectFieldScript;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.lookup.SearchLookup;
@@ -40,6 +41,7 @@
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.function.Function;
 import java.util.function.Supplier;
 
 public class DateScriptFieldType extends AbstractScriptFieldType<DateFieldScript.LeafFactory> {
@@ -71,7 +73,7 @@ private static class Builder extends AbstractScriptFieldType.Builder<DateFieldSc
         }, Object::toString).acceptsNull();
 
         Builder(String name) {
-            super(name, DateFieldScript.CONTEXT, DateFieldScript.PARSE_FROM_SOURCE);
+            super(name, DateFieldScript.CONTEXT);
         }
 
         @Override
@@ -89,12 +91,22 @@ AbstractScriptFieldType<?> createFieldType(String name, DateFieldScript.Factory
             DateFormatter dateTimeFormatter = DateFormatter.forPattern(pattern).withLocale(locale);
             return new DateScriptFieldType(name, factory, dateTimeFormatter, script, meta);
         }
+
+        @Override
+        DateFieldScript.Factory getParseFromSourceFactory() {
+            return DateFieldScript.PARSE_FROM_SOURCE;
+        }
+
+        @Override
+        DateFieldScript.Factory getObjectSubfieldFactory(Function<SearchLookup, ObjectFieldScript.LeafFactory> parentScriptFactory) {
+            return DateFieldScript.objectAdapter(parentScriptFactory);
+        }
     }
 
     public static RuntimeField sourceOnly(String name, DateFormatter dateTimeFormatter) {
         Builder builder = new Builder(name);
         builder.format.setValue(dateTimeFormatter.pattern());
-        return builder.createRuntimeField(DateFieldScript.PARSE_FROM_SOURCE);
+        return builder.createRuntimeField(null, DateFieldScript.PARSE_FROM_SOURCE);
     }
 
     private final DateFormatter dateTimeFormatter;

server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java

@@ -880,8 +880,10 @@ private static Mapper getLeafMapper(final DocumentParserContext context,
         // don't create a dynamic mapping for it and don't index it.
         String fieldPath = context.path().pathAsText(fieldName);
         MappedFieldType fieldType = context.mappingLookup().getFieldType(fieldPath);
+        //TODO test that we don't index sub-fields that are part of a runtime object, unless explicitly mapped.
         if (fieldType != null) {
-            //we haven't found a mapper with this name above, which means if a field type is found it is for sure a runtime field.
+            //we have looked for the mapper above and we did not find it. If we do find a MappedFieldType for the field,
+            //we can assume it comes from a runtime field. That is what the assertion enforces.
             assert fieldType.hasDocValues() == false && fieldType.isAggregatable() && fieldType.isSearchable();
             return new NoOpFieldMapper(subfields[subfields.length - 1], fieldType.name());
         }

server/src/main/java/org/elasticsearch/index/mapper/DoubleScriptFieldType.java

@@ -18,6 +18,7 @@
 import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType;
 import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.script.DoubleFieldScript;
+import org.elasticsearch.script.ObjectFieldScript;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.lookup.SearchLookup;
@@ -29,6 +30,7 @@
 import java.time.ZoneId;
 import java.util.Collection;
 import java.util.Map;
+import java.util.function.Function;
 import java.util.function.Supplier;
 
 public final class DoubleScriptFieldType extends AbstractScriptFieldType<DoubleFieldScript.LeafFactory> {
@@ -37,7 +39,7 @@ public final class DoubleScriptFieldType extends AbstractScriptFieldType<DoubleF
 
     private static class Builder extends AbstractScriptFieldType.Builder<DoubleFieldScript.Factory> {
         Builder(String name) {
-            super(name, DoubleFieldScript.CONTEXT, DoubleFieldScript.PARSE_FROM_SOURCE);
+            super(name, DoubleFieldScript.CONTEXT);
         }
 
         @Override
@@ -47,10 +49,20 @@ AbstractScriptFieldType<?> createFieldType(String name,
                                                    Map<String, String> meta) {
             return new DoubleScriptFieldType(name, factory, script, meta);
         }
+
+        @Override
+        DoubleFieldScript.Factory getParseFromSourceFactory() {
+            return DoubleFieldScript.PARSE_FROM_SOURCE;
+        }
+
+        @Override
+        DoubleFieldScript.Factory getObjectSubfieldFactory(Function<SearchLookup, ObjectFieldScript.LeafFactory> parentScriptFactory) {
+            return DoubleFieldScript.objectAdapter(parentScriptFactory);
+        }
     }
 
     public static RuntimeField sourceOnly(String name) {
-        return new Builder(name).createRuntimeField(DoubleFieldScript.PARSE_FROM_SOURCE);
+        return new Builder(name).createRuntimeField(null, DoubleFieldScript.PARSE_FROM_SOURCE);
     }
 
     DoubleScriptFieldType(

server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java

@@ -197,7 +197,7 @@ private static boolean applyMatchingTemplate(DocumentParserContext context,
             if (parser == null) {
                 throw new MapperParsingException("failed to find type parsed [" + mappingType + "] for [" + fullName + "]");
             }
-            RuntimeField runtimeField = parser.parse(fullName, mapping, parserContext);
+            RuntimeField runtimeField = parser.parse(fullName, mapping, parserContext, null, null);
             Runtime.createDynamicField(runtimeField, context);
         } else {
             Mapper.Builder builder = parseDynamicTemplateMapping(name, mappingType, mapping, dateFormatter, context);

server/src/main/java/org/elasticsearch/index/mapper/GeoPointScriptFieldType.java

@@ -22,6 +22,7 @@
 import org.elasticsearch.index.fielddata.GeoPointScriptFieldData;
 import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.script.GeoPointFieldScript;
+import org.elasticsearch.script.ObjectFieldScript;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.search.lookup.SearchLookup;
 import org.elasticsearch.search.runtime.GeoPointScriptFieldDistanceFeatureQuery;
@@ -31,19 +32,31 @@
 import java.time.ZoneId;
 import java.util.Arrays;
 import java.util.Map;
+import java.util.function.Function;
 import java.util.function.Supplier;
 
 public final class GeoPointScriptFieldType extends AbstractScriptFieldType<GeoPointFieldScript.LeafFactory> implements GeoShapeQueryable {
 
     public static final RuntimeField.Parser PARSER = new RuntimeField.Parser(name ->
-        new Builder<>(name, GeoPointFieldScript.CONTEXT, GeoPointFieldScript.PARSE_FROM_SOURCE) {
+        new Builder<>(name, GeoPointFieldScript.CONTEXT) {
             @Override
             AbstractScriptFieldType<?> createFieldType(String name,
                                                        GeoPointFieldScript.Factory factory,
                                                        Script script,
                                                        Map<String, String> meta) {
                 return new GeoPointScriptFieldType(name, factory, getScript(), meta());
             }
+
+            @Override
+            GeoPointFieldScript.Factory getParseFromSourceFactory() {
+                return GeoPointFieldScript.PARSE_FROM_SOURCE;
+            }
+
+            @Override
+            GeoPointFieldScript.Factory getObjectSubfieldFactory(
+                Function<SearchLookup, ObjectFieldScript.LeafFactory> parentScriptFactory) {
+                return GeoPointFieldScript.objectAdapter(parentScriptFactory);
+            }
         });
 
     GeoPointScriptFieldType(

server/src/main/java/org/elasticsearch/index/mapper/IpScriptFieldType.java

@@ -22,6 +22,7 @@
 import org.elasticsearch.index.fielddata.IpScriptFieldData;
 import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.script.IpFieldScript;
+import org.elasticsearch.script.ObjectFieldScript;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.lookup.SearchLookup;
@@ -37,19 +38,30 @@
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.function.Function;
 import java.util.function.Supplier;
 
 public final class IpScriptFieldType extends AbstractScriptFieldType<IpFieldScript.LeafFactory> {
 
     public static final RuntimeField.Parser PARSER = new RuntimeField.Parser(name ->
-        new Builder<>(name, IpFieldScript.CONTEXT, IpFieldScript.PARSE_FROM_SOURCE) {
+        new Builder<>(name, IpFieldScript.CONTEXT) {
             @Override
             AbstractScriptFieldType<?> createFieldType(String name,
                                                        IpFieldScript.Factory factory,
                                                        Script script,
                                                        Map<String, String> meta) {
                 return new IpScriptFieldType(name, factory, getScript(), meta());
             }
+
+            @Override
+            IpFieldScript.Factory getParseFromSourceFactory() {
+                return IpFieldScript.PARSE_FROM_SOURCE;
+            }
+
+            @Override
+            IpFieldScript.Factory getObjectSubfieldFactory(Function<SearchLookup, ObjectFieldScript.LeafFactory> parentScriptFactory) {
+                return IpFieldScript.objectAdapter(parentScriptFactory);
+            }
         });
 
     IpScriptFieldType(

server/src/main/java/org/elasticsearch/index/mapper/KeywordScriptFieldType.java

@@ -16,6 +16,7 @@
 import org.elasticsearch.common.unit.Fuzziness;
 import org.elasticsearch.index.fielddata.StringScriptFieldData;
 import org.elasticsearch.index.query.SearchExecutionContext;
+import org.elasticsearch.script.ObjectFieldScript;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.StringFieldScript;
 import org.elasticsearch.search.lookup.SearchLookup;
@@ -33,6 +34,7 @@
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
+import java.util.function.Function;
 import java.util.function.Supplier;
 
 import static java.util.stream.Collectors.toSet;
@@ -43,7 +45,7 @@ public final class KeywordScriptFieldType extends AbstractScriptFieldType<String
 
     private static class Builder extends AbstractScriptFieldType.Builder<StringFieldScript.Factory> {
         Builder(String name) {
-            super(name, StringFieldScript.CONTEXT, StringFieldScript.PARSE_FROM_SOURCE);
+            super(name, StringFieldScript.CONTEXT);
         }
 
         @Override
@@ -53,10 +55,20 @@ AbstractScriptFieldType<?> createFieldType(String name,
                                                    Map<String, String> meta) {
             return new KeywordScriptFieldType(name, factory, script, meta);
         }
+
+        @Override
+        StringFieldScript.Factory getParseFromSourceFactory() {
+            return StringFieldScript.PARSE_FROM_SOURCE;
+        }
+
+        @Override
+        StringFieldScript.Factory getObjectSubfieldFactory(Function<SearchLookup, ObjectFieldScript.LeafFactory> parentScriptFactory) {
+            return StringFieldScript.objectAdapter(parentScriptFactory);
+        }
     }
 
     public static RuntimeField sourceOnly(String name) {
-        return new Builder(name).createRuntimeField(StringFieldScript.PARSE_FROM_SOURCE);
+        return new Builder(name).createRuntimeField(null, StringFieldScript.PARSE_FROM_SOURCE);
     }
 
     public KeywordScriptFieldType(