Emit multiple fields from a runtime field script

modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/org.elasticsearch.script.composite_field.txt

@@ -0,0 +1,21 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License
+# 2.0 and the Server Side Public License, v 1; you may not use this file except
+# in compliance with, at your election, the Elastic License 2.0 or the Server
+# Side Public License, v 1.
+#
+
+# The whitelist for composite runtime fields
+
+# These two whitelists are required for painless to find the classes
+class org.elasticsearch.script.CompositeFieldScript @no_import {
+}
+class org.elasticsearch.script.CompositeFieldScript$Factory @no_import {
+}
+
+static_import {
+    # The `emit` callback to collect values for the fields
+    void emit(org.elasticsearch.script.CompositeFieldScript, String, Object) bound_to org.elasticsearch.script.CompositeFieldScript$EmitField
+    void emit(org.elasticsearch.script.CompositeFieldScript, Map) bound_to org.elasticsearch.script.CompositeFieldScript$EmitMap
+}

modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/110_composite.yml

@@ -0,0 +1,104 @@
+---
+setup:
+  - do:
+      indices.create:
+        index: http_logs
+        body:
+          settings:
+            number_of_shards: 1
+            number_of_replicas: 0
+          mappings:
+            runtime:
+              http:
+                type: composite
+                script:
+                  source: |
+                    emit(grok('%{COMMONAPACHELOG}').extract(doc["message"].value));
+                fields:
+                  clientip:
+                    type: ip
+                  verb:
+                    type: keyword
+                  response:
+                    type: long
+            properties:
+              timestamp:
+                type: date
+              message:
+                type: keyword
+  - do:
+      bulk:
+        index: http_logs
+        refresh: true
+        body: |
+          {"index":{}}
+          {"timestamp": "1998-04-30T14:30:17-05:00", "message" : "40.135.0.0 - - [30/Apr/1998:14:30:17 -0500] \"GET /images/hm_bg.jpg HTTP/1.0\" 200 24736"}
+          {"index":{}}
+          {"timestamp": "1998-04-30T14:30:53-05:00", "message" : "232.0.0.0 - - [30/Apr/1998:14:30:53 -0500] \"GET /images/hm_bg.jpg HTTP/1.0\" 200 24736"}
+          {"index":{}}
+          {"timestamp": "1998-04-30T14:31:12-05:00", "message" : "26.1.0.0 - - [30/Apr/1998:14:31:12 -0500] \"GET /images/hm_bg.jpg HTTP/1.0\" 200 24736"}
+          {"index":{}}
+          {"timestamp": "1998-04-30T14:31:19-05:00", "message" : "247.37.0.0 - - [30/Apr/1998:14:31:19 -0500] \"GET /french/splash_inet.html HTTP/1.0\" 200 3781"}
+          {"index":{}}
+          {"timestamp": "1998-04-30T14:31:22-05:00", "message" : "247.37.0.0 - - [30/Apr/1998:14:31:22 -0500] \"GET /images/hm_nbg.jpg HTTP/1.0\" 304 0"}
+          {"index":{}}
+          {"timestamp": "1998-04-30T14:31:27-05:00", "message" : "252.0.0.0 - - [30/Apr/1998:14:31:27 -0500] \"GET /images/hm_bg.jpg HTTP/1.0\" 200 24736"}
+          {"index":{}}
+          {"timestamp": "1998-04-30T14:31:28-05:00", "message" : "not a valid apache log"}
+
+---
+fetch:
+  - do:
+      search:
+        index: http_logs
+        body:
+          sort: timestamp
+          fields:
+            - http.clientip
+            - http.verb
+            - http.response
+  - match: {hits.total.value: 7}
+  - match: {hits.hits.0.fields.http\.clientip: [40.135.0.0] }
+  - match: {hits.hits.0.fields.http\.verb: [GET] }
+  - match: {hits.hits.0.fields.http\.response: [200] }
+  - is_false: hits.hits.6.fields.http\.clientip
+  - is_false: hits.hits.6.fields.http\.verb
+  - is_false: hits.hits.6.fields.http\.response
+
+---
+query:
+  - do:
+      search:
+        index: http_logs
+        body:
+          query:
+            term:
+              http.verb: GET
+  - match: { hits.total.value: 6 }
+
+  - do:
+      search:
+        index: http_logs
+        body:
+          query:
+            range:
+              http.clientip:
+                from: 232.0.0.0
+                to: 253.0.0.0
+  - match: { hits.total.value: 4 }
+
+---
+"terms agg":
+  - do:
+      search:
+        index: http_logs
+        body:
+          aggs:
+            response:
+              terms:
+                field: http.response
+  - match: {hits.total.value: 7}
+  - match: {aggregations.response.buckets.0.key: 200 }
+  - match: {aggregations.response.buckets.0.doc_count: 5 }
+  - match: {aggregations.response.buckets.1.key: 304 }
+  - match: {aggregations.response.buckets.1.doc_count: 1 }

server/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldType.java

@@ -18,6 +18,7 @@
 import org.elasticsearch.common.time.DateMathParser;
 import org.elasticsearch.common.unit.Fuzziness;
 import org.elasticsearch.index.query.SearchExecutionContext;
+import org.elasticsearch.script.CompositeFieldScript;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptContext;
 import org.elasticsearch.script.ScriptType;
@@ -194,33 +195,42 @@ protected final LeafFactory leafFactory(SearchExecutionContext context) {
 
     abstract static class Builder<Factory> extends RuntimeField.Builder {
         private final ScriptContext<Factory> scriptContext;
-        private final Factory parseFromSourceFactory;
 
         final FieldMapper.Parameter<Script> script = new FieldMapper.Parameter<>(
             "script",
             true,
             () -> null,
             Builder::parseScript,
-            initializerNotSupported()
+            RuntimeField.initializerNotSupported()
         ).setSerializerCheck((id, ic, v) -> ic);
 
-        Builder(String name, ScriptContext<Factory> scriptContext, Factory parseFromSourceFactory) {
+        Builder(String name, ScriptContext<Factory> scriptContext) {
             super(name);
             this.scriptContext = scriptContext;
-            this.parseFromSourceFactory = parseFromSourceFactory;
         }
 
+        abstract Factory getParseFromSourceFactory();
+
+        abstract Factory getCompositeLeafFactory(Function<SearchLookup, CompositeFieldScript.LeafFactory> parentScriptFactory);
+
         @Override
-        protected final RuntimeField createRuntimeField(MappingParserContext parserContext) {
+        protected final RuntimeField createRuntimeField(MappingParserContext parserContext,
+                                                        String parent,
+                                                        Function<SearchLookup, CompositeFieldScript.LeafFactory> parentScriptFactory) {
             if (script.get() == null) {
-                return createRuntimeField(parseFromSourceFactory);
+                if (parentScriptFactory == null) {
+                    return createRuntimeField(parent, getParseFromSourceFactory());
+                }
+                return createRuntimeField(parent, getCompositeLeafFactory(parentScriptFactory));
             }
+            assert parent == null && parentScriptFactory == null : "a script is set, hence we can't be parsing sub-fields";
             Factory factory = parserContext.scriptCompiler().compile(script.getValue(), scriptContext);
-            return createRuntimeField(factory);
+            return createRuntimeField(null, factory);
         }
 
-        final RuntimeField createRuntimeField(Factory scriptFactory) {
-            AbstractScriptFieldType<?> fieldType = createFieldType(name, scriptFactory, getScript(), meta());
+        final RuntimeField createRuntimeField(String parent, Factory scriptFactory) {
+            String fullName = parent == null ? name : parent + "." + name;
+            AbstractScriptFieldType<?> fieldType = createFieldType(fullName, scriptFactory, getScript(), meta());
             return new LeafRuntimeField(name, fieldType, getParameters());
         }
 
@@ -240,16 +250,12 @@ protected final Script getScript() {
             return script.get();
         }
 
-        private static Script parseScript(String name, MappingParserContext parserContext, Object scriptObject) {
+        static Script parseScript(String name, MappingParserContext parserContext, Object scriptObject) {
             Script script = Script.parse(scriptObject);
             if (script.getType() == ScriptType.STORED) {
                 throw new IllegalArgumentException("stored scripts are not supported for runtime field [" + name + "]");
             }
             return script;
         }
     }
-
-    static <T> Function<FieldMapper, T> initializerNotSupported() {
-        return mapper -> { throw new UnsupportedOperationException(); };
-    }
 }

server/src/main/java/org/elasticsearch/index/mapper/BooleanScriptFieldType.java

@@ -18,6 +18,7 @@
 import org.elasticsearch.index.fielddata.BooleanScriptFieldData;
 import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.script.BooleanFieldScript;
+import org.elasticsearch.script.CompositeFieldScript;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.lookup.SearchLookup;
@@ -27,6 +28,7 @@
 import java.time.ZoneId;
 import java.util.Collection;
 import java.util.Map;
+import java.util.function.Function;
 import java.util.function.Supplier;
 
 public final class BooleanScriptFieldType extends AbstractScriptFieldType<BooleanFieldScript.LeafFactory> {
@@ -35,7 +37,7 @@ public final class BooleanScriptFieldType extends AbstractScriptFieldType<Boolea
 
     private static class Builder extends AbstractScriptFieldType.Builder<BooleanFieldScript.Factory> {
         Builder(String name) {
-            super(name, BooleanFieldScript.CONTEXT, BooleanFieldScript.PARSE_FROM_SOURCE);
+            super(name, BooleanFieldScript.CONTEXT);
         }
 
         @Override
@@ -45,10 +47,21 @@ AbstractScriptFieldType<?> createFieldType(String name,
                                                    Map<String, String> meta) {
             return new BooleanScriptFieldType(name, factory, script, meta);
         }
+
+        @Override
+        BooleanFieldScript.Factory getParseFromSourceFactory() {
+            return BooleanFieldScript.PARSE_FROM_SOURCE;
+        }
+
+        @Override
+        BooleanFieldScript.Factory getCompositeLeafFactory(Function<SearchLookup, CompositeFieldScript.LeafFactory> parentScriptFactory) {
+            return BooleanFieldScript.leafAdapter(parentScriptFactory);
+        }
+
     }
 
     public static RuntimeField sourceOnly(String name) {
-        return new Builder(name).createRuntimeField(BooleanFieldScript.PARSE_FROM_SOURCE);
+        return new Builder(name).createRuntimeField(null, BooleanFieldScript.PARSE_FROM_SOURCE);
     }
 
     BooleanScriptFieldType(

server/src/main/java/org/elasticsearch/index/mapper/CompositeRuntimeField.java

@@ -0,0 +1,130 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.index.mapper;
+
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.script.CompositeFieldScript;
+import org.elasticsearch.script.Script;
+import org.elasticsearch.search.lookup.SearchLookup;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+/**
+ * A runtime field of type object. Defines a script at the top level, which emits multiple sub-fields.
+ * The sub-fields are declared within the object in order to be made available to the field_caps and search API.
+ */
+public class CompositeRuntimeField implements RuntimeField {
+
+    public static final String CONTENT_TYPE = "composite";
+
+    public static final Parser PARSER = new Parser(name ->
+        new RuntimeField.Builder(name) {
+            private final FieldMapper.Parameter<Script> script = new FieldMapper.Parameter<>(
+                "script",
+                false,
+                () -> null,
+                AbstractScriptFieldType.Builder::parseScript,
+                RuntimeField.initializerNotSupported()
+            ).setValidator(s -> {
+                if (s == null) {
+                    throw new IllegalArgumentException("composite runtime field [" + name + "] must declare a [script]");
+                }
+            });
+
+            private final FieldMapper.Parameter<Map<String, Object>> fields = new FieldMapper.Parameter<Map<String, Object>>(
+                "fields",
+                false,
+                Collections::emptyMap,
+                (f, p, o) -> parseFields(f, o),
+                RuntimeField.initializerNotSupported()
+            ).setValidator(objectMap -> {
+                if (objectMap == null || objectMap.isEmpty()) {
+                    throw new IllegalArgumentException("composite runtime field [" + name + "] must declare its [fields]");
+                }
+            });
+
+            @Override
+            protected List<FieldMapper.Parameter<?>> getParameters() {
+                List<FieldMapper.Parameter<?>> parameters = new ArrayList<>(super.getParameters());
+                parameters.add(script);
+                parameters.add(fields);
+                return Collections.unmodifiableList(parameters);
+            }
+
+            @Override
+            protected RuntimeField createRuntimeField(MappingParserContext parserContext,
+                                                      String parent,
+                                                      Function<SearchLookup, CompositeFieldScript.LeafFactory> parentScriptFactory) {
+                if (parent != null) {
+                    throw new IllegalArgumentException(
+                        "Runtime field [" + name + "] of type [composite] cannot be nested within field [" + parent + "]"
+                    );
+                }
+                assert parentScriptFactory == null : "parent is null, we can't be parsing subfields";
+                CompositeFieldScript.Factory factory = parserContext.scriptCompiler().compile(script.get(), CompositeFieldScript.CONTEXT);
+                Map<String, RuntimeField> runtimeFields = RuntimeField.parseRuntimeFields(fields.getValue(),
+                    parserContext, name, searchLookup -> factory.newFactory(name, script.get().getParams(), searchLookup), false);
+                return new CompositeRuntimeField(name, getParameters(), runtimeFields.values());
+            }
+        });
+
+    private final String name;
+    private final List<FieldMapper.Parameter<?>> parameters;
+    private final Collection<RuntimeField> subfields;
+
+    CompositeRuntimeField(String name, List<FieldMapper.Parameter<?>> parameters, Collection<RuntimeField> subfields) {
+        this.name = name;
+        this.parameters = parameters;
+        this.subfields = subfields;
+    }
+
+    @Override
+    public String name() {
+        return name;
+    }
+
+    @Override
+    public Collection<MappedFieldType> asMappedFieldTypes() {
+        return subfields.stream().flatMap(runtimeField -> runtimeField.asMappedFieldTypes().stream()).collect(Collectors.toList());
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject(name);
+        builder.field("type", "composite");
+        boolean includeDefaults = params.paramAsBoolean("include_defaults", false);
+        for (FieldMapper.Parameter<?> parameter : parameters) {
+            parameter.toXContent(builder, includeDefaults);
+        }
+        builder.startObject("fields");
+        for (RuntimeField subfield : subfields) {
+            subfield.toXContent(builder, params);
+        }
+        builder.endObject();
+        builder.endObject();
+        return builder;
+    }
+
+    private static Map<String, Object> parseFields(String name, Object fieldsObject) {
+        if (fieldsObject instanceof Map == false) {
+            throw new MapperParsingException("[fields] must be an object, got " + fieldsObject.getClass().getSimpleName() +
+                "[" + fieldsObject + "] for field [" + name +"]");
+        }
+        @SuppressWarnings("unchecked")
+        Map<String, Object> fields = (Map<String, Object>) fieldsObject;
+        return fields;
+    }
+}