Emit multiple fields from a runtime field script

modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/org.elasticsearch.script.composite_field.txt

@@ -0,0 +1,21 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License
+# 2.0 and the Server Side Public License, v 1; you may not use this file except
+# in compliance with, at your election, the Elastic License 2.0 or the Server
+# Side Public License, v 1.
+#
+
+# The whitelist for composite runtime fields
+
+# These two whitelists are required for painless to find the classes
+class org.elasticsearch.script.CompositeFieldScript @no_import {
+}
+class org.elasticsearch.script.CompositeFieldScript$Factory @no_import {
+}
+
+static_import {
+    # The `emit` callback to collect values for the fields
+    void emit(org.elasticsearch.script.CompositeFieldScript, String, Object) bound_to org.elasticsearch.script.CompositeFieldScript$EmitField
+    void emit(org.elasticsearch.script.CompositeFieldScript, Map) bound_to org.elasticsearch.script.CompositeFieldScript$EmitMap
+}

modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/110_composite.yml

@@ -0,0 +1,104 @@
+---
+setup:
+  - do:
+      indices.create:
+        index: http_logs
+        body:
+          settings:
+            number_of_shards: 1
+            number_of_replicas: 0
+          mappings:
+            runtime:
+              http:
+                type: composite
+                script:
+                  source: |
+                    emit(grok('%{COMMONAPACHELOG}').extract(doc["message"].value));
+                fields:
+                  clientip:
+                    type: ip
+                  verb:
+                    type: keyword
+                  response:
+                    type: long
+            properties:
+              timestamp:
+                type: date
+              message:
+                type: keyword
+  - do:
+      bulk:
+        index: http_logs
+        refresh: true
+        body: |
+          {"index":{}}
+          {"timestamp": "1998-04-30T14:30:17-05:00", "message" : "40.135.0.0 - - [30/Apr/1998:14:30:17 -0500] \"GET /images/hm_bg.jpg HTTP/1.0\" 200 24736"}
+          {"index":{}}
+          {"timestamp": "1998-04-30T14:30:53-05:00", "message" : "232.0.0.0 - - [30/Apr/1998:14:30:53 -0500] \"GET /images/hm_bg.jpg HTTP/1.0\" 200 24736"}
+          {"index":{}}
+          {"timestamp": "1998-04-30T14:31:12-05:00", "message" : "26.1.0.0 - - [30/Apr/1998:14:31:12 -0500] \"GET /images/hm_bg.jpg HTTP/1.0\" 200 24736"}
+          {"index":{}}
+          {"timestamp": "1998-04-30T14:31:19-05:00", "message" : "247.37.0.0 - - [30/Apr/1998:14:31:19 -0500] \"GET /french/splash_inet.html HTTP/1.0\" 200 3781"}
+          {"index":{}}
+          {"timestamp": "1998-04-30T14:31:22-05:00", "message" : "247.37.0.0 - - [30/Apr/1998:14:31:22 -0500] \"GET /images/hm_nbg.jpg HTTP/1.0\" 304 0"}
+          {"index":{}}
+          {"timestamp": "1998-04-30T14:31:27-05:00", "message" : "252.0.0.0 - - [30/Apr/1998:14:31:27 -0500] \"GET /images/hm_bg.jpg HTTP/1.0\" 200 24736"}
+          {"index":{}}
+          {"timestamp": "1998-04-30T14:31:28-05:00", "message" : "not a valid apache log"}
+
+---
+fetch:
+  - do:
+      search:
+        index: http_logs
+        body:
+          sort: timestamp
+          fields:
+            - http.clientip
+            - http.verb
+            - http.response
+  - match: {hits.total.value: 7}
+  - match: {hits.hits.0.fields.http\.clientip: [40.135.0.0] }
+  - match: {hits.hits.0.fields.http\.verb: [GET] }
+  - match: {hits.hits.0.fields.http\.response: [200] }
+  - is_false: hits.hits.6.fields.http\.clientip
+  - is_false: hits.hits.6.fields.http\.verb
+  - is_false: hits.hits.6.fields.http\.response
+
+---
+query:
+  - do:
+      search:
+        index: http_logs
+        body:
+          query:
+            term:
+              http.verb: GET
+  - match: { hits.total.value: 6 }
+
+  - do:
+      search:
+        index: http_logs
+        body:
+          query:
+            range:
+              http.clientip:
+                from: 232.0.0.0
+                to: 253.0.0.0
+  - match: { hits.total.value: 4 }
+
+---
+"terms agg":
+  - do:
+      search:
+        index: http_logs
+        body:
+          aggs:
+            response:
+              terms:
+                field: http.response
+  - match: {hits.total.value: 7}
+  - match: {aggregations.response.buckets.0.key: 200 }
+  - match: {aggregations.response.buckets.0.doc_count: 5 }
+  - match: {aggregations.response.buckets.1.key: 304 }
+  - match: {aggregations.response.buckets.1.doc_count: 1 }

server/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldType.java

@@ -18,6 +18,7 @@
 import org.elasticsearch.common.time.DateMathParser;
 import org.elasticsearch.common.unit.Fuzziness;
 import org.elasticsearch.index.query.SearchExecutionContext;
+import org.elasticsearch.script.CompositeFieldScript;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptContext;
 import org.elasticsearch.script.ScriptType;
@@ -194,33 +195,56 @@ protected final LeafFactory leafFactory(SearchExecutionContext context) {
 
     abstract static class Builder<Factory> extends RuntimeField.Builder {
         private final ScriptContext<Factory> scriptContext;
-        private final Factory parseFromSourceFactory;
 
         final FieldMapper.Parameter<Script> script = new FieldMapper.Parameter<>(
             "script",
             true,
             () -> null,
             Builder::parseScript,
-            initializerNotSupported()
+            RuntimeField.initializerNotSupported()
         ).setSerializerCheck((id, ic, v) -> ic);
 
-        Builder(String name, ScriptContext<Factory> scriptContext, Factory parseFromSourceFactory) {
+        Builder(String name, ScriptContext<Factory> scriptContext) {
             super(name);
             this.scriptContext = scriptContext;
-            this.parseFromSourceFactory = parseFromSourceFactory;
         }
 
+        abstract Factory getParseFromSourceFactory();
+
+        abstract Factory getCompositeLeafFactory(Function<SearchLookup, CompositeFieldScript.LeafFactory> parentScriptFactory);
+
         @Override
-        protected final RuntimeField createRuntimeField(MappingParserContext parserContext) {
+        protected RuntimeField createRuntimeField(MappingParserContext parserContext) {
             if (script.get() == null) {
-                return createRuntimeField(parseFromSourceFactory);
+                return new LeafRuntimeField(
+                    name,
+                    createFieldType(name, getParseFromSourceFactory(), getScript(), meta()),
+                    getParameters()
+                );
             }
             Factory factory = parserContext.scriptCompiler().compile(script.getValue(), scriptContext);
-            return createRuntimeField(factory);
+            return new LeafRuntimeField(name, createFieldType(name, factory, getScript(), meta()), getParameters());
         }
 
-        final RuntimeField createRuntimeField(Factory scriptFactory) {
-            AbstractScriptFieldType<?> fieldType = createFieldType(name, scriptFactory, getScript(), meta());
+        @Override
+        protected final RuntimeField createChildRuntimeField(MappingParserContext parserContext,
+                                                        String parent,
+                                                        Function<SearchLookup, CompositeFieldScript.LeafFactory> parentScriptFactory) {
+            if (script.isConfigured()) {
+                throw new IllegalArgumentException("Cannot use [script] parameter on sub-field [" + name +
+                    "] of composite field [" + parent + "]");
+            }
+            String fullName = parent + "." + name;
+            return new LeafRuntimeField(
+                name,
+                createFieldType(fullName, getCompositeLeafFactory(parentScriptFactory), getScript(), meta()),
+                getParameters()
+            );
+        }
+
+        final RuntimeField createRuntimeField(String parent, Factory scriptFactory) {
+            String fullName = parent == null ? name : parent + "." + name;
+            AbstractScriptFieldType<?> fieldType = createFieldType(fullName, scriptFactory, getScript(), meta());
             return new LeafRuntimeField(name, fieldType, getParameters());
         }
 
@@ -240,16 +264,12 @@ protected final Script getScript() {
             return script.get();
         }
 
-        private static Script parseScript(String name, MappingParserContext parserContext, Object scriptObject) {
+        static Script parseScript(String name, MappingParserContext parserContext, Object scriptObject) {
             Script script = Script.parse(scriptObject);
             if (script.getType() == ScriptType.STORED) {
                 throw new IllegalArgumentException("stored scripts are not supported for runtime field [" + name + "]");
             }
             return script;
         }
     }
-
-    static <T> Function<FieldMapper, T> initializerNotSupported() {
-        return mapper -> { throw new UnsupportedOperationException(); };
-    }
 }

server/src/main/java/org/elasticsearch/index/mapper/BooleanScriptFieldType.java

@@ -18,6 +18,7 @@
 import org.elasticsearch.index.fielddata.BooleanScriptFieldData;
 import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.script.BooleanFieldScript;
+import org.elasticsearch.script.CompositeFieldScript;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.lookup.SearchLookup;
@@ -27,6 +28,7 @@
 import java.time.ZoneId;
 import java.util.Collection;
 import java.util.Map;
+import java.util.function.Function;
 import java.util.function.Supplier;
 
 public final class BooleanScriptFieldType extends AbstractScriptFieldType<BooleanFieldScript.LeafFactory> {
@@ -35,7 +37,7 @@ public final class BooleanScriptFieldType extends AbstractScriptFieldType<Boolea
 
     private static class Builder extends AbstractScriptFieldType.Builder<BooleanFieldScript.Factory> {
         Builder(String name) {
-            super(name, BooleanFieldScript.CONTEXT, BooleanFieldScript.PARSE_FROM_SOURCE);
+            super(name, BooleanFieldScript.CONTEXT);
         }
 
         @Override
@@ -45,10 +47,21 @@ AbstractScriptFieldType<?> createFieldType(String name,
                                                    Map<String, String> meta) {
             return new BooleanScriptFieldType(name, factory, script, meta);
         }
+
+        @Override
+        BooleanFieldScript.Factory getParseFromSourceFactory() {
+            return BooleanFieldScript.PARSE_FROM_SOURCE;
+        }
+
+        @Override
+        BooleanFieldScript.Factory getCompositeLeafFactory(Function<SearchLookup, CompositeFieldScript.LeafFactory> parentScriptFactory) {
+            return BooleanFieldScript.leafAdapter(parentScriptFactory);
+        }
+
     }
 
     public static RuntimeField sourceOnly(String name) {
-        return new Builder(name).createRuntimeField(BooleanFieldScript.PARSE_FROM_SOURCE);
+        return new Builder(name).createRuntimeField(null, BooleanFieldScript.PARSE_FROM_SOURCE);
     }
 
     BooleanScriptFieldType(