Add support for script parameter to boolean field mapper (elastic#71454)

Relates to elastic#68984

docs/reference/mapping/types/boolean.asciidoc

@@ -188,7 +188,27 @@ The following parameters are accepted by `boolean` fields:
 
     Accepts any of the true or false values listed above. The value is
     substituted for any explicit `null` values. Defaults to `null`, which
-    means the field is treated as missing.
+    means the field is treated as missing. Note that this cannot be set
+    if the `script` parameter is used.
+
+`on_script_error`::
+
+    Defines what to do if the script defined by the `script` parameter
+    throws an error at indexing time. Accepts `reject` (default), which
+    will cause the entire document to be rejected, and `ignore`, which
+    will register the field in the document's
+    <<mapping-ignored-field,`_ignored`>> metadata field and continue
+    indexing. This parameter can only be set if the `script` field is
+    also set.
+
+`script`::
+
+    If this parameter is set, then the field will index values generated
+    by this script, rather than reading the values directly from the
+    source. If a value is set for this field on the input document, then
+    the document will be rejected with an error.
+    Scripts are in the same format as their
+    <<runtime-mapping-fields,runtime equivalent>>.
 
 <<mapping-store,`store`>>::
 

modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java

@@ -545,7 +545,7 @@ public SearchAsYouTypeFieldMapper(String simpleName,
                                       PrefixFieldMapper prefixField,
                                       ShingleFieldMapper[] shingleFields,
                                       Builder builder) {
-        super(simpleName, mappedFieldType, indexAnalyzers, MultiFields.empty(), copyTo);
+        super(simpleName, mappedFieldType, indexAnalyzers, MultiFields.empty(), copyTo, false, null);
         this.prefixField = prefixField;
         this.shingleFields = shingleFields;
         this.maxShingleSize = builder.maxShingleSize.getValue();

modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/63_boolean_calculated_at_index.yml

@@ -0,0 +1,134 @@
+---
+setup:
+  - do:
+      indices.create:
+        index: sensor
+        body:
+          settings:
+            number_of_shards: 1
+            number_of_replicas: 0
+          mappings:
+            properties:
+              timestamp:
+                type: date
+              temperature:
+                type: long
+              voltage:
+                type: double
+              node:
+                type: keyword
+              over_v:
+                type: boolean
+                script:
+                  source: |
+                    for (def v : doc['voltage']) {
+                      emit(v >= params.min_v);
+                    }
+                  params:
+                    min_v: 5.0
+              # Test fetching from _source
+              over_v_from_source:
+                type: boolean
+                script:
+                  source: |
+                    emit(params._source.voltage >= 5.0);
+              # Test many booleans
+              big_vals:
+                type: boolean
+                script:
+                  source: |
+                    for (def v : doc['temperature']) {
+                      emit(v >= 200);
+                    }
+                    for (def v : doc['voltage']) {
+                      emit(v >= 5.0);
+                    }
+
+
+  - do:
+      bulk:
+        index: sensor
+        refresh: true
+        body: |
+          {"index":{}}
+          {"timestamp": 1516729294000, "temperature": 200, "voltage": 5.2, "node": "a"}
+          {"index":{}}
+          {"timestamp": 1516642894000, "temperature": 201, "voltage": 5.8, "node": "b"}
+          {"index":{}}
+          {"timestamp": 1516556494000, "temperature": 202, "voltage": 5.1, "node": "a"}
+          {"index":{}}
+          {"timestamp": 1516470094000, "temperature": 198, "voltage": 5.6, "node": "b"}
+          {"index":{}}
+          {"timestamp": 1516383694000, "temperature": 200, "voltage": 4.2, "node": "c"}
+          {"index":{}}
+          {"timestamp": 1516297294000, "temperature": 202, "voltage": 4.0, "node": "c"}
+
+---
+"get mapping":
+  - do:
+      indices.get_mapping:
+        index: sensor
+  - match: {sensor.mappings.properties.over_v.type: boolean }
+  - match:
+      sensor.mappings.properties.over_v.script.source: |
+        for (def v : doc['voltage']) {
+          emit(v >= params.min_v);
+        }
+  - match: {sensor.mappings.properties.over_v.script.params: {min_v: 5.0} }
+  - match: {sensor.mappings.properties.over_v.script.lang: painless }
+
+---
+"fetch fields":
+  - do:
+      search:
+        index: sensor
+        body:
+          sort: timestamp
+          fields: [over_v, over_v_from_source, big_vals]
+  - match: {hits.total.value: 6}
+  - match: {hits.hits.0.fields.over_v: [false] }
+  - match: {hits.hits.0.fields.over_v_from_source: [false] }
+  - match: {hits.hits.0.fields.big_vals: [false, true] } # doc values are sorted with falses before trues
+
+---
+"docvalue_fields":
+  - do:
+      search:
+        index: sensor
+        body:
+          sort: timestamp
+          docvalue_fields: [over_v, over_v_from_source, big_vals]
+  - match: {hits.total.value: 6}
+  - match: {hits.hits.0.fields.over_v: [false] }
+  - match: {hits.hits.0.fields.over_v_from_source: [false] }
+  - match: {hits.hits.0.fields.big_vals: [false, true] } # doc values are sorted with falses before trues
+
+---
+"terms agg":
+  - do:
+      search:
+        index: sensor
+        body:
+          aggs:
+            over_v:
+              terms:
+                field: over_v
+  - match: {hits.total.value: 6}
+  - match: {aggregations.over_v.buckets.0.key_as_string: "true"}
+  - match: {aggregations.over_v.buckets.0.doc_count: 4}
+  - match: {aggregations.over_v.buckets.1.key_as_string: "false"}
+  - match: {aggregations.over_v.buckets.1.doc_count: 2}
+
+---
+"term query":
+  - do:
+      search:
+        index: sensor
+        body:
+          query:
+            term:
+              over_v: true
+          sort:
+            timestamp: asc
+  - match: {hits.total.value: 4}
+  - match: {hits.hits.0._source.voltage: 5.6}

server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java

@@ -142,7 +142,7 @@ protected AbstractGeometryFieldMapper(String simpleName, MappedFieldType mappedF
                                           Explicit<Boolean> ignoreMalformed, Explicit<Boolean> ignoreZValue,
                                           MultiFields multiFields, CopyTo copyTo,
                                           Indexer<Parsed, Processed> indexer, Parser<Parsed> parser) {
-        super(simpleName, mappedFieldType, indexAnalyzers, multiFields, copyTo);
+        super(simpleName, mappedFieldType, indexAnalyzers, multiFields, copyTo, false, null);
         this.ignoreMalformed = ignoreMalformed;
         this.ignoreZValue = ignoreZValue;
         this.indexer = indexer;

server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java

@@ -13,6 +13,7 @@
 import org.apache.lucene.document.SortedNumericDocValuesField;
 import org.apache.lucene.document.StoredField;
 import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermRangeQuery;
 import org.apache.lucene.util.BytesRef;
@@ -25,7 +26,11 @@
 import org.elasticsearch.index.fielddata.IndexNumericFieldData.NumericType;
 import org.elasticsearch.index.fielddata.plain.SortedNumericIndexFieldData;
 import org.elasticsearch.index.query.SearchExecutionContext;
+import org.elasticsearch.script.BooleanFieldScript;
+import org.elasticsearch.script.Script;
+import org.elasticsearch.script.ScriptCompiler;
 import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.lookup.FieldValues;
 import org.elasticsearch.search.lookup.SearchLookup;
 
 import java.io.IOException;
@@ -34,6 +39,7 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.function.Supplier;
 
 /**
@@ -74,44 +80,69 @@ public static class Builder extends FieldMapper.Builder {
             .acceptsNull();
 
         private final Parameter<Float> boost = Parameter.boostParam();
+        private final Parameter<Script> script = Parameter.scriptParam(m -> toType(m).script);
+        private final Parameter<String> onScriptError = Parameter.onScriptErrorParam(m -> toType(m).onScriptError, script);
+
         private final Parameter<Map<String, String>> meta = Parameter.metaParam();
 
-        public Builder(String name) {
+        private final ScriptCompiler scriptCompiler;
+
+        public Builder(String name, ScriptCompiler scriptCompiler) {
             super(name);
+            this.scriptCompiler = Objects.requireNonNull(scriptCompiler);
+            this.script.precludesParameters(nullValue);
+            this.script.setValidator(s -> {
+                if (s != null && indexed.get() == false && docValues.get() == false) {
+                    throw new MapperParsingException("Cannot define script on field with index:false and doc_values:false");
+                }
+            });
         }
 
         @Override
         protected List<Parameter<?>> getParameters() {
-            return Arrays.asList(meta, boost, docValues, indexed, nullValue, stored);
+            return Arrays.asList(meta, boost, docValues, indexed, nullValue, stored, script, onScriptError);
         }
 
         @Override
         public BooleanFieldMapper build(ContentPath contentPath) {
             MappedFieldType ft = new BooleanFieldType(buildFullName(contentPath), indexed.getValue(), stored.getValue(),
-                docValues.getValue(), nullValue.getValue(), meta.getValue());
+                docValues.getValue(), nullValue.getValue(), scriptValues(), meta.getValue());
             ft.setBoost(boost.getValue());
             return new BooleanFieldMapper(name, ft, multiFieldsBuilder.build(this, contentPath), copyTo.build(), this);
         }
+
+        private FieldValues<Boolean> scriptValues() {
+            if (script.get() == null) {
+                return null;
+            }
+            BooleanFieldScript.Factory scriptFactory = scriptCompiler.compile(script.get(), BooleanFieldScript.CONTEXT);
+            return scriptFactory == null ? null : (lookup, ctx, doc, consumer) -> scriptFactory
+                .newFactory(name, script.get().getParams(), lookup)
+                .newInstance(ctx)
+                .runForDoc(doc, consumer);
+        }
     }
 
-    public static final TypeParser PARSER = new TypeParser((n, c) -> new Builder(n));
+    public static final TypeParser PARSER = new TypeParser((n, c) -> new Builder(n, c.scriptCompiler()));
 
     public static final class BooleanFieldType extends TermBasedFieldType {
 
         private final Boolean nullValue;
+        private final FieldValues<Boolean> scriptValues;
 
         public BooleanFieldType(String name, boolean isSearchable, boolean isStored, boolean hasDocValues,
-                                Boolean nullValue, Map<String, String> meta) {
+                                Boolean nullValue, FieldValues<Boolean> scriptValues, Map<String, String> meta) {
             super(name, isSearchable, isStored, hasDocValues, TextSearchInfo.SIMPLE_MATCH_ONLY, meta);
             this.nullValue = nullValue;
+            this.scriptValues = scriptValues;
         }
 
         public BooleanFieldType(String name) {
-            this(name, true, false, true, false, Collections.emptyMap());
+            this(name, true, false, true, false, null, Collections.emptyMap());
         }
 
         public BooleanFieldType(String name, boolean searchable) {
-            this(name, searchable, false, true, false, Collections.emptyMap());
+            this(name, searchable, false, true, false, null, Collections.emptyMap());
         }
 
         @Override
@@ -124,7 +155,9 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format)
             if (format != null) {
                 throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
             }
-
+            if (this.scriptValues != null) {
+                return FieldValues.valueFetcher(this.scriptValues, context);
+            }
             return new SourceValueFetcher(name(), context, nullValue) {
                 @Override
                 protected Boolean parseSourceValue(Object value) {
@@ -211,14 +244,21 @@ public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower
     private final boolean indexed;
     private final boolean hasDocValues;
     private final boolean stored;
+    private final Script script;
+    private final FieldValues<Boolean> scriptValues;
+    private final ScriptCompiler scriptCompiler;
 
     protected BooleanFieldMapper(String simpleName, MappedFieldType mappedFieldType,
                                  MultiFields multiFields, CopyTo copyTo, Builder builder) {
-        super(simpleName, mappedFieldType, Lucene.KEYWORD_ANALYZER, multiFields, copyTo);
+        super(simpleName, mappedFieldType, Lucene.KEYWORD_ANALYZER, multiFields, copyTo,
+            builder.script.get() != null, builder.onScriptError.getValue());
         this.nullValue = builder.nullValue.getValue();
         this.stored = builder.stored.getValue();
         this.indexed = builder.indexed.getValue();
         this.hasDocValues = builder.docValues.getValue();
+        this.script = builder.script.get();
+        this.scriptValues = builder.scriptValues();
+        this.scriptCompiler = builder.scriptCompiler;
     }
 
     @Override
@@ -243,7 +283,10 @@ protected void parseCreateField(ParseContext context) throws IOException {
                 value = context.parser().booleanValue();
             }
         }
+        indexValue(context, value);
+    }
 
+    private void indexValue(ParseContext context, Boolean value) {
         if (value == null) {
             return;
         }
@@ -260,14 +303,18 @@ protected void parseCreateField(ParseContext context) throws IOException {
         }
     }
 
+    @Override
+    protected void indexScriptValues(SearchLookup searchLookup, LeafReaderContext readerContext, int doc, ParseContext parseContext) {
+        this.scriptValues.valuesForDoc(searchLookup, readerContext, doc, value -> indexValue(parseContext, value));
+    }
+
     @Override
     public FieldMapper.Builder getMergeBuilder() {
-        return new Builder(simpleName()).init(this);
+        return new Builder(simpleName(), scriptCompiler).init(this);
     }
 
     @Override
     protected String contentType() {
         return CONTENT_TYPE;
     }
-
 }

server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java

@@ -15,6 +15,7 @@
 import org.elasticsearch.common.time.DateFormatter;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.mapper.ObjectMapper.Dynamic;
+import org.elasticsearch.script.ScriptCompiler;
 
 import java.io.IOException;
 import java.time.format.DateTimeParseException;
@@ -274,7 +275,7 @@ public void newDynamicLongField(ParseContext context, String name) throws IOExce
                 new NumberFieldMapper.Builder(
                     name,
                     NumberFieldMapper.NumberType.LONG,
-                    null,
+                    ScriptCompiler.NONE,
                     context.indexSettings().getSettings()
                 ), context);
         }
@@ -287,13 +288,13 @@ public void newDynamicDoubleField(ParseContext context, String name) throws IOEx
             createDynamicField(new NumberFieldMapper.Builder(
                 name,
                 NumberFieldMapper.NumberType.FLOAT,
-                null,
+                ScriptCompiler.NONE,
                 context.indexSettings().getSettings()), context);
         }
 
         @Override
         public void newDynamicBooleanField(ParseContext context, String name) throws IOException {
-            createDynamicField(new BooleanFieldMapper.Builder(name), context);
+            createDynamicField(new BooleanFieldMapper.Builder(name, ScriptCompiler.NONE), context);
         }
 
         @Override