diff --git a/docs/reference/mapping/types/keyword.asciidoc b/docs/reference/mapping/types/keyword.asciidoc index 29ee102d6c8b7..cf82a6ee23958 100644 --- a/docs/reference/mapping/types/keyword.asciidoc +++ b/docs/reference/mapping/types/keyword.asciidoc @@ -8,7 +8,7 @@ The keyword family includes the following field types: * <>, which is used for structured content such as IDs, email -addresses, hostnames, status codes, zip codes, or tags. +addresses, hostnames, status codes, zip codes, or tags. * <> for keyword fields that always contain the same value. * <> for unstructured machine-generated content. @@ -105,6 +105,28 @@ The following parameters are accepted by `keyword` fields: Accepts a string value which is substituted for any explicit `null` values. Defaults to `null`, which means the field is treated as missing. + Note that this cannot be set if the `script` value is used. + +`on_script_error`:: + + Defines what to do if the script defined by the `script` parameter + throws an error at indexing time. Accepts `reject` (default), which + will cause the entire document to be rejected, and `ignore`, which + will register the field in the document's + <> metadata field and continue + indexing. This parameter can only be set if the `script` field is + also set. + +`script`:: + + If this parameter is set, then the field will index values generated + by this script, rather than reading the values directly from the + source. If a value is set for this field on the input document, then + the document will be rejected with an error. + Scripts are in the same format as their + <>. Values emitted by the + script are normalized as usual, and will be ignored if they are longer + that the value set on `ignore_above`. <>:: diff --git a/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/13_keyword_calculated_at_index.yml b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/13_keyword_calculated_at_index.yml new file mode 100644 index 0000000000000..1c10a017a5c33 --- /dev/null +++ b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/13_keyword_calculated_at_index.yml @@ -0,0 +1,173 @@ +--- +setup: + - do: + indices.create: + index: sensor + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + properties: + timestamp: + type: date + temperature: + type: long + voltage: + type: double + node: + type: keyword + store: true + day_of_week: + type: keyword + script: | + emit(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT)); + # Test fetching from _source + day_of_week_from_source: + type: keyword + script: | + Instant instant = Instant.ofEpochMilli(params._source.timestamp); + ZonedDateTime dt = ZonedDateTime.ofInstant(instant, ZoneId.of("UTC")); + emit(dt.dayOfWeek.getDisplayName(TextStyle.FULL, Locale.ROOT)); + # Test fetching many values + day_of_week_letters: + type: keyword + script: | + for (String dow: doc['day_of_week']) { + for (int i = 0; i < dow.length(); i++) { + emit(dow.charAt(i).toString()); + } + } + prefixed_node: + type: keyword + script: + source: | + for (String node : params._fields.node.values) { + emit(params.prefix + node); + } + params: + prefix: node_ + + - do: + bulk: + index: sensor + refresh: true + body: | + {"index":{}} + {"timestamp": 1516729294000, "temperature": 200, "voltage": 5.2, "node": "a"} + {"index":{}} + {"timestamp": 1516642894000, "temperature": 201, "voltage": 5.8, "node": "b"} + {"index":{}} + {"timestamp": 1516556494000, "temperature": 202, "voltage": 5.1, "node": "a"} + {"index":{}} + {"timestamp": 1516470094000, "temperature": 198, "voltage": 5.6, "node": "b"} + {"index":{}} + {"timestamp": 1516383694000, "temperature": 200, "voltage": 4.2, "node": "c"} + {"index":{}} + {"timestamp": 1516297294000, "temperature": 202, "voltage": 4.0, "node": "c"} + +--- +"get mapping": + - do: + indices.get_mapping: + index: sensor + - match: {sensor.mappings.properties.day_of_week.type: keyword } + - match: + sensor.mappings.properties.day_of_week.script.source: | + emit(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT)); + - match: {sensor.mappings.properties.day_of_week.script.lang: painless } + +--- +"fetch fields": + - do: + search: + index: sensor + body: + sort: timestamp + fields: [day_of_week, day_of_week_from_source, day_of_week_letters, prefixed_node] + - match: {hits.total.value: 6} + - match: {hits.hits.0.fields.day_of_week: [Thursday] } + - match: {hits.hits.0.fields.day_of_week_from_source: [Thursday] } + - match: {hits.hits.0.fields.day_of_week_letters: [T, h, u, r, s, d, a, y] } + - match: {hits.hits.0.fields.prefixed_node: [node_c] } + +--- +"docvalue_fields": + - do: + search: + index: sensor + body: + sort: timestamp + docvalue_fields: [day_of_week, day_of_week_from_source, day_of_week_letters, prefixed_node] + - match: {hits.total.value: 6} + - match: {hits.hits.0.fields.day_of_week: [Thursday] } + - match: {hits.hits.0.fields.day_of_week_from_source: [Thursday] } + - match: {hits.hits.0.fields.day_of_week_letters: [T, a, d, h, r, s, u, y] } + - match: {hits.hits.0.fields.prefixed_node: [node_c] } + +--- +"terms agg": + - do: + search: + index: sensor + body: + size: 0 + aggs: + dow: + terms: + field: day_of_week + - match: {hits.total.value: 6} + - match: {aggregations.dow.buckets.0.key: Friday} + - match: {aggregations.dow.buckets.0.doc_count: 1} + - match: {aggregations.dow.buckets.1.key: Monday} + - match: {aggregations.dow.buckets.1.doc_count: 1} + +--- +"term query": + - do: + search: + index: sensor + body: + query: + term: + day_of_week: Monday + - match: {hits.total.value: 1} + - match: {hits.hits.0._source.voltage: 5.8} + +--- +"highlight term query": + - do: + search: + index: sensor + body: + query: + term: + day_of_week: Monday + highlight: + fields: + day_of_week: {} + + - match: { hits.hits.0.highlight.day_of_week : [ "Monday" ] } + +--- +"match query": + - do: + search: + index: sensor + body: + query: + match: + day_of_week: Monday + - match: {hits.total.value: 1} + - match: {hits.hits.0._source.voltage: 5.8} + + - do: + search: + index: sensor + body: + query: + match: + day_of_week: + query: Monday + analyzer: standard + - match: {hits.total.value: 0} diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifier.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifier.java index 58b673490c90d..ff3cb976f9911 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifier.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifier.java @@ -25,6 +25,7 @@ import org.elasticsearch.index.mapper.MapperRegistry; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.similarity.SimilarityService; +import org.elasticsearch.script.ScriptCompiler; import org.elasticsearch.script.ScriptService; import java.util.AbstractMap; @@ -50,15 +51,15 @@ public class IndexMetadataVerifier { private final NamedXContentRegistry xContentRegistry; private final MapperRegistry mapperRegistry; private final IndexScopedSettings indexScopedSettings; - private final ScriptService scriptService; + private final ScriptCompiler scriptService; public IndexMetadataVerifier(Settings settings, NamedXContentRegistry xContentRegistry, MapperRegistry mapperRegistry, - IndexScopedSettings indexScopedSettings, ScriptService scriptService) { + IndexScopedSettings indexScopedSettings, ScriptCompiler scriptCompiler) { this.settings = settings; this.xContentRegistry = xContentRegistry; this.mapperRegistry = mapperRegistry; this.indexScopedSettings = indexScopedSettings; - this.scriptService = scriptService; + this.scriptService = scriptCompiler; } /** diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentLeafReader.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentLeafReader.java index 2e4bb77a9764c..8832fbe08acec 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentLeafReader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentLeafReader.java @@ -124,6 +124,7 @@ public SortedNumericDocValues getSortedNumericDocValues(String field) throws IOE @Override public SortedSetDocValues getSortedSetDocValues(String field) throws IOException { + checkField(field); List values = document.getFields().stream() .filter(f -> Objects.equals(f.name(), field)) .filter(f -> f.fieldType().docValuesType() == DocValuesType.SORTED_SET) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index 3af8e955bb996..24009001e3892 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -14,6 +14,7 @@ import org.apache.lucene.document.FieldType; import org.apache.lucene.document.SortedSetDocValuesField; import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.xcontent.XContentParser; @@ -23,7 +24,11 @@ import org.elasticsearch.index.fielddata.plain.SortedSetOrdinalsIndexFieldData; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.index.similarity.SimilarityProvider; +import org.elasticsearch.script.Script; +import org.elasticsearch.script.ScriptCompiler; +import org.elasticsearch.script.StringFieldScript; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.lookup.FieldValues; import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; @@ -32,6 +37,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.function.Supplier; /** @@ -92,15 +98,29 @@ public static class Builder extends FieldMapper.Builder { private final Parameter> meta = Parameter.metaParam(); private final Parameter boost = Parameter.boostParam(); + private final Parameter