From 91d37c2c8a7e02badf2b6e182215cee36896657a Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Tue, 8 Dec 2020 14:47:51 +0100 Subject: [PATCH] Add xpack info and usage endpoints for runtime fields Relates to #59332 --- .../elasticsearch/xpack/core/XPackField.java | 31 +- .../RuntimeFieldsFeatureSetUsage.java | 274 ++++++++++++++++++ .../RuntimeFieldsFeatureSetUsageTests.java | 138 +++++++++ .../datastreams/DataStreamFeatureSet.java | 2 +- .../xpack/runtimefields/RuntimeFields.java | 8 + .../RuntimeFieldsFeatureSet.java | 51 ++++ .../200_runtime_fields_stats.yml | 205 +++++++++++++ 7 files changed, 701 insertions(+), 8 deletions(-) create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/runtimefields/RuntimeFieldsFeatureSetUsage.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/runtimefields/RuntimeFieldsFeatureSetUsageTests.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFieldsFeatureSet.java create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/200_runtime_fields_stats.yml diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java index 66a3fd0539d16..b5f70bd6448be 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java @@ -55,20 +55,37 @@ public final class XPackField { public static final String ANALYTICS = "analytics"; /** Name constant for the enrich plugin. */ public static final String ENRICH = "enrich"; - /** Name constant for the constant-keyword plugin. */ + /** + * Name constant for the constant-keyword plugin. + */ public static final String CONSTANT_KEYWORD = "constant_keyword"; - /** Name constant for the searchable snapshots feature. */ + /** + * Name constant for the searchable snapshots feature. + */ public static final String SEARCHABLE_SNAPSHOTS = "searchable_snapshots"; - /** Name constant for the data streams feature. */ + /** + * Name constant for the data streams feature. + */ public static final String DATA_STREAMS = "data_streams"; - /** Name constant for the data tiers feature. */ + /** + * Name constant for the data tiers feature. + */ public static final String DATA_TIERS = "data_tiers"; - /** Name constant for the aggregate_metric plugin. */ + /** + * Name constant for the aggregate_metric plugin. + */ public static final String AGGREGATE_METRIC = "aggregate_metric"; - /** Name constant for the operator privileges feature. */ + /** + * Name constant for the runtime fields plugin. + */ + public static final String RUNTIME_FIELDS = "runtime_fields"; + /** + * Name constant for the operator privileges feature. + */ public static final String OPERATOR_PRIVILEGES = "operator_privileges"; - private XPackField() {} + private XPackField() { + } public static String featureSettingPrefix(String featureName) { return XPackField.SETTINGS_NAME + "." + featureName; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/runtimefields/RuntimeFieldsFeatureSetUsage.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/runtimefields/RuntimeFieldsFeatureSetUsage.java new file mode 100644 index 0000000000000..8106ed461005e --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/runtimefields/RuntimeFieldsFeatureSetUsage.java @@ -0,0 +1,274 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.runtimefields; + +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.MappingMetadata; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.xpack.core.XPackFeatureSet; +import org.elasticsearch.xpack.core.XPackField; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class RuntimeFieldsFeatureSetUsage extends XPackFeatureSet.Usage { + + public static RuntimeFieldsFeatureSetUsage fromMetadata(Iterable metadata) { + Map fieldTypes = new HashMap<>(); + for (IndexMetadata indexMetadata : metadata) { + if (indexMetadata.isSystem()) { + // Don't include system indices in statistics about mappings, we care about the user's indices. + continue; + } + Set indexFieldTypes = new HashSet<>(); + MappingMetadata mappingMetadata = indexMetadata.mapping(); + if (mappingMetadata != null) { + Object runtimeObject = mappingMetadata.getSourceAsMap().get("runtime"); + if (runtimeObject instanceof Map == false) { + continue; + } + Map runtimeMappings = (Map) runtimeObject; + for (Object runtimeFieldMappingObject : runtimeMappings.values()) { + if (runtimeFieldMappingObject instanceof Map == false) { + continue; + } + Map runtimeFieldMapping = (Map) runtimeFieldMappingObject; + Object typeObject = runtimeFieldMapping.get("type"); + if (typeObject == null) { + continue; + } + String type = typeObject.toString(); + RuntimeFieldStats stats = fieldTypes.computeIfAbsent(type, RuntimeFieldStats::new); + stats.count++; + if (indexFieldTypes.add(type)) { + stats.indexCount++; + } + Object scriptObject = runtimeFieldMapping.get("script"); + if (scriptObject == null) { + stats.scriptLessCount++; + } else if (scriptObject instanceof Map) { + Map script = (Map) scriptObject; + Object sourceObject = script.get("source"); + if (sourceObject != null) { + String scriptSource = sourceObject.toString(); + int chars = scriptSource.length(); + long lines = scriptSource.split("\\n").length; + int docUsages = countOccurrences(scriptSource, "doc[\\[\\.]"); + int sourceUsages = countOccurrences(scriptSource, "params\\._source"); + stats.update(chars, lines, sourceUsages, docUsages); + } + Object langObject = script.get("lang"); + if (langObject != null) { + stats.scriptLangs.add(langObject.toString()); + } + } + } + } + } + List runtimeFieldStats = new ArrayList<>(fieldTypes.values()); + runtimeFieldStats.sort(Comparator.comparing(RuntimeFieldStats::type)); + return new RuntimeFieldsFeatureSetUsage(Collections.unmodifiableList(runtimeFieldStats)); + } + + private final List stats; + + RuntimeFieldsFeatureSetUsage(List stats) { + super(XPackField.RUNTIME_FIELDS, true, true); + this.stats = stats; + } + + public RuntimeFieldsFeatureSetUsage(StreamInput in) throws IOException { + super(in); + this.stats = in.readList(RuntimeFieldStats::new); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeList(stats); + } + + List getRuntimeFieldStats() { + return stats; + } + + @Override + protected void innerXContent(XContentBuilder builder, Params params) throws IOException { + super.innerXContent(builder, params); + builder.startArray("field_types"); + for (RuntimeFieldStats stats : stats) { + stats.toXContent(builder, params); + } + builder.endArray(); + } + + @Override + public Version getMinimalSupportedVersion() { + return Version.V_7_11_0; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + RuntimeFieldsFeatureSetUsage that = (RuntimeFieldsFeatureSetUsage) o; + return stats.equals(that.stats); + } + + @Override + public int hashCode() { + return Objects.hash(stats); + } + + private static int countOccurrences(String script, String keyword) { + int occurrences = 0; + Pattern pattern = Pattern.compile(keyword); + Matcher matcher = pattern.matcher(script); + while (matcher.find()) { + occurrences++; + } + return occurrences; + } + + static final class RuntimeFieldStats implements Writeable, ToXContentObject { + private final String type; + private int count = 0; + private int indexCount = 0; + private final Set scriptLangs; + private long scriptLessCount = 0; + private long maxLines = 0; + private long totalLines = 0; + private long maxChars = 0; + private long totalChars = 0; + private long maxSourceUsages = 0; + private long totalSourceUsages = 0; + private long maxDocUsages = 0; + private long totalDocUsages = 0; + + RuntimeFieldStats(String type) { + this.type = Objects.requireNonNull(type); + this.scriptLangs = new HashSet<>(); + } + + RuntimeFieldStats(StreamInput in) throws IOException { + this.type = in.readString(); + this.count = in.readInt(); + this.indexCount = in.readInt(); + this.scriptLangs = in.readSet(StreamInput::readString); + this.scriptLessCount = in.readLong(); + this.maxLines = in.readLong(); + this.totalLines = in.readLong(); + this.maxChars = in.readLong(); + this.totalChars = in.readLong(); + this.maxSourceUsages = in.readLong(); + this.totalSourceUsages = in.readLong(); + this.maxDocUsages = in.readLong(); + this.totalDocUsages = in.readLong(); + } + + String type() { + return type; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(type); + out.writeInt(count); + out.writeInt(indexCount); + out.writeCollection(scriptLangs, StreamOutput::writeString); + out.writeLong(scriptLessCount); + out.writeLong(maxLines); + out.writeLong(totalLines); + out.writeLong(maxChars); + out.writeLong(totalChars); + out.writeLong(maxSourceUsages); + out.writeLong(totalSourceUsages); + out.writeLong(maxDocUsages); + out.writeLong(totalDocUsages); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field("name", type); + builder.field("count", count); + builder.field("index_count", indexCount); + builder.field("scriptless_count", scriptLessCount); + builder.array("lang", scriptLangs.toArray(new String[0])); + builder.field("lines_max", maxLines); + builder.field("lines_total", totalLines); + builder.field("chars_max", maxChars); + builder.field("chars_total", totalChars); + builder.field("source_max", maxSourceUsages); + builder.field("source_total", totalSourceUsages); + builder.field("doc_max", maxDocUsages); + builder.field("doc_total", totalDocUsages); + builder.endObject(); + return builder; + } + + void update(int chars, long lines, int sourceUsages, int docUsages) { + this.maxChars = Math.max(this.maxChars, chars); + this.totalChars += chars; + this.maxLines = Math.max(this.maxLines, lines); + this.totalLines += lines; + this.totalSourceUsages += sourceUsages; + this.maxSourceUsages = Math.max(this.maxSourceUsages, sourceUsages); + this.totalDocUsages += docUsages; + this.maxDocUsages = Math.max(this.maxDocUsages, docUsages); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + RuntimeFieldStats that = (RuntimeFieldStats) o; + return count == that.count && + indexCount == that.indexCount && + scriptLessCount == that.scriptLessCount && + maxLines == that.maxLines && + totalLines == that.totalLines && + maxChars == that.maxChars && + totalChars == that.totalChars && + maxSourceUsages == that.maxSourceUsages && + totalSourceUsages == that.totalSourceUsages && + maxDocUsages == that.maxDocUsages && + totalDocUsages == that.totalDocUsages && + type.equals(that.type) && + scriptLangs.equals(that.scriptLangs); + } + + @Override + public int hashCode() { + return Objects.hash(type, count, indexCount, scriptLangs, scriptLessCount, maxLines, totalLines, maxChars, totalChars, + maxSourceUsages, totalSourceUsages, maxDocUsages, totalDocUsages); + } + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/runtimefields/RuntimeFieldsFeatureSetUsageTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/runtimefields/RuntimeFieldsFeatureSetUsageTests.java new file mode 100644 index 0000000000000..e932cc0c5b6a6 --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/runtimefields/RuntimeFieldsFeatureSetUsageTests.java @@ -0,0 +1,138 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.runtimefields; + +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.script.Script; +import org.elasticsearch.test.AbstractWireSerializingTestCase; +import org.elasticsearch.xpack.core.runtimefields.RuntimeFieldsFeatureSetUsage.RuntimeFieldStats; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +public class RuntimeFieldsFeatureSetUsageTests extends AbstractWireSerializingTestCase { + + public void testToXContent() throws IOException { + Settings settings = Settings.builder() + .put("index.number_of_replicas", 0) + .put("index.number_of_shards", 1) + .put("index.version.created", Version.CURRENT) + .build(); + Script script1 = new Script("doc['field'] + doc.field + params._source.field"); + Script script2 = new Script("doc['field']"); + Script script3 = new Script("params._source.field + params._source.field \n + params._source.field"); + Script script4 = new Script("params._source.field"); + IndexMetadata meta = IndexMetadata.builder("index").settings(settings) + .putMapping("_doc", "{" + + " \"runtime\" : {" + + " \"keyword1\": {" + + " \"type\": \"keyword\"," + + " \"script\": " + Strings.toString(script1) + + " }," + + " \"keyword2\": {" + + " \"type\": \"keyword\"" + + " }," + + " \"keyword3\": {" + + " \"type\": \"keyword\"," + + " \"script\": " + Strings.toString(script2) + + " }," + + " \"long\": {" + + " \"type\": \"long\"," + + " \"script\": " + Strings.toString(script3) + + " }," + + " \"long2\": {" + + " \"type\": \"long\"," + + " \"script\": " + Strings.toString(script4) + + " }" + + " }" + + "}") + .build(); + + RuntimeFieldsFeatureSetUsage featureSetUsage = RuntimeFieldsFeatureSetUsage.fromMetadata( + org.elasticsearch.common.collect.List.of(meta, meta)); + assertEquals("{\n" + + " \"available\" : true,\n" + + " \"enabled\" : true,\n" + + " \"field_types\" : [\n" + + " {\n" + + " \"name\" : \"keyword\",\n" + + " \"count\" : 6,\n" + + " \"index_count\" : 2,\n" + + " \"scriptless_count\" : 2,\n" + + " \"lang\" : [\n" + + " \"painless\"\n" + + " ],\n" + + " \"lines_max\" : 1,\n" + + " \"lines_total\" : 4,\n" + + " \"chars_max\" : 47,\n" + + " \"chars_total\" : 118,\n" + + " \"source_max\" : 1,\n" + + " \"source_total\" : 2,\n" + + " \"doc_max\" : 2,\n" + + " \"doc_total\" : 6\n" + + " },\n" + + " {\n" + + " \"name\" : \"long\",\n" + + " \"count\" : 4,\n" + + " \"index_count\" : 2,\n" + + " \"scriptless_count\" : 0,\n" + + " \"lang\" : [\n" + + " \"painless\"\n" + + " ],\n" + + " \"lines_max\" : 2,\n" + + " \"lines_total\" : 6,\n" + + " \"chars_max\" : 68,\n" + + " \"chars_total\" : 176,\n" + + " \"source_max\" : 3,\n" + + " \"source_total\" : 8,\n" + + " \"doc_max\" : 0,\n" + + " \"doc_total\" : 0\n" + + " }\n" + + " ]\n" + + "}", Strings.toString(featureSetUsage, true, true)); + } + + @Override + protected RuntimeFieldsFeatureSetUsage createTestInstance() { + int numItems = randomIntBetween(0, 10); + List stats = new ArrayList<>(numItems); + for (int i = 0; i < numItems; i++) { + stats.add(randomRuntimeFieldStats("type" + i)); + } + return new RuntimeFieldsFeatureSetUsage(stats); + } + + private static RuntimeFieldStats randomRuntimeFieldStats(String type) { + RuntimeFieldStats stats = new RuntimeFieldStats(type); + if (randomBoolean()) { + stats.update(randomIntBetween(1, 100), randomLongBetween(100, 1000), randomIntBetween(1, 10), randomIntBetween(1, 10)); + } + return stats; + } + + @Override + protected RuntimeFieldsFeatureSetUsage mutateInstance(RuntimeFieldsFeatureSetUsage instance) throws IOException { + List runtimeFieldStats = instance.getRuntimeFieldStats(); + if (runtimeFieldStats.size() == 0) { + return new RuntimeFieldsFeatureSetUsage(Collections.singletonList(randomRuntimeFieldStats("type"))); + } + List mutated = new ArrayList<>(runtimeFieldStats); + mutated.remove(randomIntBetween(0, mutated.size() - 1)); + return new RuntimeFieldsFeatureSetUsage(mutated); + } + + @Override + protected Writeable.Reader instanceReader() { + return RuntimeFieldsFeatureSetUsage::new; + } +} diff --git a/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/DataStreamFeatureSet.java b/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/DataStreamFeatureSet.java index fcbcc934aff7e..8230c77cd36f0 100644 --- a/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/DataStreamFeatureSet.java +++ b/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/DataStreamFeatureSet.java @@ -19,7 +19,7 @@ public class DataStreamFeatureSet implements XPackFeatureSet { - private ClusterService clusterService; + private final ClusterService clusterService; @Inject public DataStreamFeatureSet(ClusterService clusterService) { diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java index 679c3724f92b9..09957682b8708 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java @@ -6,6 +6,7 @@ package org.elasticsearch.xpack.runtimefields; +import org.elasticsearch.common.inject.Module; import org.elasticsearch.index.mapper.BooleanFieldMapper; import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.GeoPointFieldMapper; @@ -17,6 +18,7 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.ScriptPlugin; import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.xpack.core.XPackPlugin; import org.elasticsearch.xpack.runtimefields.mapper.BooleanFieldScript; import org.elasticsearch.xpack.runtimefields.mapper.BooleanScriptFieldType; import org.elasticsearch.xpack.runtimefields.mapper.DateFieldScript; @@ -32,6 +34,8 @@ import org.elasticsearch.xpack.runtimefields.mapper.LongScriptFieldType; import org.elasticsearch.xpack.runtimefields.mapper.StringFieldScript; +import java.util.Collection; +import java.util.Collections; import java.util.List; import java.util.Map; @@ -62,4 +66,8 @@ public List> getContexts() { StringFieldScript.CONTEXT ); } + + public Collection createGuiceModules() { + return Collections.singletonList(b -> XPackPlugin.bindFeatureSet(b, RuntimeFieldsFeatureSet.class)); + } } diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFieldsFeatureSet.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFieldsFeatureSet.java new file mode 100644 index 0000000000000..a21b3ca92ab8b --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFieldsFeatureSet.java @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.xpack.core.XPackFeatureSet; +import org.elasticsearch.xpack.core.XPackField; +import org.elasticsearch.xpack.core.runtimefields.RuntimeFieldsFeatureSetUsage; + +import java.util.Map; + +public class RuntimeFieldsFeatureSet implements XPackFeatureSet { + + private final ClusterService clusterService; + + @Inject + public RuntimeFieldsFeatureSet(ClusterService clusterService) { + this.clusterService = clusterService; + } + + @Override + public String name() { + return XPackField.RUNTIME_FIELDS; + } + + @Override + public boolean available() { + return true; + } + + @Override + public boolean enabled() { + return true; + } + + @Override + public Map nativeCodeInfo() { + return null; + } + + @Override + public void usage(ActionListener listener) { + listener.onResponse(RuntimeFieldsFeatureSetUsage.fromMetadata(clusterService.state().metadata())); + } +} diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/200_runtime_fields_stats.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/200_runtime_fields_stats.yml new file mode 100644 index 0000000000000..2a2369b5b52ba --- /dev/null +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/200_runtime_fields_stats.yml @@ -0,0 +1,205 @@ +--- +"Usage stats without runtime fields": + - do: + indices.create: + index: sensor + + - do: {xpack.info: {}} + - match: { features.runtime_fields.available: true } + - match: { features.runtime_fields.enabled: true } + + - do: {xpack.usage: {}} + - match: { runtime_fields.available: true } + - match: { runtime_fields.enabled: true } + - length: { runtime_fields.field_types: 0 } + +--- +"Usage stats with runtime fields": + - do: + indices.create: + index: sensor + body: + mappings: + runtime: + message_from_source: + type: keyword + day_of_week: + type: keyword + script: | + emit(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT)); + # Test fetching from _source + day_of_week_from_source: + type: keyword + script: | + Instant instant = Instant.ofEpochMilli(params._source.timestamp); + ZonedDateTime dt = ZonedDateTime.ofInstant(instant, ZoneId.of("UTC")); + emit(dt.dayOfWeek.getDisplayName(TextStyle.FULL, Locale.ROOT)); + millis_ago: + type: date + script: + source: | + for (def dt : doc['timestamp']) { + emit(System.currentTimeMillis() - dt.toInstant().toEpochMilli()); + } + tomorrow: + type: date + script: + source: | + for (def dt : doc['timestamp']) { + emit(dt.plus(params.days, ChronoUnit.DAYS).toEpochMilli()); + } + params: + days: 1 + voltage_times_ten: + type: long + script: + source: | + for (double v : doc['voltage']) { + emit((long)(v * params.multiplier)); + } + params: + multiplier: 10 + voltage_percent_from_source: + type: double + script: + source: | + emit(params._source.voltage / params.max); + params: + max: 5.8 + over_v: + type: boolean + script: + source: | + for (def v : doc['voltage']) { + emit(v >= params.min_v); + } + params: + min_v: 5.0 + ip: + type: ip + script: + source: | + Matcher m = /([^ ]+) .+/.matcher(doc["message"].value); + if (m.matches()) { + emit(m.group(1)); + } + location_from_source: + type: geo_point + script: + source: | + emit(params._source.location.lat, params._source.location.lon); + properties: + timestamp: + type: date + message: + type: keyword + voltage: + type: double + location: + type: geo_point + + - do: {xpack.info: {}} + - match: { features.runtime_fields.available: true } + - match: { features.runtime_fields.enabled: true } + + - do: {xpack.usage: {}} + - match: { runtime_fields.available: true } + - match: { runtime_fields.enabled: true } + - length: { runtime_fields.field_types: 7 } + - match: { runtime_fields.field_types.0.name: boolean } + - match: { runtime_fields.field_types.0.lang: [painless] } + - match: { runtime_fields.field_types.0.count: 1 } + - match: { runtime_fields.field_types.0.index_count: 1 } + - match: { runtime_fields.field_types.0.scriptless_count: 0 } + - match: { runtime_fields.field_types.0.source_max: 0 } + - match: { runtime_fields.field_types.0.source_total: 0 } + - match: { runtime_fields.field_types.0.lines_max: 3 } + - match: { runtime_fields.field_types.0.lines_total: 3 } + - is_true: runtime_fields.field_types.0.chars_max + - is_true: runtime_fields.field_types.0.chars_total + - match: { runtime_fields.field_types.0.doc_max: 1 } + - match: { runtime_fields.field_types.0.doc_total: 1 } + + - match: { runtime_fields.field_types.1.name: date } + - match: { runtime_fields.field_types.1.lang: [painless] } + - match: { runtime_fields.field_types.1.count: 2 } + - match: { runtime_fields.field_types.1.index_count: 1 } + - match: { runtime_fields.field_types.1.scriptless_count: 0 } + - match: { runtime_fields.field_types.1.source_max: 0 } + - match: { runtime_fields.field_types.1.source_total: 0 } + - match: { runtime_fields.field_types.1.lines_max: 3 } + - match: { runtime_fields.field_types.1.lines_total: 6 } + - is_true: runtime_fields.field_types.1.chars_max + - is_true: runtime_fields.field_types.1.chars_total + - match: { runtime_fields.field_types.1.doc_max: 1 } + - match: { runtime_fields.field_types.1.doc_total: 2 } + + - match: { runtime_fields.field_types.2.name: double } + - match: { runtime_fields.field_types.2.lang: [painless] } + - match: { runtime_fields.field_types.2.count: 1 } + - match: { runtime_fields.field_types.2.index_count: 1 } + - match: { runtime_fields.field_types.2.scriptless_count: 0 } + - match: { runtime_fields.field_types.2.source_max: 1 } + - match: { runtime_fields.field_types.2.source_total: 1 } + - match: { runtime_fields.field_types.2.lines_max: 1 } + - match: { runtime_fields.field_types.2.lines_total: 1 } + - is_true: runtime_fields.field_types.2.chars_max + - is_true: runtime_fields.field_types.2.chars_total + - match: { runtime_fields.field_types.2.doc_max: 0 } + - match: { runtime_fields.field_types.2.doc_total: 0 } + + - match: { runtime_fields.field_types.3.name: geo_point } + - match: { runtime_fields.field_types.3.lang: [painless] } + - match: { runtime_fields.field_types.3.count: 1 } + - match: { runtime_fields.field_types.3.index_count: 1 } + - match: { runtime_fields.field_types.3.scriptless_count: 0 } + - match: { runtime_fields.field_types.3.source_max: 2 } + - match: { runtime_fields.field_types.3.source_total: 2 } + - match: { runtime_fields.field_types.3.lines_max: 1 } + - match: { runtime_fields.field_types.3.lines_total: 1 } + - is_true: runtime_fields.field_types.3.chars_max + - is_true: runtime_fields.field_types.3.chars_total + - match: { runtime_fields.field_types.3.doc_max: 0 } + - match: { runtime_fields.field_types.3.doc_total: 0 } + + - match: { runtime_fields.field_types.4.name: ip } + - match: { runtime_fields.field_types.4.lang: [painless] } + - match: { runtime_fields.field_types.4.count: 1 } + - match: { runtime_fields.field_types.4.index_count: 1 } + - match: { runtime_fields.field_types.4.scriptless_count: 0 } + - match: { runtime_fields.field_types.4.source_max: 0 } + - match: { runtime_fields.field_types.4.source_total: 0 } + - match: { runtime_fields.field_types.4.lines_max: 4 } + - match: { runtime_fields.field_types.4.lines_total: 4 } + - is_true: runtime_fields.field_types.4.chars_max + - is_true: runtime_fields.field_types.4.chars_total + - match: { runtime_fields.field_types.4.doc_max: 1 } + - match: { runtime_fields.field_types.4.doc_total: 1 } + + - match: { runtime_fields.field_types.5.name: keyword } + - match: { runtime_fields.field_types.5.lang: [painless] } + - match: { runtime_fields.field_types.5.count: 3 } + - match: { runtime_fields.field_types.5.index_count: 1 } + - match: { runtime_fields.field_types.5.scriptless_count: 1 } + - match: { runtime_fields.field_types.5.source_max: 1 } + - match: { runtime_fields.field_types.5.source_total: 1 } + - match: { runtime_fields.field_types.5.lines_max: 3 } + - match: { runtime_fields.field_types.5.lines_total: 4 } + - is_true: runtime_fields.field_types.5.chars_max + - is_true: runtime_fields.field_types.5.chars_total + - match: { runtime_fields.field_types.5.doc_max: 1 } + - match: { runtime_fields.field_types.5.doc_total: 1 } + + - match: { runtime_fields.field_types.6.name: long } + - match: { runtime_fields.field_types.6.lang: [painless] } + - match: { runtime_fields.field_types.6.count: 1 } + - match: { runtime_fields.field_types.6.index_count: 1 } + - match: { runtime_fields.field_types.6.scriptless_count: 0 } + - match: { runtime_fields.field_types.6.source_max: 0 } + - match: { runtime_fields.field_types.6.source_total: 0 } + - match: { runtime_fields.field_types.6.lines_max: 3 } + - match: { runtime_fields.field_types.6.lines_total: 3 } + - is_true: runtime_fields.field_types.6.chars_max + - is_true: runtime_fields.field_types.6.chars_total + - match: { runtime_fields.field_types.6.doc_max: 1 } + - match: { runtime_fields.field_types.6.doc_total: 1 }