From 58bc365ec948bb9af569de3f73001d6c9c10b9ce Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Wed, 6 Oct 2021 10:29:54 -1000 Subject: [PATCH 1/8] Add time series information to field caps Exposes information about dimensions and metrics via field caps. This information will be needed for PromQL support. Relates to #74660 --- .../org/elasticsearch/client/SearchIT.java | 6 +- docs/reference/search/field-caps.asciidoc | 27 ++- .../test/field_caps/40_time_series.yml | 221 ++++++++++++++++++ .../search/fieldcaps/FieldCapabilitiesIT.java | 27 ++- .../action/fieldcaps/FieldCapabilities.java | 175 ++++++++++++-- .../fieldcaps/IndexFieldCapabilities.java | 31 ++- .../TransportFieldCapabilitiesAction.java | 26 ++- .../index/mapper/MappedFieldType.java | 7 + .../FieldCapabilitiesResponseTests.java | 3 +- .../fieldcaps/FieldCapabilitiesTests.java | 141 +++++++++-- .../MergedFieldCapabilitiesResponseTests.java | 30 ++- .../analyses/ClassificationTests.java | 2 +- .../xpack/eql/analysis/CancellationTests.java | 6 +- .../ml/dataframe/DestinationIndexTests.java | 2 +- .../ExtractedFieldsDetectorTests.java | 2 +- .../xpack/sql/analysis/CancellationTests.java | 6 +- .../analysis/index/IndexResolverTests.java | 16 +- .../common/DocumentConversionUtilsTests.java | 3 +- .../AggregationSchemaAndResultTests.java | 2 +- .../transforms/pivot/SchemaUtilTests.java | 3 + 20 files changed, 645 insertions(+), 91 deletions(-) create mode 100644 rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/field_caps/40_time_series.yml diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java index 040004e822758..8ee95aeb53d51 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java @@ -1253,11 +1253,11 @@ public void testFieldCaps() throws IOException { assertEquals(2, ratingResponse.size()); FieldCapabilities expectedKeywordCapabilities = new FieldCapabilities( - "rating", "keyword", false, true, true, new String[]{"index2"}, null, null, Collections.emptyMap()); + "rating", "keyword", false, true, true, false, null, null, null, new String[]{"index2"}, null, Collections.emptyMap()); assertEquals(expectedKeywordCapabilities, ratingResponse.get("keyword")); FieldCapabilities expectedLongCapabilities = new FieldCapabilities( - "rating", "long", false, true, true, new String[]{"index1"}, null, null, Collections.emptyMap()); + "rating", "long", false, true, true, false, null, null, null, new String[]{"index1"}, null, Collections.emptyMap()); assertEquals(expectedLongCapabilities, ratingResponse.get("long")); // Check the capabilities for the 'field' field. @@ -1266,7 +1266,7 @@ public void testFieldCaps() throws IOException { assertEquals(1, fieldResponse.size()); FieldCapabilities expectedTextCapabilities = new FieldCapabilities( - "field", "text", false, true, false, null, null, null, Collections.emptyMap()); + "field", "text", false, true, false, false, null, null, null, null, null, Collections.emptyMap()); assertEquals(expectedTextCapabilities, fieldResponse.get("text")); } diff --git a/docs/reference/search/field-caps.asciidoc b/docs/reference/search/field-caps.asciidoc index 80ee6bfc24788..f5e95c04c8731 100644 --- a/docs/reference/search/field-caps.asciidoc +++ b/docs/reference/search/field-caps.asciidoc @@ -111,6 +111,12 @@ field types are all described as the `keyword` type family. `aggregatable`:: Whether this field can be aggregated on all indices. +`time_series_dimension`:: + Whether this field is used as a time series dimension. + +`time_series_metrics`:: + Contains metric type if this fields is used as a time series metrics, null if the field is not used as metric. + `indices`:: The list of indices where this field has the same type family, or null if all indices have the same type family for the field. @@ -123,6 +129,10 @@ field types are all described as the `keyword` type family. The list of indices where this field is not aggregatable, or null if all indices have the same definition for the field. +`non_dimension_indices`:: + The list of indices where this field is not marked as a dimension field, or null if all + indices have the same definition for the field. + `meta`:: Merged metadata across all indices as a map of string keys to arrays of values. A value length of 1 indicates that all indices had the same value for this key, @@ -163,6 +173,7 @@ The API returns the following response: "metadata_field": false, "searchable": true, "aggregatable": false, + "time_series_dimension": false, "indices": [ "index1", "index2" ], "non_aggregatable_indices": [ "index1" ] <2> }, @@ -170,6 +181,7 @@ The API returns the following response: "metadata_field": false, "searchable": false, "aggregatable": true, + "time_series_dimension": false, "indices": [ "index3", "index4" ], "non_searchable_indices": [ "index4" ] <3> } @@ -178,8 +190,8 @@ The API returns the following response: "text": { "metadata_field": false, "searchable": true, - "aggregatable": false - + "aggregatable": false, + "time_series_dimension": false } } } @@ -215,6 +227,7 @@ in some indices but not all: "metadata_field": false, "searchable": true, "aggregatable": false, + "time_series_dimension": false, "indices": [ "index1", "index2" ], "non_aggregatable_indices": [ "index1" ] }, @@ -222,6 +235,7 @@ in some indices but not all: "metadata_field": false, "searchable": false, "aggregatable": true, + "time_series_dimension": false, "indices": [ "index3", "index4" ], "non_searchable_indices": [ "index4" ] }, @@ -229,7 +243,8 @@ in some indices but not all: "metadata_field": false, "indices": [ "index5" ], "searchable": false, - "aggregatable": false + "aggregatable": false, + "time_series_dimension": false } }, "title": { @@ -237,13 +252,15 @@ in some indices but not all: "metadata_field": false, "indices": [ "index1", "index2", "index3", "index4" ], "searchable": true, - "aggregatable": false + "aggregatable": false, + "time_series_dimension": false }, "unmapped": { <2> "metadata_field": false, "indices": [ "index5" ], "searchable": false, - "aggregatable": false + "aggregatable": false, + "time_series_dimension": false } } } diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/field_caps/40_time_series.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/field_caps/40_time_series.yml new file mode 100644 index 0000000000000..7795c85b6e52d --- /dev/null +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/field_caps/40_time_series.yml @@ -0,0 +1,221 @@ +--- +setup: + - skip: + version: " - 7.99.99" + reason: introduced in 8.0.0 + + - do: + indices.create: + index: tsdb_index1 + body: + settings: + index: + number_of_replicas: 0 + number_of_shards: 2 + mappings: + properties: + "@timestamp": + type: date + metricset: + type: keyword + time_series_dimension: true + non_tsdb_field: + type: keyword + k8s: + properties: + pod: + properties: + availability_zone: + type: short + time_series_dimension: true + uid: + type: keyword + time_series_dimension: true + name: + type: keyword + ip: + type: ip + time_series_dimension: true + network: + properties: + tx: + type: long + time_series_metric: counter + rx: + type: integer + time_series_metric: gauge + packets_dropped: + type: long + time_series_metric: gauge + latency: + type: double + time_series_metric: gauge + + - do: + indices.create: + index: tsdb_index2 + body: + settings: + index: + number_of_replicas: 0 + number_of_shards: 2 + mappings: + properties: + "@timestamp": + type: date + metricset: + type: keyword + non_tsdb_field: + type: keyword + k8s: + properties: + pod: + properties: + availability_zone: + type: short + time_series_dimension: true + uid: + type: keyword + time_series_dimension: true + name: + type: keyword + ip: + type: ip + time_series_dimension: true + network: + properties: + tx: + type: long + time_series_metric: gauge + rx: + type: integer + packets_dropped: + type: long + time_series_metric: gauge + latency: + type: double + time_series_metric: gauge + +--- +"Get simple time series field caps": + + - skip: + version: " - 7.99.99" + reason: introduced in 8.0.0 + + - do: + field_caps: + index: 'tsdb_index1' + fields: [ "metricset", "non_tsdb_field", "k8s.pod.*" ] + + - match: {fields.metricset.keyword.searchable: true} + - match: {fields.metricset.keyword.aggregatable: true} + - match: {fields.metricset.keyword.time_series_dimension: true} + - is_false: fields.metricset.keyword.time_series_metric + - is_false: fields.metricset.keyword.indices + - is_false: fields.metricset.keyword.non_searchable_indices + - is_false: fields.metricset.keyword.non_aggregatable_indices + - is_false: fields.metricset.keyword.non_dimension_indices + + - match: {fields.non_tsdb_field.keyword.searchable: true} + - match: {fields.non_tsdb_field.keyword.aggregatable: true} + - match: {fields.non_tsdb_field.keyword.time_series_dimension: false} + - is_false: fields.non_tsdb_field.keyword.time_series_metric + - is_false: fields.non_tsdb_field.keyword.indices + - is_false: fields.non_tsdb_field.keyword.non_searchable_indices + - is_false: fields.non_tsdb_field.keyword.non_aggregatable_indices + - is_false: fields.non_tsdb_field.keyword.non_dimension_indices + + - match: {fields.k8s\.pod\.availability_zone.short.time_series_dimension: true} + - is_false: fields.k8s\.pod\.availability_zone.short.time_series_metric + - is_false: fields.k8s\.pod\.availability_zone.short.non_dimension_indices + + - match: {fields.k8s\.pod\.uid.keyword.time_series_dimension: true} + - is_false: fields.k8s\.pod\.uid.keyword.time_series_metric + - is_false: fields.k8s\.pod\.uid.keyword.non_dimension_indices + + - match: {fields.k8s\.pod\.name.keyword.time_series_dimension: false} + - is_false: fields.k8s\.pod\.name.keyword.time_series_metric + - is_false: fields.k8s\.pod\.name.keyword.non_dimension_indices + + - match: {fields.k8s\.pod\.ip.ip.time_series_dimension: true} + - is_false: fields.k8s\.pod\.ip.ip.time_series_metric + - is_false: fields.k8s\.pod\.ip.ip.non_dimension_indices + + - match: {fields.k8s\.pod\.network\.tx.long.time_series_dimension: false} + - match: {fields.k8s\.pod\.network\.tx.long.time_series_metric: counter} + - is_false: fields.k8s\.pod\.network\.tx.long.non_dimension_indices + + - match: {fields.k8s\.pod\.network\.rx.integer.time_series_dimension: false} + - match: {fields.k8s\.pod\.network\.rx.integer.time_series_metric: gauge} + - is_false: fields.k8s\.pod\.network\.rx.integer.non_dimension_indices + + - match: {fields.k8s\.pod\.network\.packets_dropped.long.time_series_dimension: false} + - match: {fields.k8s\.pod\.network\.packets_dropped.long.time_series_metric: gauge} + - is_false: fields.k8s\.pod\.network\.packets_dropped.long.non_dimension_indices + + - match: {fields.k8s\.pod\.network\.latency.double.time_series_dimension: false} + - match: {fields.k8s\.pod\.network\.latency.double.time_series_metric: gauge} + - is_false: fields.k8s\.pod\.network\.latency.double.non_dimension_indices + +--- +"Get time series field caps with conflicts": + + - skip: + version: " - 7.99.99" + reason: introduced in 8.0.0 + + - do: + field_caps: + index: tsdb_index1,tsdb_index2 + fields: [ "metricset", "non_tsdb_field", "k8s.pod.*" ] + + - match: {fields.metricset.keyword.searchable: true} + - match: {fields.metricset.keyword.aggregatable: true} + - match: {fields.metricset.keyword.time_series_dimension: false} + - is_false: fields.metricset.keyword.time_series_metric + - is_false: fields.metricset.keyword.indices + - is_false: fields.metricset.keyword.non_searchable_indices + - is_false: fields.metricset.keyword.non_aggregatable_indices + - match: {fields.metricset.keyword.non_dimension_indices: ["tsdb_index2"]} + + - match: {fields.non_tsdb_field.keyword.searchable: true} + - match: {fields.non_tsdb_field.keyword.aggregatable: true} + - match: {fields.non_tsdb_field.keyword.time_series_dimension: false} + - is_false: fields.non_tsdb_field.keyword.time_series_metric + - is_false: fields.non_tsdb_field.keyword.indices + - is_false: fields.non_tsdb_field.keyword.non_searchable_indices + - is_false: fields.non_tsdb_field.keyword.non_aggregatable_indices + - is_false: fields.non_tsdb_field.keyword.non_dimension_indices + + - match: {fields.k8s\.pod\.availability_zone.short.time_series_dimension: true} + - is_false: fields.k8s\.pod\.availability_zone.short.time_series_metric + - is_false: fields.k8s\.pod\.availability_zone.short.non_dimension_indices + + - match: {fields.k8s\.pod\.uid.keyword.time_series_dimension: true} + - is_false: fields.k8s\.pod\.uid.keyword.time_series_metric + - is_false: fields.k8s\.pod\.uid.keyword.non_dimension_indices + + - match: {fields.k8s\.pod\.name.keyword.time_series_dimension: false} + - is_false: fields.k8s\.pod\.name.keyword.time_series_metric + - is_false: fields.k8s\.pod\.name.keyword.non_dimension_indices + + - match: {fields.k8s\.pod\.ip.ip.time_series_dimension: true} + - is_false: fields.k8s\.pod\.ip.ip.time_series_metric + - is_false: fields.k8s\.pod\.ip.ip.non_dimension_indices + + - match: {fields.k8s\.pod\.network\.tx.long.time_series_dimension: false} + - is_false: fields.k8s\.pod\.network\.tx.long.time_series_metric + - is_false: fields.k8s\.pod\.network\.tx.long.non_dimension_indices + + - match: {fields.k8s\.pod\.network\.rx.integer.time_series_dimension: false} + - is_false: fields.k8s\.pod\.network\.rx.integer.time_series_metric + - is_false: fields.k8s\.pod\.network\.rx.integer.non_dimension_indices + + - match: {fields.k8s\.pod\.network\.packets_dropped.long.time_series_dimension: false} + - match: {fields.k8s\.pod\.network\.packets_dropped.long.time_series_metric: gauge} + - is_false: fields.k8s\.pod\.network\.packets_dropped.long.non_dimension_indices + + - match: {fields.k8s\.pod\.network\.latency.double.time_series_dimension: false} + - match: {fields.k8s\.pod\.network\.latency.double.time_series_metric: gauge} + - is_false: fields.k8s\.pod\.network\.latency.double.non_dimension_indices diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java index b477781930238..4b1da2d9138ce 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java @@ -124,14 +124,14 @@ public void testFieldAlias() { assertTrue(distance.containsKey("double")); assertEquals( - new FieldCapabilities("distance", "double", false, true, true, new String[] {"old_index"}, null, null, - Collections.emptyMap()), + new FieldCapabilities("distance", "double", false, true, true, false, null, new String[] {"old_index"}, null, null, null, + Collections.emptyMap()), distance.get("double")); assertTrue(distance.containsKey("text")); assertEquals( - new FieldCapabilities("distance", "text", false, true, false, new String[] {"new_index"}, null, null, - Collections.emptyMap()), + new FieldCapabilities("distance", "text", false, true, false, false, null, new String[] {"new_index"}, null, null, null, + Collections.emptyMap()), distance.get("text")); // Check the capabilities for the 'route_length_miles' alias. @@ -140,7 +140,8 @@ public void testFieldAlias() { assertTrue(routeLength.containsKey("double")); assertEquals( - new FieldCapabilities("route_length_miles", "double", false, true, true, null, null, null, Collections.emptyMap()), + new FieldCapabilities("route_length_miles", "double", false, true, true, false, null, null, null, null, null, + Collections.emptyMap()), routeLength.get("double")); } @@ -181,14 +182,14 @@ public void testWithUnmapped() { assertTrue(oldField.containsKey("long")); assertEquals( - new FieldCapabilities("old_field", "long", false, true, true, new String[] {"old_index"}, null, null, - Collections.emptyMap()), + new FieldCapabilities("old_field", "long", false, true, true, false, null, new String[] {"old_index"}, null, null, null, + Collections.emptyMap()), oldField.get("long")); assertTrue(oldField.containsKey("unmapped")); assertEquals( - new FieldCapabilities("old_field", "unmapped", false, false, false, new String[] {"new_index"}, null, null, - Collections.emptyMap()), + new FieldCapabilities("old_field", "unmapped", false, false, false, false, null, new String[] {"new_index"}, null, null, null, + Collections.emptyMap()), oldField.get("unmapped")); Map newField = response.getField("new_field"); @@ -196,7 +197,8 @@ public void testWithUnmapped() { assertTrue(newField.containsKey("long")); assertEquals( - new FieldCapabilities("new_field", "long", false, true, true, null, null, null, Collections.emptyMap()), + new FieldCapabilities("new_field", "long", false, true, true, false, null, null, null, null, null, + Collections.emptyMap()), newField.get("long")); } @@ -260,7 +262,7 @@ public void testMetadataFields() { assertTrue(idField.containsKey("_id")); assertEquals( - new FieldCapabilities("_id", "_id", true, true, false, null, null, null, Collections.emptyMap()), + new FieldCapabilities("_id", "_id", true, true, false, false, null, null, null, null, null, Collections.emptyMap()), idField.get("_id")); Map testField = response.getField("_test"); @@ -268,7 +270,8 @@ public void testMetadataFields() { assertTrue(testField.containsKey("keyword")); assertEquals( - new FieldCapabilities("_test", "keyword", true, true, true, null, null, null, Collections.emptyMap()), + new FieldCapabilities("_test", "keyword", true, true, true, false, null, null, null, null, null, + Collections.emptyMap()), testField.get("keyword")); } } diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java index 14e9e70428651..b5f0d3748ecd5 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java @@ -8,6 +8,7 @@ package org.elasticsearch.action.fieldcaps; +import org.elasticsearch.Version; import org.elasticsearch.common.xcontent.ParseField; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; @@ -17,6 +18,7 @@ import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.mapper.TimeSeriesParams; import java.io.IOException; import java.util.ArrayList; @@ -32,6 +34,9 @@ import java.util.function.Function; import java.util.stream.Collectors; +import static org.elasticsearch.index.mapper.TimeSeriesParams.TIME_SERIES_DIMENSION_PARAM; +import static org.elasticsearch.index.mapper.TimeSeriesParams.TIME_SERIES_METRIC_PARAM; + /** * Describes the capabilities of a field optionally merged across multiple indices. */ @@ -41,9 +46,12 @@ public class FieldCapabilities implements Writeable, ToXContentObject { private static final ParseField IS_METADATA_FIELD = new ParseField("metadata_field"); private static final ParseField SEARCHABLE_FIELD = new ParseField("searchable"); private static final ParseField AGGREGATABLE_FIELD = new ParseField("aggregatable"); + private static final ParseField TIME_SERIES_DIMENSION_FIELD = new ParseField(TIME_SERIES_DIMENSION_PARAM); + private static final ParseField TIME_SERIES_METRIC_FIELD = new ParseField(TIME_SERIES_METRIC_PARAM); private static final ParseField INDICES_FIELD = new ParseField("indices"); private static final ParseField NON_SEARCHABLE_INDICES_FIELD = new ParseField("non_searchable_indices"); private static final ParseField NON_AGGREGATABLE_INDICES_FIELD = new ParseField("non_aggregatable_indices"); + private static final ParseField NON_DIMENSION_INDICES_FIELD = new ParseField("non_dimension_indices"); private static final ParseField META_FIELD = new ParseField("meta"); private final String name; @@ -51,10 +59,13 @@ public class FieldCapabilities implements Writeable, ToXContentObject { private final boolean isMetadataField; private final boolean isSearchable; private final boolean isAggregatable; + private final boolean isDimension; + private final TimeSeriesParams.MetricType metricType; private final String[] indices; private final String[] nonSearchableIndices; private final String[] nonAggregatableIndices; + private final String[] nonDimensionIndices; private final Map> meta; @@ -65,30 +76,39 @@ public class FieldCapabilities implements Writeable, ToXContentObject { * @param isMetadataField Whether this field is a metadata field. * @param isSearchable Whether this field is indexed for search. * @param isAggregatable Whether this field can be aggregated on. + * @param isDimension Whether this field can be used as dimension + * @param metricType If this field is a metric field, returns the metric's type or null for non-metrics fields * @param indices The list of indices where this field name is defined as {@code type}, * or null if all indices have the same {@code type} for the field. * @param nonSearchableIndices The list of indices where this field is not searchable, * or null if the field is searchable in all indices. * @param nonAggregatableIndices The list of indices where this field is not aggregatable, * or null if the field is aggregatable in all indices. + * @param nonDimensionIndices The list of indices where this field is not a dimension * @param meta Merged metadata across indices. */ public FieldCapabilities(String name, String type, boolean isMetadataField, boolean isSearchable, boolean isAggregatable, + boolean isDimension, + TimeSeriesParams.MetricType metricType, String[] indices, String[] nonSearchableIndices, String[] nonAggregatableIndices, + String[] nonDimensionIndices, Map> meta) { this.name = name; this.type = type; this.isMetadataField = isMetadataField; this.isSearchable = isSearchable; this.isAggregatable = isAggregatable; + this.isDimension = isDimension; + this.metricType = metricType; this.indices = indices; this.nonSearchableIndices = nonSearchableIndices; this.nonAggregatableIndices = nonAggregatableIndices; + this.nonDimensionIndices = nonDimensionIndices; this.meta = Objects.requireNonNull(meta); } @@ -98,9 +118,21 @@ public FieldCapabilities(String name, String type, this.isMetadataField = in.readBoolean(); this.isSearchable = in.readBoolean(); this.isAggregatable = in.readBoolean(); + if (in.getVersion().onOrAfter(Version.V_8_0_0)) { + this.isDimension = in.readBoolean(); + this.metricType = in.readOptionalEnum(TimeSeriesParams.MetricType.class); + } else { + this.isDimension = false; + this.metricType = null; + } this.indices = in.readOptionalStringArray(); this.nonSearchableIndices = in.readOptionalStringArray(); this.nonAggregatableIndices = in.readOptionalStringArray(); + if (in.getVersion().onOrAfter(Version.V_8_0_0)) { + this.nonDimensionIndices = in.readOptionalStringArray(); + } else { + this.nonDimensionIndices = null; + } meta = in.readMap(StreamInput::readString, i -> i.readSet(StreamInput::readString)); } @@ -111,9 +143,16 @@ public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(isMetadataField); out.writeBoolean(isSearchable); out.writeBoolean(isAggregatable); + if (out.getVersion().onOrAfter(Version.V_8_0_0)) { + out.writeBoolean(isDimension); + out.writeOptionalEnum(metricType); + } out.writeOptionalStringArray(indices); out.writeOptionalStringArray(nonSearchableIndices); out.writeOptionalStringArray(nonAggregatableIndices); + if (out.getVersion().onOrAfter(Version.V_8_0_0)) { + out.writeOptionalStringArray(nonDimensionIndices); + } out.writeMap(meta, StreamOutput::writeString, (o, set) -> o.writeCollection(set, StreamOutput::writeString)); } @@ -124,6 +163,10 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(IS_METADATA_FIELD.getPreferredName(), isMetadataField); builder.field(SEARCHABLE_FIELD.getPreferredName(), isSearchable); builder.field(AGGREGATABLE_FIELD.getPreferredName(), isAggregatable); + builder.field(TIME_SERIES_DIMENSION_FIELD.getPreferredName(), isDimension); + if (metricType != null) { + builder.field(TIME_SERIES_METRIC_FIELD.getPreferredName(), metricType); + } if (indices != null) { builder.array(INDICES_FIELD.getPreferredName(), indices); } @@ -133,6 +176,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (nonAggregatableIndices != null) { builder.array(NON_AGGREGATABLE_INDICES_FIELD.getPreferredName(), nonAggregatableIndices); } + if (nonDimensionIndices != null) { + builder.field(NON_DIMENSION_INDICES_FIELD.getPreferredName(), nonDimensionIndices); + } if (meta.isEmpty() == false) { builder.startObject("meta"); List>> entries = new ArrayList<>(meta.entrySet()); @@ -156,26 +202,38 @@ public static FieldCapabilities fromXContent(String name, XContentParser parser) private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( "field_capabilities", true, - (a, name) -> new FieldCapabilities(name, + (a, name) -> new FieldCapabilities( + name, (String) a[0], a[3] == null ? false : (boolean) a[3], (boolean) a[1], (boolean) a[2], - a[4] != null ? ((List) a[4]).toArray(new String[0]) : null, - a[5] != null ? ((List) a[5]).toArray(new String[0]) : null, + a[4] == null ? false : (boolean) a[4], + a[5] != null ? Enum.valueOf(TimeSeriesParams.MetricType.class, (String) a[5]) : null, a[6] != null ? ((List) a[6]).toArray(new String[0]) : null, - a[7] != null ? ((Map>) a[7]) : Collections.emptyMap())); + a[7] != null ? ((List) a[7]).toArray(new String[0]) : null, + a[8] != null ? ((List) a[8]).toArray(new String[0]) : null, + a[9] != null ? ((List) a[9]).toArray(new String[0]) : null, + a[10] != null ? ((Map>) a[10]) : Collections.emptyMap() + ) + ); static { - PARSER.declareString(ConstructingObjectParser.constructorArg(), TYPE_FIELD); - PARSER.declareBoolean(ConstructingObjectParser.constructorArg(), SEARCHABLE_FIELD); - PARSER.declareBoolean(ConstructingObjectParser.constructorArg(), AGGREGATABLE_FIELD); - PARSER.declareBoolean(ConstructingObjectParser.optionalConstructorArg(), IS_METADATA_FIELD); - PARSER.declareStringArray(ConstructingObjectParser.optionalConstructorArg(), INDICES_FIELD); - PARSER.declareStringArray(ConstructingObjectParser.optionalConstructorArg(), NON_SEARCHABLE_INDICES_FIELD); - PARSER.declareStringArray(ConstructingObjectParser.optionalConstructorArg(), NON_AGGREGATABLE_INDICES_FIELD); - PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), - (parser, context) -> parser.map(HashMap::new, p -> Set.copyOf(p.list())), META_FIELD); + PARSER.declareString(ConstructingObjectParser.constructorArg(), TYPE_FIELD); // 0 + PARSER.declareBoolean(ConstructingObjectParser.constructorArg(), SEARCHABLE_FIELD); // 1 + PARSER.declareBoolean(ConstructingObjectParser.constructorArg(), AGGREGATABLE_FIELD); // 2 + PARSER.declareBoolean(ConstructingObjectParser.optionalConstructorArg(), IS_METADATA_FIELD); // 3 + PARSER.declareBoolean(ConstructingObjectParser.optionalConstructorArg(), TIME_SERIES_DIMENSION_FIELD); // 4 + PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), TIME_SERIES_METRIC_FIELD); // 5 + PARSER.declareStringArray(ConstructingObjectParser.optionalConstructorArg(), INDICES_FIELD); // 6 + PARSER.declareStringArray(ConstructingObjectParser.optionalConstructorArg(), NON_SEARCHABLE_INDICES_FIELD); // 7 + PARSER.declareStringArray(ConstructingObjectParser.optionalConstructorArg(), NON_AGGREGATABLE_INDICES_FIELD); // 8 + PARSER.declareStringArray(ConstructingObjectParser.optionalConstructorArg(), NON_DIMENSION_INDICES_FIELD); // 9 + PARSER.declareObject( + ConstructingObjectParser.optionalConstructorArg(), + (parser, context) -> parser.map(HashMap::new, p -> Set.copyOf(p.list())), + META_FIELD + ); // 10 } /** @@ -206,6 +264,20 @@ public boolean isSearchable() { return isSearchable; } + /** + * Whether this field is a dimension in any indices. + */ + public boolean isDimension() { + return isDimension; + } + + /** + * The metric type + */ + public TimeSeriesParams.MetricType getMetricType() { + return metricType; + } + /** * The type of the field. */ @@ -237,6 +309,14 @@ public String[] nonAggregatableIndices() { return nonAggregatableIndices; } + + /** + * The list of indices where this field has different dimension or metric flag + */ + public String[] nonDimensionIndices() { + return nonDimensionIndices; + } + /** * Return merged metadata across indices. */ @@ -252,20 +332,24 @@ public boolean equals(Object o) { return isMetadataField == that.isMetadataField && isSearchable == that.isSearchable && isAggregatable == that.isAggregatable && + isDimension == that.isDimension && + Objects.equals(metricType, that.metricType) && Objects.equals(name, that.name) && Objects.equals(type, that.type) && Arrays.equals(indices, that.indices) && Arrays.equals(nonSearchableIndices, that.nonSearchableIndices) && Arrays.equals(nonAggregatableIndices, that.nonAggregatableIndices) && + Arrays.equals(nonDimensionIndices, that.nonDimensionIndices) && Objects.equals(meta, that.meta); } @Override public int hashCode() { - int result = Objects.hash(name, type, isMetadataField, isSearchable, isAggregatable, meta); + int result = Objects.hash(name, type, isMetadataField, isSearchable, isAggregatable, isDimension, metricType, meta); result = 31 * result + Arrays.hashCode(indices); result = 31 * result + Arrays.hashCode(nonSearchableIndices); result = 31 * result + Arrays.hashCode(nonAggregatableIndices); + result = 31 * result + Arrays.hashCode(nonDimensionIndices); return result; } @@ -280,6 +364,9 @@ static class Builder { private boolean isMetadataField; private boolean isSearchable; private boolean isAggregatable; + private boolean isDimension; + private TimeSeriesParams.MetricType metricType; + private boolean mertricTypeIsSet; private List indiceList; private Map> meta; @@ -288,6 +375,9 @@ static class Builder { this.type = type; this.isSearchable = true; this.isAggregatable = true; + this.isDimension = true; + this.metricType = null; + this.mertricTypeIsSet = false; this.indiceList = new ArrayList<>(); this.meta = new HashMap<>(); } @@ -295,12 +385,30 @@ static class Builder { /** * Collect the field capabilities for an index. */ - void add(String index, boolean isMetadataField, boolean search, boolean agg, Map meta) { - IndexCaps indexCaps = new IndexCaps(index, search, agg); + void add( + String index, + boolean isMetadataField, + boolean search, + boolean agg, + boolean isDimension, + TimeSeriesParams.MetricType metricType, + Map meta + ) { + IndexCaps indexCaps = new IndexCaps(index, search, agg, isDimension, metricType != null); indiceList.add(indexCaps); this.isSearchable &= search; this.isAggregatable &= agg; this.isMetadataField |= isMetadataField; + this.isDimension &= isDimension; + // If we have discrepancy in metric types - we ignore it + if (this.mertricTypeIsSet) { + if (this.metricType != metricType) { + this.metricType = null; + } + } else { + this.mertricTypeIsSet = true; + this.metricType = metricType; + } for (Map.Entry entry : meta.entrySet()) { this.meta.computeIfAbsent(entry.getKey(), key -> new HashSet<>()) .add(entry.getValue()); @@ -347,12 +455,37 @@ FieldCapabilities build(boolean withIndices) { } else { nonAggregatableIndices = null; } + + + final String[] nonDimensionIndices; + if (isDimension == false && indiceList.stream().anyMatch((caps) -> caps.isDimension)) { + // Collect all indices that disagree on the dimension flag + nonDimensionIndices = indiceList.stream() + .filter((caps) -> caps.isDimension == false) + .map(caps -> caps.name) + .toArray(String[]::new); + } else { + nonDimensionIndices = null; + } + final Function>, Set> entryValueFunction = Map.Entry::getValue; Map> immutableMeta = meta.entrySet().stream() .collect(Collectors.toUnmodifiableMap( Map.Entry::getKey, entryValueFunction.andThen(Set::copyOf))); - return new FieldCapabilities(name, type, isMetadataField, isSearchable, isAggregatable, - indices, nonSearchableIndices, nonAggregatableIndices, immutableMeta); + return new FieldCapabilities( + name, + type, + isMetadataField, + isSearchable, + isAggregatable, + isDimension, + metricType, + indices, + nonSearchableIndices, + nonAggregatableIndices, + nonDimensionIndices, + immutableMeta + ); } } @@ -360,11 +493,15 @@ private static class IndexCaps { final String name; final boolean isSearchable; final boolean isAggregatable; + final boolean isMetric; + final boolean isDimension; - IndexCaps(String name, boolean isSearchable, boolean isAggregatable) { + IndexCaps(String name, boolean isSearchable, boolean isAggregatable, boolean isDimension, boolean isMetric) { this.name = name; this.isSearchable = isSearchable; this.isAggregatable = isAggregatable; + this.isMetric = isMetric; + this.isDimension = isDimension; } } } diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/IndexFieldCapabilities.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/IndexFieldCapabilities.java index a59a75a4a5e09..e0164196bedc1 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/IndexFieldCapabilities.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/IndexFieldCapabilities.java @@ -8,10 +8,12 @@ package org.elasticsearch.action.fieldcaps; +import org.elasticsearch.Version; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.util.StringLiteralDeduplicator; +import org.elasticsearch.index.mapper.TimeSeriesParams; import java.io.IOException; import java.util.Map; @@ -29,6 +31,8 @@ public class IndexFieldCapabilities implements Writeable { private final boolean isMetadatafield; private final boolean isSearchable; private final boolean isAggregatable; + private final boolean isDimension; + private final TimeSeriesParams.MetricType metricType; private final Map meta; /** @@ -41,6 +45,8 @@ public class IndexFieldCapabilities implements Writeable { IndexFieldCapabilities(String name, String type, boolean isMetadatafield, boolean isSearchable, boolean isAggregatable, + boolean isDimension, + TimeSeriesParams.MetricType metricType, Map meta) { this.name = name; @@ -48,6 +54,8 @@ public class IndexFieldCapabilities implements Writeable { this.isMetadatafield = isMetadatafield; this.isSearchable = isSearchable; this.isAggregatable = isAggregatable; + this.isDimension = isDimension; + this.metricType = metricType; this.meta = meta; } @@ -57,6 +65,13 @@ public class IndexFieldCapabilities implements Writeable { this.isMetadatafield = in.readBoolean(); this.isSearchable = in.readBoolean(); this.isAggregatable = in.readBoolean(); + if (in.getVersion().onOrAfter(Version.V_8_0_0)) { + this.isDimension = in.readBoolean(); + this.metricType = in.readOptionalEnum(TimeSeriesParams.MetricType.class); + } else { + this.isDimension = false; + this.metricType = null; + } this.meta = in.readMap(StreamInput::readString, StreamInput::readString); } @@ -67,6 +82,10 @@ public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(isMetadatafield); out.writeBoolean(isSearchable); out.writeBoolean(isAggregatable); + if (out.getVersion().onOrAfter(Version.V_8_0_0)) { + out.writeBoolean(isDimension); + out.writeOptionalEnum(metricType); + } out.writeMap(meta, StreamOutput::writeString, StreamOutput::writeString); } @@ -90,6 +109,14 @@ public boolean isSearchable() { return isSearchable; } + public boolean isDimension() { + return isDimension; + } + + public TimeSeriesParams.MetricType getMetricType() { + return metricType; + } + public Map meta() { return meta; } @@ -102,6 +129,8 @@ public boolean equals(Object o) { return isMetadatafield == that.isMetadatafield && isSearchable == that.isSearchable && isAggregatable == that.isAggregatable && + isDimension == that.isDimension && + Objects.equals(metricType, that.metricType) && Objects.equals(name, that.name) && Objects.equals(type, that.type) && Objects.equals(meta, that.meta); @@ -109,6 +138,6 @@ public boolean equals(Object o) { @Override public int hashCode() { - return Objects.hash(name, type, isMetadatafield, isSearchable, isAggregatable, meta); + return Objects.hash(name, type, isMetadatafield, isSearchable, isAggregatable, isDimension, metricType, meta); } } diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java index e997ca1af0a56..3e620b70ba9aa 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java @@ -275,7 +275,7 @@ private void addUnmappedFields(String[] indices, String field, Map> resp Map typeMap = responseMapBuilder.computeIfAbsent(field, f -> new HashMap<>()); FieldCapabilities.Builder builder = typeMap.computeIfAbsent(fieldCap.getType(), key -> new FieldCapabilities.Builder(field, key)); - builder.add(response.getIndexName(), isMetadataField, fieldCap.isSearchable(), fieldCap.isAggregatable(), fieldCap.meta()); + builder.add( + response.getIndexName(), + isMetadataField, + fieldCap.isSearchable(), + fieldCap.isAggregatable(), + fieldCap.isDimension(), + fieldCap.getMetricType(), + fieldCap.meta() + ); } } @@ -350,8 +358,16 @@ private FieldCapabilitiesIndexResponse shardOperation(final FieldCapabilitiesInd MappedFieldType ft = searchExecutionContext.getFieldType(field); boolean isMetadataField = searchExecutionContext.isMetadataField(field); if (isMetadataField || fieldPredicate.test(ft.name())) { - IndexFieldCapabilities fieldCap = new IndexFieldCapabilities(field, - ft.familyTypeName(), isMetadataField, ft.isSearchable(), ft.isAggregatable(), ft.meta()); + IndexFieldCapabilities fieldCap = new IndexFieldCapabilities( + field, + ft.familyTypeName(), + isMetadataField, + ft.isSearchable(), + ft.isAggregatable(), + ft.isDimension(), + ft.getMetricType(), + ft.meta() + ); responseMap.put(field, fieldCap); } else { continue; @@ -376,7 +392,7 @@ private FieldCapabilitiesIndexResponse shardOperation(final FieldCapabilitiesInd if (mapper != null) { String type = mapper.isNested() ? "nested" : "object"; IndexFieldCapabilities fieldCap = new IndexFieldCapabilities(parentField, type, - false, false, false, Collections.emptyMap()); + false, false, false, false, null, Collections.emptyMap()); responseMap.put(parentField, fieldCap); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java index 5e13e3cc3f10a..6d2970b9ec656 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java @@ -172,6 +172,13 @@ public boolean isDimension() { return false; } + /** + * @return metric type or null if the field is not a metric field + */ + public TimeSeriesParams.MetricType getMetricType() { + return null; + } + /** Generates a query that will only match documents that contain the given value. * The default implementation returns a {@link TermQuery} over the value bytes * @throws IllegalArgumentException if {@code value} cannot be converted to the expected data type or if the field is not searchable diff --git a/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponseTests.java b/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponseTests.java index 7d7f92faec272..adc60135f0161 100644 --- a/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponseTests.java @@ -14,6 +14,7 @@ import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.mapper.TimeSeriesParams; import org.elasticsearch.test.AbstractWireSerializingTestCase; import org.hamcrest.Matchers; @@ -73,7 +74,7 @@ private static IndexFieldCapabilities randomFieldCaps(String fieldName) { } return new IndexFieldCapabilities(fieldName, randomAlphaOfLengthBetween(5, 20), - randomBoolean(), randomBoolean(), randomBoolean(), meta); + randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomFrom(TimeSeriesParams.MetricType.values()), meta); } @Override diff --git a/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesTests.java b/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesTests.java index 6b93bac3d3796..593f3b4d6cf67 100644 --- a/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesTests.java +++ b/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesTests.java @@ -10,6 +10,7 @@ import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.mapper.TimeSeriesParams; import org.elasticsearch.test.AbstractSerializingTestCase; import java.io.IOException; @@ -40,72 +41,119 @@ protected Writeable.Reader instanceReader() { public void testBuilder() { FieldCapabilities.Builder builder = new FieldCapabilities.Builder("field", "type"); - builder.add("index1", false, true, false, Collections.emptyMap()); - builder.add("index2", false, true, false, Collections.emptyMap()); - builder.add("index3", false, true, false, Collections.emptyMap()); + builder.add("index1", false, true, false, false, null, Collections.emptyMap()); + builder.add("index2", false, true, false, false, null, Collections.emptyMap()); + builder.add("index3", false, true, false, false, null, Collections.emptyMap()); { FieldCapabilities cap1 = builder.build(false); assertThat(cap1.isSearchable(), equalTo(true)); assertThat(cap1.isAggregatable(), equalTo(false)); + assertThat(cap1.isDimension(), equalTo(false)); + assertNull(cap1.getMetricType()); assertNull(cap1.indices()); assertNull(cap1.nonSearchableIndices()); assertNull(cap1.nonAggregatableIndices()); + assertNull(cap1.nonDimensionIndices()); assertEquals(Collections.emptyMap(), cap1.meta()); FieldCapabilities cap2 = builder.build(true); assertThat(cap2.isSearchable(), equalTo(true)); assertThat(cap2.isAggregatable(), equalTo(false)); + assertThat(cap2.isDimension(), equalTo(false)); + assertNull(cap2.getMetricType()); assertThat(cap2.indices().length, equalTo(3)); assertThat(cap2.indices(), equalTo(new String[]{"index1", "index2", "index3"})); assertNull(cap2.nonSearchableIndices()); assertNull(cap2.nonAggregatableIndices()); + assertNull(cap2.nonDimensionIndices()); assertEquals(Collections.emptyMap(), cap2.meta()); } builder = new FieldCapabilities.Builder("field", "type"); - builder.add("index1", false, false, true, Collections.emptyMap()); - builder.add("index2", false, true, false, Collections.emptyMap()); - builder.add("index3", false, false, false, Collections.emptyMap()); + builder.add("index1", false, false, true, true, null, Collections.emptyMap()); + builder.add("index2", false, true, false, false, TimeSeriesParams.MetricType.counter, Collections.emptyMap()); + builder.add("index3", false, false, false, false, null, Collections.emptyMap()); { FieldCapabilities cap1 = builder.build(false); assertThat(cap1.isSearchable(), equalTo(false)); assertThat(cap1.isAggregatable(), equalTo(false)); + assertThat(cap1.isDimension(), equalTo(false)); + assertNull(cap1.getMetricType()); assertNull(cap1.indices()); assertThat(cap1.nonSearchableIndices(), equalTo(new String[]{"index1", "index3"})); assertThat(cap1.nonAggregatableIndices(), equalTo(new String[]{"index2", "index3"})); + assertThat(cap1.nonDimensionIndices(), equalTo(new String[]{"index2", "index3"})); assertEquals(Collections.emptyMap(), cap1.meta()); FieldCapabilities cap2 = builder.build(true); assertThat(cap2.isSearchable(), equalTo(false)); assertThat(cap2.isAggregatable(), equalTo(false)); + assertThat(cap2.isDimension(), equalTo(false)); + assertNull(cap2.getMetricType()); assertThat(cap2.indices().length, equalTo(3)); assertThat(cap2.indices(), equalTo(new String[]{"index1", "index2", "index3"})); assertThat(cap2.nonSearchableIndices(), equalTo(new String[]{"index1", "index3"})); assertThat(cap2.nonAggregatableIndices(), equalTo(new String[]{"index2", "index3"})); + assertThat(cap2.nonDimensionIndices(), equalTo(new String[]{"index2", "index3"})); assertEquals(Collections.emptyMap(), cap2.meta()); } builder = new FieldCapabilities.Builder("field", "type"); - builder.add("index1", false, true, true, Collections.emptyMap()); - builder.add("index2", false, true, true, Map.of("foo", "bar")); - builder.add("index3", false, true, true, Map.of("foo", "quux")); + builder.add("index1", false, true, true, true, TimeSeriesParams.MetricType.counter, Collections.emptyMap()); + builder.add("index2", false, true, true, true, TimeSeriesParams.MetricType.counter, Map.of("foo", "bar")); + builder.add("index3", false, true, true, true, TimeSeriesParams.MetricType.counter, Map.of("foo", "quux")); { FieldCapabilities cap1 = builder.build(false); assertThat(cap1.isSearchable(), equalTo(true)); assertThat(cap1.isAggregatable(), equalTo(true)); + assertThat(cap1.isDimension(), equalTo(true)); + assertThat(cap1.getMetricType(), equalTo(TimeSeriesParams.MetricType.counter)); assertNull(cap1.indices()); assertNull(cap1.nonSearchableIndices()); assertNull(cap1.nonAggregatableIndices()); + assertNull(cap1.nonDimensionIndices()); assertEquals(Map.of("foo", Set.of("bar", "quux")), cap1.meta()); FieldCapabilities cap2 = builder.build(true); assertThat(cap2.isSearchable(), equalTo(true)); assertThat(cap2.isAggregatable(), equalTo(true)); + assertThat(cap2.isDimension(), equalTo(true)); + assertThat(cap2.getMetricType(), equalTo(TimeSeriesParams.MetricType.counter)); assertThat(cap2.indices().length, equalTo(3)); assertThat(cap2.indices(), equalTo(new String[]{"index1", "index2", "index3"})); assertNull(cap2.nonSearchableIndices()); assertNull(cap2.nonAggregatableIndices()); + assertNull(cap2.nonDimensionIndices()); + assertEquals(Map.of("foo", Set.of("bar", "quux")), cap2.meta()); + } + + builder = new FieldCapabilities.Builder("field", "type"); + builder.add("index1", false, true, true, true, TimeSeriesParams.MetricType.counter, Collections.emptyMap()); + builder.add("index2", false, true, true, true, TimeSeriesParams.MetricType.gauge, Map.of("foo", "bar")); + builder.add("index3", false, true, true, true, TimeSeriesParams.MetricType.counter, Map.of("foo", "quux")); + { + FieldCapabilities cap1 = builder.build(false); + assertThat(cap1.isSearchable(), equalTo(true)); + assertThat(cap1.isAggregatable(), equalTo(true)); + assertThat(cap1.isDimension(), equalTo(true)); + assertNull(cap1.getMetricType()); + assertNull(cap1.indices()); + assertNull(cap1.nonSearchableIndices()); + assertNull(cap1.nonAggregatableIndices()); + assertNull(cap1.nonDimensionIndices()); + assertEquals(Map.of("foo", Set.of("bar", "quux")), cap1.meta()); + + FieldCapabilities cap2 = builder.build(true); + assertThat(cap2.isSearchable(), equalTo(true)); + assertThat(cap2.isAggregatable(), equalTo(true)); + assertThat(cap2.isDimension(), equalTo(true)); + assertNull(cap2.getMetricType()); + assertThat(cap2.indices().length, equalTo(3)); + assertThat(cap2.indices(), equalTo(new String[]{"index1", "index2", "index3"})); + assertNull(cap2.nonSearchableIndices()); + assertNull(cap2.nonAggregatableIndices()); + assertNull(cap2.nonDimensionIndices()); assertEquals(Map.of("foo", Set.of("bar", "quux")), cap2.meta()); } } @@ -133,6 +181,14 @@ static FieldCapabilities randomFieldCaps(String fieldName) { } } + String[] nonDimensionIndices = null; + if (randomBoolean()) { + nonDimensionIndices = new String[randomIntBetween(0, 5)]; + for (int i = 0; i < nonDimensionIndices.length; i++) { + nonDimensionIndices[i] = randomAlphaOfLengthBetween(5, 20); + } + } + Map> meta; switch (randomInt(2)) { case 0: @@ -146,9 +202,20 @@ static FieldCapabilities randomFieldCaps(String fieldName) { break; } - return new FieldCapabilities(fieldName, - randomAlphaOfLengthBetween(5, 20), randomBoolean(), randomBoolean(), randomBoolean(), - indices, nonSearchableIndices, nonAggregatableIndices, meta); + return new FieldCapabilities( + fieldName, + randomAlphaOfLengthBetween(5, 20), + randomBoolean(), + randomBoolean(), + randomBoolean(), + randomBoolean(), + randomFrom(TimeSeriesParams.MetricType.values()), + indices, + nonSearchableIndices, + nonAggregatableIndices, + nonDimensionIndices, + meta + ); } @Override @@ -158,11 +225,14 @@ protected FieldCapabilities mutateInstance(FieldCapabilities instance) { boolean isMetadataField = instance.isMetadataField(); boolean isSearchable = instance.isSearchable(); boolean isAggregatable = instance.isAggregatable(); + boolean isDimension = instance.isDimension(); + TimeSeriesParams.MetricType metricType = instance.getMetricType(); String[] indices = instance.indices(); String[] nonSearchableIndices = instance.nonSearchableIndices(); String[] nonAggregatableIndices = instance.nonAggregatableIndices(); + String[] nonDimensionIndices = instance.nonDimensionIndices(); Map> meta = instance.meta(); - switch (between(0, 8)) { + switch (between(0, 11)) { case 0: name += randomAlphaOfLengthBetween(1, 10); break; @@ -229,10 +299,51 @@ protected FieldCapabilities mutateInstance(FieldCapabilities instance) { case 8: isMetadataField = isMetadataField == false; break; + case 9: + isDimension = isDimension == false; + break; + case 10: + if (metricType == null) { + metricType = randomFrom(TimeSeriesParams.MetricType.values()); + } else { + if (randomBoolean()) { + metricType = null; + } else { + metricType = randomValueOtherThan(metricType, () -> randomFrom(TimeSeriesParams.MetricType.values())); + } + } + break; + case 11: + String[] newTimeSeriesDimensionsConflictsIndices; + int startTimeSeriesDimensionsConflictsPos = 0; + if (nonDimensionIndices == null) { + newTimeSeriesDimensionsConflictsIndices = new String[between(1, 10)]; + } else { + newTimeSeriesDimensionsConflictsIndices = Arrays.copyOf(nonDimensionIndices, + nonDimensionIndices.length + between(1, 10)); + startTimeSeriesDimensionsConflictsPos = nonDimensionIndices.length; + } + for (int i = startTimeSeriesDimensionsConflictsPos; i < newTimeSeriesDimensionsConflictsIndices.length; i++) { + newTimeSeriesDimensionsConflictsIndices[i] = randomAlphaOfLengthBetween(5, 20); + } + nonDimensionIndices = newTimeSeriesDimensionsConflictsIndices; + break; default: throw new AssertionError(); } - return new FieldCapabilities(name, type, isMetadataField, isSearchable, isAggregatable, - indices, nonSearchableIndices, nonAggregatableIndices, meta); + return new FieldCapabilities( + name, + type, + isMetadataField, + isSearchable, + isAggregatable, + isDimension, + metricType, + indices, + nonSearchableIndices, + nonAggregatableIndices, + nonDimensionIndices, + meta + ); } } diff --git a/server/src/test/java/org/elasticsearch/action/fieldcaps/MergedFieldCapabilitiesResponseTests.java b/server/src/test/java/org/elasticsearch/action/fieldcaps/MergedFieldCapabilitiesResponseTests.java index 7442d84f05e90..a00055d983b2a 100644 --- a/server/src/test/java/org/elasticsearch/action/fieldcaps/MergedFieldCapabilitiesResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/fieldcaps/MergedFieldCapabilitiesResponseTests.java @@ -15,6 +15,7 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.mapper.TimeSeriesParams; import org.elasticsearch.test.AbstractSerializingTestCase; import java.io.IOException; @@ -115,6 +116,7 @@ public void testToXContent() throws IOException { " \"metadata_field\": false," + " \"searchable\": false," + " \"aggregatable\": true," + + " \"time_series_dimension\": true," + " \"indices\": [\"index3\", \"index4\"]," + " \"non_searchable_indices\": [\"index4\"] " + " }," + @@ -123,8 +125,11 @@ public void testToXContent() throws IOException { " \"metadata_field\": false," + " \"searchable\": true," + " \"aggregatable\": false," + + " \"time_series_dimension\": false," + + " \"time_series_metric\": \"counter\"," + " \"indices\": [\"index1\", \"index2\"]," + - " \"non_aggregatable_indices\": [\"index1\"] " + + " \"non_aggregatable_indices\": [\"index1\"]," + + " \"non_dimension_indices\":[\"index4\"] " + " }" + " }," + " \"title\": { " + @@ -132,7 +137,8 @@ public void testToXContent() throws IOException { " \"type\": \"text\"," + " \"metadata_field\": false," + " \"searchable\": true," + - " \"aggregatable\": false" + + " \"aggregatable\": false," + + " \"time_series_dimension\": false" + " }" + " }" + " }," + @@ -147,20 +153,20 @@ public void testToXContent() throws IOException { private static FieldCapabilitiesResponse createSimpleResponse() { Map titleCapabilities = new HashMap<>(); - titleCapabilities.put("text", new FieldCapabilities("title", "text", false, true, false, - null, null, null, Collections.emptyMap())); + titleCapabilities.put("text", new FieldCapabilities("title", "text", false, true, false, false, null, + null, null, null, null, Collections.emptyMap())); Map ratingCapabilities = new HashMap<>(); ratingCapabilities.put("long", new FieldCapabilities("rating", "long", - false, true, false, - new String[]{"index1", "index2"}, - null, - new String[]{"index1"}, Collections.emptyMap())); + false, true, false, false, TimeSeriesParams.MetricType.counter, + new String[]{"index1", "index2"}, null, new String[]{"index1"}, new String[]{"index4"}, + Collections.emptyMap() + )); ratingCapabilities.put("keyword", new FieldCapabilities("rating", "keyword", - false, false, true, - new String[]{"index3", "index4"}, - new String[]{"index4"}, - null, Collections.emptyMap())); + false, false, true, true, null, + new String[]{"index3", "index4"}, new String[]{"index4"}, null, null, + Collections.emptyMap() + )); Map> responses = new HashMap<>(); responses.put("title", titleCapabilities); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/ClassificationTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/ClassificationTests.java index 08b439dc8f928..bf816148d489d 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/ClassificationTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/ClassificationTests.java @@ -518,6 +518,6 @@ public Long getCardinality(String field) { } private static FieldCapabilities createFieldCapabilities(String field, String type) { - return new FieldCapabilities(field, type, false, true, true, null, null, null, Collections.emptyMap()); + return new FieldCapabilities(field, type, false, true, true, false, null, null, null, null, null, Collections.emptyMap()); } } diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/CancellationTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/CancellationTests.java index c4377d351822d..f25902858ae11 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/CancellationTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/CancellationTests.java @@ -106,11 +106,11 @@ public void onFailure(Exception e) { private Map> fields(String[] indices) { FieldCapabilities fooField = - new FieldCapabilities("foo", "integer", false, true, true, indices, null, null, emptyMap()); + new FieldCapabilities("foo", "integer", false, true, true, false, null, indices, null, null, null, emptyMap()); FieldCapabilities categoryField = - new FieldCapabilities("event.category", "keyword", false, true, true, indices, null, null, emptyMap()); + new FieldCapabilities("event.category", "keyword", false, true, true, false, null, indices, null, null, null, emptyMap()); FieldCapabilities timestampField = - new FieldCapabilities("@timestamp", "date", false, true, true, indices, null, null, emptyMap()); + new FieldCapabilities("@timestamp", "date", false, true, true, false, null, indices, null, null, null, emptyMap()); Map> fields = new HashMap<>(); fields.put(fooField.getName(), singletonMap(fooField.getName(), fooField)); fields.put(categoryField.getName(), singletonMap(categoryField.getName(), categoryField)); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/DestinationIndexTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/DestinationIndexTests.java index 0fe48f85c18f2..9ec1bd5cca6ff 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/DestinationIndexTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/DestinationIndexTests.java @@ -470,6 +470,6 @@ private static DataFrameAnalyticsConfig createConfig(DataFrameAnalysis analysis) } private static FieldCapabilities createFieldCapabilities(String field, String type) { - return new FieldCapabilities(field, type, false, true, true, null, null, null, Collections.emptyMap()); + return new FieldCapabilities(field, type, false, true, true, false, null, null, null, null, null, Collections.emptyMap()); } } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetectorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetectorTests.java index 9fe27e3baf95d..cbdcc9241e5c5 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetectorTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetectorTests.java @@ -1331,7 +1331,7 @@ private MockFieldCapsResponseBuilder addField(String field, boolean isMetadataFi Map caps = new HashMap<>(); for (String type : types) { caps.put(type, new FieldCapabilities(field, type, - isMetadataField, true, isAggregatable, null, null, null, Collections.emptyMap())); + isMetadataField, true, isAggregatable, false, null, null, null, null, null, Collections.emptyMap())); } fieldCaps.put(field, caps); return this; diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/CancellationTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/CancellationTests.java index cc8b5ff137fb3..ef731a60e6ecc 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/CancellationTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/CancellationTests.java @@ -87,11 +87,11 @@ public void onFailure(Exception e) { private Map> fields(String[] indices) { FieldCapabilities fooField = - new FieldCapabilities("foo", "integer", false, true, true, indices, null, null, emptyMap()); + new FieldCapabilities("foo", "integer", false, true, true, false, null, indices, null, null, null, emptyMap()); FieldCapabilities categoryField = - new FieldCapabilities("event.category", "keyword", false, true, true, indices, null, null, emptyMap()); + new FieldCapabilities("event.category", "keyword", false, true, true, false, null, indices, null, null, null, emptyMap()); FieldCapabilities timestampField = - new FieldCapabilities("@timestamp", "date", false, true, true, indices, null, null, emptyMap()); + new FieldCapabilities("@timestamp", "date", false, true, true, false, null, indices, null, null, null, emptyMap()); Map> fields = new HashMap<>(); fields.put(fooField.getName(), singletonMap(fooField.getName(), fooField)); fields.put(categoryField.getName(), singletonMap(categoryField.getName(), categoryField)); diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java index af11af45c72eb..4533e8e08eccd 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java @@ -247,10 +247,12 @@ public void testMergeIncompatibleCapabilitiesOfObjectFields() throws Exception { addFieldCaps(fieldCaps, fieldName + ".keyword", "keyword", true, true); Map multi = new HashMap<>(); - multi.put("long", new FieldCapabilities(fieldName, "long", false, true, true, new String[] { "one-index" }, null, null, - Collections.emptyMap())); - multi.put("text", new FieldCapabilities(fieldName, "text", false, true, false, new String[] { "another-index" }, null, null, - Collections.emptyMap())); + multi.put("long", new FieldCapabilities(fieldName, "long", false, true, true, false, null, new String[] { "one-index" }, null, null, + null, Collections.emptyMap() + )); + multi.put("text", new FieldCapabilities(fieldName, "text", false, true, false, false, null, new String[] { "another-index" }, null, + null, null, Collections.emptyMap() + )); fieldCaps.put(fieldName, multi); @@ -321,7 +323,7 @@ public void testMultipleCompatibleIndicesWithDifferentFields() { public void testIndexWithNoMapping() { Map> versionFC = singletonMap("_version", singletonMap("_index", new FieldCapabilities("_version", "_version", true, false, false, - null, null, null, Collections.emptyMap()))); + false, null, null, null, null, null, Collections.emptyMap()))); assertTrue(mergedMappings("*", new String[] { "empty" }, versionFC).isValid()); } @@ -396,7 +398,7 @@ private static class UpdateableFieldCapabilities extends FieldCapabilities { List nonAggregatableIndices = new ArrayList<>(); UpdateableFieldCapabilities(String name, String type, boolean isSearchable, boolean isAggregatable) { - super(name, type, false, isSearchable, isAggregatable, null, null, null, Collections.emptyMap()); + super(name, type, false, isSearchable, isAggregatable, false, null, null, null, null, null, Collections.emptyMap()); } @Override @@ -443,7 +445,7 @@ private void addFieldCaps(Map> fieldCaps, boolean isAggregatable) { Map cap = new HashMap<>(); cap.put(type, new FieldCapabilities(name, type, isMetadataField, - isSearchable, isAggregatable, null, null, null, Collections.emptyMap())); + isSearchable, isAggregatable, false, null, null, null, null, null, Collections.emptyMap())); fieldCaps.put(name, cap); } diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/common/DocumentConversionUtilsTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/common/DocumentConversionUtilsTests.java index eb583f2f3ce81..fcb474a6abc5a 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/common/DocumentConversionUtilsTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/common/DocumentConversionUtilsTests.java @@ -117,6 +117,7 @@ public void testExtractFieldMappings() { private static FieldCapabilities createFieldCapabilities(String name, String type) { return new FieldCapabilities( - name, type, false, true, true, Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY, Collections.emptyMap()); + name, type, false, true, true, false, null, Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY, null, + Collections.emptyMap()); } } diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/AggregationSchemaAndResultTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/AggregationSchemaAndResultTests.java index 4e9b2d5f69c3c..acdbff18fbe78 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/AggregationSchemaAndResultTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/AggregationSchemaAndResultTests.java @@ -90,7 +90,7 @@ protected void field, Collections.singletonMap( type, - new FieldCapabilities(field, type, false, true, true, null, null, null, emptyMap()) + new FieldCapabilities(field, type, false, true, true, false, null, null, null, null, null, emptyMap()) ) ); } diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/SchemaUtilTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/SchemaUtilTests.java index d406393b8b15a..646959046154c 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/SchemaUtilTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/SchemaUtilTests.java @@ -217,9 +217,12 @@ private static FieldCapabilities createFieldCapabilities(String name, String typ false, true, true, + false, + null, Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY, + null, Collections.emptyMap() ); } From e0da632919c66b41bb4c113dfcff5812f0b50662 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Wed, 6 Oct 2021 12:24:29 -1000 Subject: [PATCH 2/8] Fix tests --- .../src/test/java/org/elasticsearch/client/SearchIT.java | 4 ++-- x-pack/qa/runtime-fields/build.gradle | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java index 8ee95aeb53d51..77edf8d8e7f2a 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java @@ -1253,11 +1253,11 @@ public void testFieldCaps() throws IOException { assertEquals(2, ratingResponse.size()); FieldCapabilities expectedKeywordCapabilities = new FieldCapabilities( - "rating", "keyword", false, true, true, false, null, null, null, new String[]{"index2"}, null, Collections.emptyMap()); + "rating", "keyword", false, true, true, false, null, new String[]{"index2"}, null, null, null, Collections.emptyMap()); assertEquals(expectedKeywordCapabilities, ratingResponse.get("keyword")); FieldCapabilities expectedLongCapabilities = new FieldCapabilities( - "rating", "long", false, true, true, false, null, null, null, new String[]{"index1"}, null, Collections.emptyMap()); + "rating", "long", false, true, true, false, null, new String[]{"index1"}, null, null, null, Collections.emptyMap()); assertEquals(expectedLongCapabilities, ratingResponse.get("long")); // Check the capabilities for the 'field' field. diff --git a/x-pack/qa/runtime-fields/build.gradle b/x-pack/qa/runtime-fields/build.gradle index 2701c52b4a2b4..c8dc8a378dda2 100644 --- a/x-pack/qa/runtime-fields/build.gradle +++ b/x-pack/qa/runtime-fields/build.gradle @@ -98,6 +98,7 @@ subprojects { 'search/330_fetch_fields/error includes glob pattern', // we need a @timestamp field to be defined in index mapping 'search/380_sort_segments_on_timestamp/*', + 'field_caps/40_time_series/*', /////// NOT SUPPORTED /////// ].join(',') } From 3a3b5354b39fae123fabda36aece251a50119592 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Mon, 11 Oct 2021 09:42:13 -1000 Subject: [PATCH 3/8] Address review comments --- .../org/elasticsearch/client/SearchIT.java | 6 +- docs/reference/search/field-caps.asciidoc | 19 ++--- .../test/field_caps/40_time_series.yml | 26 +++--- .../search/fieldcaps/FieldCapabilitiesIT.java | 27 +++--- .../action/fieldcaps/FieldCapabilities.java | 83 +++++++++++++++++-- .../fieldcaps/FieldCapabilitiesTests.java | 28 ++++++- .../MergedFieldCapabilitiesResponseTests.java | 10 +-- .../analyses/ClassificationTests.java | 2 +- .../xpack/eql/analysis/CancellationTests.java | 6 +- .../ml/dataframe/DestinationIndexTests.java | 2 +- .../ExtractedFieldsDetectorTests.java | 2 +- .../xpack/sql/analysis/CancellationTests.java | 6 +- .../analysis/index/IndexResolverTests.java | 14 ++-- .../common/DocumentConversionUtilsTests.java | 3 +- .../AggregationSchemaAndResultTests.java | 2 +- .../transforms/pivot/SchemaUtilTests.java | 3 - 16 files changed, 160 insertions(+), 79 deletions(-) diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java index 77edf8d8e7f2a..040004e822758 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java @@ -1253,11 +1253,11 @@ public void testFieldCaps() throws IOException { assertEquals(2, ratingResponse.size()); FieldCapabilities expectedKeywordCapabilities = new FieldCapabilities( - "rating", "keyword", false, true, true, false, null, new String[]{"index2"}, null, null, null, Collections.emptyMap()); + "rating", "keyword", false, true, true, new String[]{"index2"}, null, null, Collections.emptyMap()); assertEquals(expectedKeywordCapabilities, ratingResponse.get("keyword")); FieldCapabilities expectedLongCapabilities = new FieldCapabilities( - "rating", "long", false, true, true, false, null, new String[]{"index1"}, null, null, null, Collections.emptyMap()); + "rating", "long", false, true, true, new String[]{"index1"}, null, null, Collections.emptyMap()); assertEquals(expectedLongCapabilities, ratingResponse.get("long")); // Check the capabilities for the 'field' field. @@ -1266,7 +1266,7 @@ public void testFieldCaps() throws IOException { assertEquals(1, fieldResponse.size()); FieldCapabilities expectedTextCapabilities = new FieldCapabilities( - "field", "text", false, true, false, false, null, null, null, null, null, Collections.emptyMap()); + "field", "text", false, true, false, null, null, null, Collections.emptyMap()); assertEquals(expectedTextCapabilities, fieldResponse.get("text")); } diff --git a/docs/reference/search/field-caps.asciidoc b/docs/reference/search/field-caps.asciidoc index f5e95c04c8731..2b4c89edf22ad 100644 --- a/docs/reference/search/field-caps.asciidoc +++ b/docs/reference/search/field-caps.asciidoc @@ -133,6 +133,9 @@ field types are all described as the `keyword` type family. The list of indices where this field is not marked as a dimension field, or null if all indices have the same definition for the field. +`metric_conflicts_indices`:: + The list of indices where this field is present but don't have the same metrics type. + `meta`:: Merged metadata across all indices as a map of string keys to arrays of values. A value length of 1 indicates that all indices had the same value for this key, @@ -173,7 +176,6 @@ The API returns the following response: "metadata_field": false, "searchable": true, "aggregatable": false, - "time_series_dimension": false, "indices": [ "index1", "index2" ], "non_aggregatable_indices": [ "index1" ] <2> }, @@ -181,7 +183,6 @@ The API returns the following response: "metadata_field": false, "searchable": false, "aggregatable": true, - "time_series_dimension": false, "indices": [ "index3", "index4" ], "non_searchable_indices": [ "index4" ] <3> } @@ -190,8 +191,7 @@ The API returns the following response: "text": { "metadata_field": false, "searchable": true, - "aggregatable": false, - "time_series_dimension": false + "aggregatable": false } } } @@ -227,7 +227,6 @@ in some indices but not all: "metadata_field": false, "searchable": true, "aggregatable": false, - "time_series_dimension": false, "indices": [ "index1", "index2" ], "non_aggregatable_indices": [ "index1" ] }, @@ -235,7 +234,6 @@ in some indices but not all: "metadata_field": false, "searchable": false, "aggregatable": true, - "time_series_dimension": false, "indices": [ "index3", "index4" ], "non_searchable_indices": [ "index4" ] }, @@ -243,8 +241,7 @@ in some indices but not all: "metadata_field": false, "indices": [ "index5" ], "searchable": false, - "aggregatable": false, - "time_series_dimension": false + "aggregatable": false } }, "title": { @@ -252,15 +249,13 @@ in some indices but not all: "metadata_field": false, "indices": [ "index1", "index2", "index3", "index4" ], "searchable": true, - "aggregatable": false, - "time_series_dimension": false + "aggregatable": false }, "unmapped": { <2> "metadata_field": false, "indices": [ "index5" ], "searchable": false, - "aggregatable": false, - "time_series_dimension": false + "aggregatable": false } } } diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/field_caps/40_time_series.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/field_caps/40_time_series.yml index 7795c85b6e52d..837fcbbc3098e 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/field_caps/40_time_series.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/field_caps/40_time_series.yml @@ -119,7 +119,7 @@ setup: - match: {fields.non_tsdb_field.keyword.searchable: true} - match: {fields.non_tsdb_field.keyword.aggregatable: true} - - match: {fields.non_tsdb_field.keyword.time_series_dimension: false} + - is_false: fields.non_tsdb_field.keyword.time_series_dimension - is_false: fields.non_tsdb_field.keyword.time_series_metric - is_false: fields.non_tsdb_field.keyword.indices - is_false: fields.non_tsdb_field.keyword.non_searchable_indices @@ -134,7 +134,7 @@ setup: - is_false: fields.k8s\.pod\.uid.keyword.time_series_metric - is_false: fields.k8s\.pod\.uid.keyword.non_dimension_indices - - match: {fields.k8s\.pod\.name.keyword.time_series_dimension: false} + - is_false: fields.k8s\.pod\.name.keyword.time_series_dimension - is_false: fields.k8s\.pod\.name.keyword.time_series_metric - is_false: fields.k8s\.pod\.name.keyword.non_dimension_indices @@ -142,19 +142,19 @@ setup: - is_false: fields.k8s\.pod\.ip.ip.time_series_metric - is_false: fields.k8s\.pod\.ip.ip.non_dimension_indices - - match: {fields.k8s\.pod\.network\.tx.long.time_series_dimension: false} + - is_false: fields.k8s\.pod\.network\.tx.long.time_series_dimension - match: {fields.k8s\.pod\.network\.tx.long.time_series_metric: counter} - is_false: fields.k8s\.pod\.network\.tx.long.non_dimension_indices - - match: {fields.k8s\.pod\.network\.rx.integer.time_series_dimension: false} + - is_false: fields.k8s\.pod\.network\.rx.integer.time_series_dimension - match: {fields.k8s\.pod\.network\.rx.integer.time_series_metric: gauge} - is_false: fields.k8s\.pod\.network\.rx.integer.non_dimension_indices - - match: {fields.k8s\.pod\.network\.packets_dropped.long.time_series_dimension: false} + - is_false: fields.k8s\.pod\.network\.packets_dropped.long.time_series_dimension - match: {fields.k8s\.pod\.network\.packets_dropped.long.time_series_metric: gauge} - is_false: fields.k8s\.pod\.network\.packets_dropped.long.non_dimension_indices - - match: {fields.k8s\.pod\.network\.latency.double.time_series_dimension: false} + - is_false: fields.k8s\.pod\.network\.latency.double.time_series_dimension - match: {fields.k8s\.pod\.network\.latency.double.time_series_metric: gauge} - is_false: fields.k8s\.pod\.network\.latency.double.non_dimension_indices @@ -172,7 +172,7 @@ setup: - match: {fields.metricset.keyword.searchable: true} - match: {fields.metricset.keyword.aggregatable: true} - - match: {fields.metricset.keyword.time_series_dimension: false} + - is_false: fields.metricset.keyword.time_series_dimension - is_false: fields.metricset.keyword.time_series_metric - is_false: fields.metricset.keyword.indices - is_false: fields.metricset.keyword.non_searchable_indices @@ -181,7 +181,7 @@ setup: - match: {fields.non_tsdb_field.keyword.searchable: true} - match: {fields.non_tsdb_field.keyword.aggregatable: true} - - match: {fields.non_tsdb_field.keyword.time_series_dimension: false} + - is_false: fields.non_tsdb_field.keyword.time_series_dimension - is_false: fields.non_tsdb_field.keyword.time_series_metric - is_false: fields.non_tsdb_field.keyword.indices - is_false: fields.non_tsdb_field.keyword.non_searchable_indices @@ -196,7 +196,7 @@ setup: - is_false: fields.k8s\.pod\.uid.keyword.time_series_metric - is_false: fields.k8s\.pod\.uid.keyword.non_dimension_indices - - match: {fields.k8s\.pod\.name.keyword.time_series_dimension: false} + - is_false: fields.k8s\.pod\.name.keyword.time_series_dimension - is_false: fields.k8s\.pod\.name.keyword.time_series_metric - is_false: fields.k8s\.pod\.name.keyword.non_dimension_indices @@ -204,18 +204,18 @@ setup: - is_false: fields.k8s\.pod\.ip.ip.time_series_metric - is_false: fields.k8s\.pod\.ip.ip.non_dimension_indices - - match: {fields.k8s\.pod\.network\.tx.long.time_series_dimension: false} + - is_false: fields.k8s\.pod\.network\.tx.long.time_series_dimension - is_false: fields.k8s\.pod\.network\.tx.long.time_series_metric - is_false: fields.k8s\.pod\.network\.tx.long.non_dimension_indices - - match: {fields.k8s\.pod\.network\.rx.integer.time_series_dimension: false} + - is_false: fields.k8s\.pod\.network\.rx.integer.time_series_dimension - is_false: fields.k8s\.pod\.network\.rx.integer.time_series_metric - is_false: fields.k8s\.pod\.network\.rx.integer.non_dimension_indices - - match: {fields.k8s\.pod\.network\.packets_dropped.long.time_series_dimension: false} + - is_false: fields.k8s\.pod\.network\.packets_dropped.long.time_series_dimension - match: {fields.k8s\.pod\.network\.packets_dropped.long.time_series_metric: gauge} - is_false: fields.k8s\.pod\.network\.packets_dropped.long.non_dimension_indices - - match: {fields.k8s\.pod\.network\.latency.double.time_series_dimension: false} + - is_false: fields.k8s\.pod\.network\.latency.double.time_series_dimension - match: {fields.k8s\.pod\.network\.latency.double.time_series_metric: gauge} - is_false: fields.k8s\.pod\.network\.latency.double.non_dimension_indices diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java index 4b1da2d9138ce..b477781930238 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java @@ -124,14 +124,14 @@ public void testFieldAlias() { assertTrue(distance.containsKey("double")); assertEquals( - new FieldCapabilities("distance", "double", false, true, true, false, null, new String[] {"old_index"}, null, null, null, - Collections.emptyMap()), + new FieldCapabilities("distance", "double", false, true, true, new String[] {"old_index"}, null, null, + Collections.emptyMap()), distance.get("double")); assertTrue(distance.containsKey("text")); assertEquals( - new FieldCapabilities("distance", "text", false, true, false, false, null, new String[] {"new_index"}, null, null, null, - Collections.emptyMap()), + new FieldCapabilities("distance", "text", false, true, false, new String[] {"new_index"}, null, null, + Collections.emptyMap()), distance.get("text")); // Check the capabilities for the 'route_length_miles' alias. @@ -140,8 +140,7 @@ public void testFieldAlias() { assertTrue(routeLength.containsKey("double")); assertEquals( - new FieldCapabilities("route_length_miles", "double", false, true, true, false, null, null, null, null, null, - Collections.emptyMap()), + new FieldCapabilities("route_length_miles", "double", false, true, true, null, null, null, Collections.emptyMap()), routeLength.get("double")); } @@ -182,14 +181,14 @@ public void testWithUnmapped() { assertTrue(oldField.containsKey("long")); assertEquals( - new FieldCapabilities("old_field", "long", false, true, true, false, null, new String[] {"old_index"}, null, null, null, - Collections.emptyMap()), + new FieldCapabilities("old_field", "long", false, true, true, new String[] {"old_index"}, null, null, + Collections.emptyMap()), oldField.get("long")); assertTrue(oldField.containsKey("unmapped")); assertEquals( - new FieldCapabilities("old_field", "unmapped", false, false, false, false, null, new String[] {"new_index"}, null, null, null, - Collections.emptyMap()), + new FieldCapabilities("old_field", "unmapped", false, false, false, new String[] {"new_index"}, null, null, + Collections.emptyMap()), oldField.get("unmapped")); Map newField = response.getField("new_field"); @@ -197,8 +196,7 @@ public void testWithUnmapped() { assertTrue(newField.containsKey("long")); assertEquals( - new FieldCapabilities("new_field", "long", false, true, true, false, null, null, null, null, null, - Collections.emptyMap()), + new FieldCapabilities("new_field", "long", false, true, true, null, null, null, Collections.emptyMap()), newField.get("long")); } @@ -262,7 +260,7 @@ public void testMetadataFields() { assertTrue(idField.containsKey("_id")); assertEquals( - new FieldCapabilities("_id", "_id", true, true, false, false, null, null, null, null, null, Collections.emptyMap()), + new FieldCapabilities("_id", "_id", true, true, false, null, null, null, Collections.emptyMap()), idField.get("_id")); Map testField = response.getField("_test"); @@ -270,8 +268,7 @@ public void testMetadataFields() { assertTrue(testField.containsKey("keyword")); assertEquals( - new FieldCapabilities("_test", "keyword", true, true, true, false, null, null, null, null, null, - Collections.emptyMap()), + new FieldCapabilities("_test", "keyword", true, true, true, null, null, null, Collections.emptyMap()), testField.get("keyword")); } } diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java index b5f0d3748ecd5..d0d882951a125 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java @@ -52,6 +52,7 @@ public class FieldCapabilities implements Writeable, ToXContentObject { private static final ParseField NON_SEARCHABLE_INDICES_FIELD = new ParseField("non_searchable_indices"); private static final ParseField NON_AGGREGATABLE_INDICES_FIELD = new ParseField("non_aggregatable_indices"); private static final ParseField NON_DIMENSION_INDICES_FIELD = new ParseField("non_dimension_indices"); + private static final ParseField METRIC_CONFLICTS_INDICES_FIELD = new ParseField("mertric_conflicts_indices"); private static final ParseField META_FIELD = new ParseField("meta"); private final String name; @@ -66,6 +67,7 @@ public class FieldCapabilities implements Writeable, ToXContentObject { private final String[] nonSearchableIndices; private final String[] nonAggregatableIndices; private final String[] nonDimensionIndices; + private final String[] metricConflictsIndices; private final Map> meta; @@ -85,6 +87,7 @@ public class FieldCapabilities implements Writeable, ToXContentObject { * @param nonAggregatableIndices The list of indices where this field is not aggregatable, * or null if the field is aggregatable in all indices. * @param nonDimensionIndices The list of indices where this field is not a dimension + * @param metricConflictsIndices The list of indices where this field is has different metric types or not mark as a metric * @param meta Merged metadata across indices. */ public FieldCapabilities(String name, String type, @@ -97,6 +100,7 @@ public FieldCapabilities(String name, String type, String[] nonSearchableIndices, String[] nonAggregatableIndices, String[] nonDimensionIndices, + String[] metricConflictsIndices, Map> meta) { this.name = name; this.type = type; @@ -109,9 +113,48 @@ public FieldCapabilities(String name, String type, this.nonSearchableIndices = nonSearchableIndices; this.nonAggregatableIndices = nonAggregatableIndices; this.nonDimensionIndices = nonDimensionIndices; + this.metricConflictsIndices = metricConflictsIndices; this.meta = Objects.requireNonNull(meta); } + /** + * Constructor for non-timeseries field caps. Useful for testing + * @param name + * @param type + * @param isMetadataField + * @param isSearchable + * @param isAggregatable + * @param indices + * @param nonSearchableIndices + * @param nonAggregatableIndices + * @param meta + */ + public FieldCapabilities(String name, String type, + boolean isMetadataField, + boolean isSearchable, + boolean isAggregatable, + String[] indices, + String[] nonSearchableIndices, + String[] nonAggregatableIndices, + Map> meta) { + this( + name, + type, + isMetadataField, + isSearchable, + isAggregatable, + false, + null, + indices, + nonSearchableIndices, + nonAggregatableIndices, + null, + null, + meta + ); + + } + FieldCapabilities(StreamInput in) throws IOException { this.name = in.readString(); this.type = in.readString(); @@ -130,8 +173,10 @@ public FieldCapabilities(String name, String type, this.nonAggregatableIndices = in.readOptionalStringArray(); if (in.getVersion().onOrAfter(Version.V_8_0_0)) { this.nonDimensionIndices = in.readOptionalStringArray(); + this.metricConflictsIndices = in.readOptionalStringArray(); } else { this.nonDimensionIndices = null; + this.metricConflictsIndices = null; } meta = in.readMap(StreamInput::readString, i -> i.readSet(StreamInput::readString)); } @@ -152,6 +197,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalStringArray(nonAggregatableIndices); if (out.getVersion().onOrAfter(Version.V_8_0_0)) { out.writeOptionalStringArray(nonDimensionIndices); + out.writeOptionalStringArray(metricConflictsIndices); } out.writeMap(meta, StreamOutput::writeString, (o, set) -> o.writeCollection(set, StreamOutput::writeString)); } @@ -163,7 +209,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(IS_METADATA_FIELD.getPreferredName(), isMetadataField); builder.field(SEARCHABLE_FIELD.getPreferredName(), isSearchable); builder.field(AGGREGATABLE_FIELD.getPreferredName(), isAggregatable); - builder.field(TIME_SERIES_DIMENSION_FIELD.getPreferredName(), isDimension); + if (isDimension) { + builder.field(TIME_SERIES_DIMENSION_FIELD.getPreferredName(), isDimension); + } if (metricType != null) { builder.field(TIME_SERIES_METRIC_FIELD.getPreferredName(), metricType); } @@ -179,6 +227,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (nonDimensionIndices != null) { builder.field(NON_DIMENSION_INDICES_FIELD.getPreferredName(), nonDimensionIndices); } + if (metricConflictsIndices != null) { + builder.field(METRIC_CONFLICTS_INDICES_FIELD.getPreferredName(), metricConflictsIndices); + } if (meta.isEmpty() == false) { builder.startObject("meta"); List>> entries = new ArrayList<>(meta.entrySet()); @@ -214,7 +265,8 @@ public static FieldCapabilities fromXContent(String name, XContentParser parser) a[7] != null ? ((List) a[7]).toArray(new String[0]) : null, a[8] != null ? ((List) a[8]).toArray(new String[0]) : null, a[9] != null ? ((List) a[9]).toArray(new String[0]) : null, - a[10] != null ? ((Map>) a[10]) : Collections.emptyMap() + a[10] != null ? ((List) a[10]).toArray(new String[0]) : null, + a[11] != null ? ((Map>) a[11]) : Collections.emptyMap() ) ); @@ -317,6 +369,13 @@ public String[] nonDimensionIndices() { return nonDimensionIndices; } + /** + * The list of indices where this field has different dimension or metric flag + */ + public String[] metricConflictsIndices() { + return metricConflictsIndices; + } + /** * Return merged metadata across indices. */ @@ -394,7 +453,7 @@ void add( TimeSeriesParams.MetricType metricType, Map meta ) { - IndexCaps indexCaps = new IndexCaps(index, search, agg, isDimension, metricType != null); + IndexCaps indexCaps = new IndexCaps(index, search, agg, isDimension, metricType); indiceList.add(indexCaps); this.isSearchable &= search; this.isAggregatable &= agg; @@ -456,7 +515,6 @@ FieldCapabilities build(boolean withIndices) { nonAggregatableIndices = null; } - final String[] nonDimensionIndices; if (isDimension == false && indiceList.stream().anyMatch((caps) -> caps.isDimension)) { // Collect all indices that disagree on the dimension flag @@ -468,6 +526,16 @@ FieldCapabilities build(boolean withIndices) { nonDimensionIndices = null; } + final String[] metricConflictsIndices; + if (metricType != null && indiceList.stream().anyMatch((caps) -> caps.metricType != metricType)) { + // Collect all indices that disagree on the dimension flag + metricConflictsIndices = indiceList.stream() + .map(caps -> caps.name) + .toArray(String[]::new); + } else { + metricConflictsIndices = null; + } + final Function>, Set> entryValueFunction = Map.Entry::getValue; Map> immutableMeta = meta.entrySet().stream() .collect(Collectors.toUnmodifiableMap( @@ -484,6 +552,7 @@ FieldCapabilities build(boolean withIndices) { nonSearchableIndices, nonAggregatableIndices, nonDimensionIndices, + metricConflictsIndices, immutableMeta ); } @@ -493,15 +562,15 @@ private static class IndexCaps { final String name; final boolean isSearchable; final boolean isAggregatable; - final boolean isMetric; final boolean isDimension; + final TimeSeriesParams.MetricType metricType; - IndexCaps(String name, boolean isSearchable, boolean isAggregatable, boolean isDimension, boolean isMetric) { + IndexCaps(String name, boolean isSearchable, boolean isAggregatable, boolean isDimension, TimeSeriesParams.MetricType metricType) { this.name = name; this.isSearchable = isSearchable; this.isAggregatable = isAggregatable; - this.isMetric = isMetric; this.isDimension = isDimension; + this.metricType = metricType; } } } diff --git a/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesTests.java b/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesTests.java index 593f3b4d6cf67..039ccdfb5a4c8 100644 --- a/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesTests.java +++ b/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesTests.java @@ -189,6 +189,14 @@ static FieldCapabilities randomFieldCaps(String fieldName) { } } + String[] metricConflictsIndices = null; + if (randomBoolean()) { + metricConflictsIndices = new String[randomIntBetween(0, 5)]; + for (int i = 0; i < metricConflictsIndices.length; i++) { + metricConflictsIndices[i] = randomAlphaOfLengthBetween(5, 20); + } + } + Map> meta; switch (randomInt(2)) { case 0: @@ -214,6 +222,7 @@ static FieldCapabilities randomFieldCaps(String fieldName) { nonSearchableIndices, nonAggregatableIndices, nonDimensionIndices, + metricConflictsIndices, meta ); } @@ -231,8 +240,9 @@ protected FieldCapabilities mutateInstance(FieldCapabilities instance) { String[] nonSearchableIndices = instance.nonSearchableIndices(); String[] nonAggregatableIndices = instance.nonAggregatableIndices(); String[] nonDimensionIndices = instance.nonDimensionIndices(); + String[] metricConflictsIndices = instance.metricConflictsIndices(); Map> meta = instance.meta(); - switch (between(0, 11)) { + switch (between(0, 12)) { case 0: name += randomAlphaOfLengthBetween(1, 10); break; @@ -328,6 +338,21 @@ protected FieldCapabilities mutateInstance(FieldCapabilities instance) { } nonDimensionIndices = newTimeSeriesDimensionsConflictsIndices; break; + case 12: + String[] newMetricConflictsIndices; + int startMetricConflictsPos = 0; + if (metricConflictsIndices == null) { + newMetricConflictsIndices = new String[between(1, 10)]; + } else { + newMetricConflictsIndices = Arrays.copyOf(metricConflictsIndices, + metricConflictsIndices.length + between(1, 10)); + startMetricConflictsPos = metricConflictsIndices.length; + } + for (int i = startMetricConflictsPos; i < newMetricConflictsIndices.length; i++) { + newMetricConflictsIndices[i] = randomAlphaOfLengthBetween(5, 20); + } + metricConflictsIndices = newMetricConflictsIndices; + break; default: throw new AssertionError(); } @@ -343,6 +368,7 @@ protected FieldCapabilities mutateInstance(FieldCapabilities instance) { nonSearchableIndices, nonAggregatableIndices, nonDimensionIndices, + metricConflictsIndices, meta ); } diff --git a/server/src/test/java/org/elasticsearch/action/fieldcaps/MergedFieldCapabilitiesResponseTests.java b/server/src/test/java/org/elasticsearch/action/fieldcaps/MergedFieldCapabilitiesResponseTests.java index a00055d983b2a..d75089e4e5b74 100644 --- a/server/src/test/java/org/elasticsearch/action/fieldcaps/MergedFieldCapabilitiesResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/fieldcaps/MergedFieldCapabilitiesResponseTests.java @@ -125,7 +125,6 @@ public void testToXContent() throws IOException { " \"metadata_field\": false," + " \"searchable\": true," + " \"aggregatable\": false," + - " \"time_series_dimension\": false," + " \"time_series_metric\": \"counter\"," + " \"indices\": [\"index1\", \"index2\"]," + " \"non_aggregatable_indices\": [\"index1\"]," + @@ -137,8 +136,7 @@ public void testToXContent() throws IOException { " \"type\": \"text\"," + " \"metadata_field\": false," + " \"searchable\": true," + - " \"aggregatable\": false," + - " \"time_series_dimension\": false" + + " \"aggregatable\": false" + " }" + " }" + " }," + @@ -154,17 +152,17 @@ public void testToXContent() throws IOException { private static FieldCapabilitiesResponse createSimpleResponse() { Map titleCapabilities = new HashMap<>(); titleCapabilities.put("text", new FieldCapabilities("title", "text", false, true, false, false, null, - null, null, null, null, Collections.emptyMap())); + null, null, null, null, null, Collections.emptyMap())); Map ratingCapabilities = new HashMap<>(); ratingCapabilities.put("long", new FieldCapabilities("rating", "long", false, true, false, false, TimeSeriesParams.MetricType.counter, new String[]{"index1", "index2"}, null, new String[]{"index1"}, new String[]{"index4"}, - Collections.emptyMap() + null, Collections.emptyMap() )); ratingCapabilities.put("keyword", new FieldCapabilities("rating", "keyword", false, false, true, true, null, - new String[]{"index3", "index4"}, new String[]{"index4"}, null, null, + new String[]{"index3", "index4"}, new String[]{"index4"}, null, null, null, Collections.emptyMap() )); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/ClassificationTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/ClassificationTests.java index bf816148d489d..08b439dc8f928 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/ClassificationTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/ClassificationTests.java @@ -518,6 +518,6 @@ public Long getCardinality(String field) { } private static FieldCapabilities createFieldCapabilities(String field, String type) { - return new FieldCapabilities(field, type, false, true, true, false, null, null, null, null, null, Collections.emptyMap()); + return new FieldCapabilities(field, type, false, true, true, null, null, null, Collections.emptyMap()); } } diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/CancellationTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/CancellationTests.java index f25902858ae11..c4377d351822d 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/CancellationTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/CancellationTests.java @@ -106,11 +106,11 @@ public void onFailure(Exception e) { private Map> fields(String[] indices) { FieldCapabilities fooField = - new FieldCapabilities("foo", "integer", false, true, true, false, null, indices, null, null, null, emptyMap()); + new FieldCapabilities("foo", "integer", false, true, true, indices, null, null, emptyMap()); FieldCapabilities categoryField = - new FieldCapabilities("event.category", "keyword", false, true, true, false, null, indices, null, null, null, emptyMap()); + new FieldCapabilities("event.category", "keyword", false, true, true, indices, null, null, emptyMap()); FieldCapabilities timestampField = - new FieldCapabilities("@timestamp", "date", false, true, true, false, null, indices, null, null, null, emptyMap()); + new FieldCapabilities("@timestamp", "date", false, true, true, indices, null, null, emptyMap()); Map> fields = new HashMap<>(); fields.put(fooField.getName(), singletonMap(fooField.getName(), fooField)); fields.put(categoryField.getName(), singletonMap(categoryField.getName(), categoryField)); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/DestinationIndexTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/DestinationIndexTests.java index 9ec1bd5cca6ff..0fe48f85c18f2 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/DestinationIndexTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/DestinationIndexTests.java @@ -470,6 +470,6 @@ private static DataFrameAnalyticsConfig createConfig(DataFrameAnalysis analysis) } private static FieldCapabilities createFieldCapabilities(String field, String type) { - return new FieldCapabilities(field, type, false, true, true, false, null, null, null, null, null, Collections.emptyMap()); + return new FieldCapabilities(field, type, false, true, true, null, null, null, Collections.emptyMap()); } } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetectorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetectorTests.java index cbdcc9241e5c5..9fe27e3baf95d 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetectorTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetectorTests.java @@ -1331,7 +1331,7 @@ private MockFieldCapsResponseBuilder addField(String field, boolean isMetadataFi Map caps = new HashMap<>(); for (String type : types) { caps.put(type, new FieldCapabilities(field, type, - isMetadataField, true, isAggregatable, false, null, null, null, null, null, Collections.emptyMap())); + isMetadataField, true, isAggregatable, null, null, null, Collections.emptyMap())); } fieldCaps.put(field, caps); return this; diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/CancellationTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/CancellationTests.java index ef731a60e6ecc..cc8b5ff137fb3 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/CancellationTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/CancellationTests.java @@ -87,11 +87,11 @@ public void onFailure(Exception e) { private Map> fields(String[] indices) { FieldCapabilities fooField = - new FieldCapabilities("foo", "integer", false, true, true, false, null, indices, null, null, null, emptyMap()); + new FieldCapabilities("foo", "integer", false, true, true, indices, null, null, emptyMap()); FieldCapabilities categoryField = - new FieldCapabilities("event.category", "keyword", false, true, true, false, null, indices, null, null, null, emptyMap()); + new FieldCapabilities("event.category", "keyword", false, true, true, indices, null, null, emptyMap()); FieldCapabilities timestampField = - new FieldCapabilities("@timestamp", "date", false, true, true, false, null, indices, null, null, null, emptyMap()); + new FieldCapabilities("@timestamp", "date", false, true, true, indices, null, null, emptyMap()); Map> fields = new HashMap<>(); fields.put(fooField.getName(), singletonMap(fooField.getName(), fooField)); fields.put(categoryField.getName(), singletonMap(categoryField.getName(), categoryField)); diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java index 4533e8e08eccd..8eb425bb72007 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java @@ -247,11 +247,11 @@ public void testMergeIncompatibleCapabilitiesOfObjectFields() throws Exception { addFieldCaps(fieldCaps, fieldName + ".keyword", "keyword", true, true); Map multi = new HashMap<>(); - multi.put("long", new FieldCapabilities(fieldName, "long", false, true, true, false, null, new String[] { "one-index" }, null, null, - null, Collections.emptyMap() + multi.put("long", new FieldCapabilities(fieldName, "long", false, true, true, new String[] { "one-index" }, null, null, + Collections.emptyMap() )); - multi.put("text", new FieldCapabilities(fieldName, "text", false, true, false, false, null, new String[] { "another-index" }, null, - null, null, Collections.emptyMap() + multi.put("text", new FieldCapabilities(fieldName, "text", false, true, false, new String[] { "another-index" }, null, null, + Collections.emptyMap() )); fieldCaps.put(fieldName, multi); @@ -323,7 +323,7 @@ public void testMultipleCompatibleIndicesWithDifferentFields() { public void testIndexWithNoMapping() { Map> versionFC = singletonMap("_version", singletonMap("_index", new FieldCapabilities("_version", "_version", true, false, false, - false, null, null, null, null, null, Collections.emptyMap()))); + null, null, null, Collections.emptyMap()))); assertTrue(mergedMappings("*", new String[] { "empty" }, versionFC).isValid()); } @@ -398,7 +398,7 @@ private static class UpdateableFieldCapabilities extends FieldCapabilities { List nonAggregatableIndices = new ArrayList<>(); UpdateableFieldCapabilities(String name, String type, boolean isSearchable, boolean isAggregatable) { - super(name, type, false, isSearchable, isAggregatable, false, null, null, null, null, null, Collections.emptyMap()); + super(name, type, false, isSearchable, isAggregatable, null, null, null, Collections.emptyMap()); } @Override @@ -445,7 +445,7 @@ private void addFieldCaps(Map> fieldCaps, boolean isAggregatable) { Map cap = new HashMap<>(); cap.put(type, new FieldCapabilities(name, type, isMetadataField, - isSearchable, isAggregatable, false, null, null, null, null, null, Collections.emptyMap())); + isSearchable, isAggregatable, null, null, null, Collections.emptyMap())); fieldCaps.put(name, cap); } diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/common/DocumentConversionUtilsTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/common/DocumentConversionUtilsTests.java index fcb474a6abc5a..eb583f2f3ce81 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/common/DocumentConversionUtilsTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/common/DocumentConversionUtilsTests.java @@ -117,7 +117,6 @@ public void testExtractFieldMappings() { private static FieldCapabilities createFieldCapabilities(String name, String type) { return new FieldCapabilities( - name, type, false, true, true, false, null, Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY, null, - Collections.emptyMap()); + name, type, false, true, true, Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY, Collections.emptyMap()); } } diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/AggregationSchemaAndResultTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/AggregationSchemaAndResultTests.java index acdbff18fbe78..4e9b2d5f69c3c 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/AggregationSchemaAndResultTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/AggregationSchemaAndResultTests.java @@ -90,7 +90,7 @@ protected void field, Collections.singletonMap( type, - new FieldCapabilities(field, type, false, true, true, false, null, null, null, null, null, emptyMap()) + new FieldCapabilities(field, type, false, true, true, null, null, null, emptyMap()) ) ); } diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/SchemaUtilTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/SchemaUtilTests.java index 646959046154c..d406393b8b15a 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/SchemaUtilTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/SchemaUtilTests.java @@ -217,12 +217,9 @@ private static FieldCapabilities createFieldCapabilities(String name, String typ false, true, true, - false, - null, Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY, - null, Collections.emptyMap() ); } From 9ef464a0c858985292daf74d62451cfe115167ff Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Mon, 11 Oct 2021 10:34:16 -1000 Subject: [PATCH 4/8] Fix javadoc --- .../action/fieldcaps/FieldCapabilities.java | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java index b886daff2925f..b2632be0c2744 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java @@ -119,15 +119,19 @@ public FieldCapabilities(String name, String type, /** * Constructor for non-timeseries field caps. Useful for testing - * @param name - * @param type - * @param isMetadataField - * @param isSearchable - * @param isAggregatable - * @param indices - * @param nonSearchableIndices - * @param nonAggregatableIndices - * @param meta + * Constructor for a set of indices. + * @param name The name of the field + * @param type The type associated with the field. + * @param isMetadataField Whether this field is a metadata field. + * @param isSearchable Whether this field is indexed for search. + * @param isAggregatable Whether this field can be aggregated on. + * @param indices The list of indices where this field name is defined as {@code type}, + * or null if all indices have the same {@code type} for the field. + * @param nonSearchableIndices The list of indices where this field is not searchable, + * or null if the field is searchable in all indices. + * @param nonAggregatableIndices The list of indices where this field is not aggregatable, + * or null if the field is aggregatable in all indices. + * @param meta Merged metadata across indices. */ public FieldCapabilities(String name, String type, boolean isMetadataField, From ffdb38236ba665922cc5d9e9ba204f15ca7f3beb Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Mon, 11 Oct 2021 11:43:25 -1000 Subject: [PATCH 5/8] Fix serialization --- .../elasticsearch/action/fieldcaps/FieldCapabilities.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java index b2632be0c2744..7fe45f332eaed 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java @@ -285,11 +285,12 @@ public static FieldCapabilities fromXContent(String name, XContentParser parser) PARSER.declareStringArray(ConstructingObjectParser.optionalConstructorArg(), NON_SEARCHABLE_INDICES_FIELD); // 7 PARSER.declareStringArray(ConstructingObjectParser.optionalConstructorArg(), NON_AGGREGATABLE_INDICES_FIELD); // 8 PARSER.declareStringArray(ConstructingObjectParser.optionalConstructorArg(), NON_DIMENSION_INDICES_FIELD); // 9 + PARSER.declareStringArray(ConstructingObjectParser.optionalConstructorArg(), METRIC_CONFLICTS_INDICES_FIELD); // 10 PARSER.declareObject( ConstructingObjectParser.optionalConstructorArg(), (parser, context) -> parser.map(HashMap::new, p -> Set.copyOf(p.list())), META_FIELD - ); // 10 + ); // 11 } /** @@ -403,6 +404,7 @@ public boolean equals(Object o) { Arrays.equals(nonSearchableIndices, that.nonSearchableIndices) && Arrays.equals(nonAggregatableIndices, that.nonAggregatableIndices) && Arrays.equals(nonDimensionIndices, that.nonDimensionIndices) && + Arrays.equals(metricConflictsIndices, that.metricConflictsIndices) && Objects.equals(meta, that.meta); } @@ -413,6 +415,7 @@ public int hashCode() { result = 31 * result + Arrays.hashCode(nonSearchableIndices); result = 31 * result + Arrays.hashCode(nonAggregatableIndices); result = 31 * result + Arrays.hashCode(nonDimensionIndices); + result = 31 * result + Arrays.hashCode(metricConflictsIndices); return result; } From 91afda583d3468319d84217a5ab5d52ed5b3aa58 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Tue, 12 Oct 2021 10:00:07 -1000 Subject: [PATCH 6/8] Fix mertric_conflicts_indices generation after refactoring and add tests for it --- .../test/field_caps/40_time_series.yml | 10 +++++ .../search/fieldcaps/FieldCapabilitiesIT.java | 41 ++++++++++++++++++- .../action/fieldcaps/FieldCapabilities.java | 2 +- 3 files changed, 50 insertions(+), 3 deletions(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/field_caps/40_time_series.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/field_caps/40_time_series.yml index 837fcbbc3098e..79606d7957c6c 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/field_caps/40_time_series.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/field_caps/40_time_series.yml @@ -178,6 +178,7 @@ setup: - is_false: fields.metricset.keyword.non_searchable_indices - is_false: fields.metricset.keyword.non_aggregatable_indices - match: {fields.metricset.keyword.non_dimension_indices: ["tsdb_index2"]} + - is_false: fields.metricset.keyword.mertric_conflicts_indices - match: {fields.non_tsdb_field.keyword.searchable: true} - match: {fields.non_tsdb_field.keyword.aggregatable: true} @@ -187,35 +188,44 @@ setup: - is_false: fields.non_tsdb_field.keyword.non_searchable_indices - is_false: fields.non_tsdb_field.keyword.non_aggregatable_indices - is_false: fields.non_tsdb_field.keyword.non_dimension_indices + - is_false: fields.non_tsdb_field.keyword.mertric_conflicts_indices - match: {fields.k8s\.pod\.availability_zone.short.time_series_dimension: true} - is_false: fields.k8s\.pod\.availability_zone.short.time_series_metric - is_false: fields.k8s\.pod\.availability_zone.short.non_dimension_indices + - is_false: fields.k8s\.pod\.availability_zone.short.mertric_conflicts_indices - match: {fields.k8s\.pod\.uid.keyword.time_series_dimension: true} - is_false: fields.k8s\.pod\.uid.keyword.time_series_metric - is_false: fields.k8s\.pod\.uid.keyword.non_dimension_indices + - is_false: fields.k8s\.pod\.uid.keyword.mertric_conflicts_indices - is_false: fields.k8s\.pod\.name.keyword.time_series_dimension - is_false: fields.k8s\.pod\.name.keyword.time_series_metric - is_false: fields.k8s\.pod\.name.keyword.non_dimension_indices + - is_false: fields.k8s\.pod\.name.keyword.mertric_conflicts_indices - match: {fields.k8s\.pod\.ip.ip.time_series_dimension: true} - is_false: fields.k8s\.pod\.ip.ip.time_series_metric - is_false: fields.k8s\.pod\.ip.ip.non_dimension_indices + - is_false: fields.k8s\.pod\.ip.ip.mertric_conflicts_indices - is_false: fields.k8s\.pod\.network\.tx.long.time_series_dimension - is_false: fields.k8s\.pod\.network\.tx.long.time_series_metric - is_false: fields.k8s\.pod\.network\.tx.long.non_dimension_indices + - match: {fields.k8s\.pod\.network\.tx.long.mertric_conflicts_indices: ["tsdb_index1", "tsdb_index2"]} - is_false: fields.k8s\.pod\.network\.rx.integer.time_series_dimension - is_false: fields.k8s\.pod\.network\.rx.integer.time_series_metric - is_false: fields.k8s\.pod\.network\.rx.integer.non_dimension_indices + - match: {fields.k8s\.pod\.network\.rx.integer.mertric_conflicts_indices: ["tsdb_index1", "tsdb_index2"]} - is_false: fields.k8s\.pod\.network\.packets_dropped.long.time_series_dimension - match: {fields.k8s\.pod\.network\.packets_dropped.long.time_series_metric: gauge} - is_false: fields.k8s\.pod\.network\.packets_dropped.long.non_dimension_indices + - is_false: fields.k8s\.pod\.network\.packets_dropped.long.mertric_conflicts_indices - is_false: fields.k8s\.pod\.network\.latency.double.time_series_dimension - match: {fields.k8s\.pod\.network\.latency.double.time_series_metric: gauge} - is_false: fields.k8s\.pod\.network\.latency.double.non_dimension_indices + - is_false: fields.k8s\.pod\.network\.latency.double.mertric_conflicts_indices diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java index c614f5f50f28e..766850555d924 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java @@ -15,11 +15,10 @@ import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.xcontent.XContentBuilder; -import org.elasticsearch.xcontent.XContentFactory; import org.elasticsearch.index.mapper.DocumentParserContext; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MetadataFieldMapper; +import org.elasticsearch.index.mapper.TimeSeriesParams; import org.elasticsearch.index.query.AbstractQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; @@ -30,6 +29,8 @@ import org.elasticsearch.plugins.SearchPlugin; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.transport.RemoteTransportException; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentFactory; import org.junit.Before; import java.io.IOException; @@ -45,8 +46,10 @@ import static java.util.Collections.singletonList; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.hamcrest.Matchers.array; import static org.hamcrest.Matchers.arrayContainingInAnyOrder; import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.equalTo; public class FieldCapabilitiesIT extends ESIntegTestCase { @@ -69,6 +72,14 @@ public void setUp() throws Exception { .startObject("playlist") .field("type", "text") .endObject() + .startObject("some_dimension") + .field("type", "keyword") + .field("time_series_dimension", true) + .endObject() + .startObject("some_metric") + .field("type", "long") + .field("time_series_metric", TimeSeriesParams.MetricType.counter) + .endObject() .startObject("secret_soundtrack") .field("type", "alias") .field("path", "playlist") @@ -98,6 +109,13 @@ public void setUp() throws Exception { .startObject("new_field") .field("type", "long") .endObject() + .startObject("some_dimension") + .field("type", "keyword") + .endObject() + .startObject("some_metric") + .field("type", "long") + .field("time_series_metric", TimeSeriesParams.MetricType.gauge) + .endObject() .endObject() .endObject() .endObject(); @@ -285,6 +303,25 @@ public void testWithRunntimeMappings() throws InterruptedException { assertTrue(runtimeField.get("keyword").isAggregatable()); } + public void testFieldMetricsAndDimensions() { + FieldCapabilitiesResponse response = client().prepareFieldCaps("old_index").setFields("some_dimension", "some_metric").get(); + assertIndices(response, "old_index"); + assertEquals(2, response.get().size()); + assertTrue(response.get().containsKey("some_dimension")); + assertTrue(response.get().get("some_dimension").get("keyword").isDimension()); + assertNull(response.get().get("some_dimension").get("keyword").nonDimensionIndices()); + assertTrue(response.get().containsKey("some_metric")); + assertEquals(TimeSeriesParams.MetricType.counter, response.get().get("some_metric").get("long").getMetricType()); + assertNull(response.get().get("some_metric").get("long").metricConflictsIndices()); + + response = client().prepareFieldCaps("old_index", "new_index").setFields("some_dimension", "some_metric").get(); + assertIndices(response, "old_index", "new_index"); + assertEquals(2, response.get().size()); + assertTrue(response.get().containsKey("some_dimension")); + assertFalse(response.get().get("some_dimension").get("keyword").isDimension()); + assertThat(response.get().get("some_dimension").get("keyword").nonDimensionIndices(), array(equalTo("new_index"))); + } + public void testFailures() throws InterruptedException { // in addition to the existing "old_index" and "new_index", create two where the test query throws an error on rewrite assertAcked(prepareCreate("index1-error")); diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java index 7fe45f332eaed..942daddc5b7db 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java @@ -534,7 +534,7 @@ FieldCapabilities build(boolean withIndices) { } final String[] metricConflictsIndices; - if (metricType != null && indiceList.stream().anyMatch((caps) -> caps.metricType != metricType)) { + if (indiceList.stream().anyMatch((caps) -> caps.metricType != metricType)) { // Collect all indices that disagree on the dimension flag metricConflictsIndices = indiceList.stream() .map(caps -> caps.name) From 155e64fb3191b2b7051022b5ba54427b2d9c4b5d Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Wed, 13 Oct 2021 07:45:10 -1000 Subject: [PATCH 7/8] Address the second round of review comments --- docs/reference/search/field-caps.asciidoc | 5 +++-- .../action/fieldcaps/FieldCapabilities.java | 9 ++++++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/docs/reference/search/field-caps.asciidoc b/docs/reference/search/field-caps.asciidoc index 2b4c89edf22ad..981c726ce72e0 100644 --- a/docs/reference/search/field-caps.asciidoc +++ b/docs/reference/search/field-caps.asciidoc @@ -114,7 +114,7 @@ field types are all described as the `keyword` type family. `time_series_dimension`:: Whether this field is used as a time series dimension. -`time_series_metrics`:: +`time_series_metric`:: Contains metric type if this fields is used as a time series metrics, null if the field is not used as metric. `indices`:: @@ -134,7 +134,8 @@ field types are all described as the `keyword` type family. indices have the same definition for the field. `metric_conflicts_indices`:: - The list of indices where this field is present but don't have the same metrics type. + The list of indices where this field is present if these indices don't have the same `time_series_metric` value for + this field. `meta`:: Merged metadata across all indices as a map of string keys to arrays of values. diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java index 942daddc5b7db..f0a7532dc80ac 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java @@ -466,7 +466,8 @@ void add( this.isAggregatable &= agg; this.isMetadataField |= isMetadataField; this.isDimension &= isDimension; - // If we have discrepancy in metric types - we ignore it + // If we have discrepancy in metric types or in some indices this field is not marked as a metric field - we will + // treat is a non-metric field and report this discrepancy in metricConflictsIndices if (this.mertricTypeIsSet) { if (this.metricType != metricType) { this.metricType = null; @@ -524,7 +525,7 @@ FieldCapabilities build(boolean withIndices) { final String[] nonDimensionIndices; if (isDimension == false && indiceList.stream().anyMatch((caps) -> caps.isDimension)) { - // Collect all indices that disagree on the dimension flag + // Collect all indices that have dimension == false if this field is marked as a dimension in at least one index nonDimensionIndices = indiceList.stream() .filter((caps) -> caps.isDimension == false) .map(caps -> caps.name) @@ -535,7 +536,9 @@ FieldCapabilities build(boolean withIndices) { final String[] metricConflictsIndices; if (indiceList.stream().anyMatch((caps) -> caps.metricType != metricType)) { - // Collect all indices that disagree on the dimension flag + // Collect all indices that have this field. If it is marked differently in different indices, we cannot really + // make a decisions which index is "right" and which index is "wrong" so collecting all indices where this field + // is present is probably the only sensible thing to do here metricConflictsIndices = indiceList.stream() .map(caps -> caps.name) .toArray(String[]::new); From 22abdeb624cef0f6806b4de23d4eee632f4514d0 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Wed, 13 Oct 2021 08:08:29 -1000 Subject: [PATCH 8/8] Improve docs --- docs/reference/search/field-caps.asciidoc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/reference/search/field-caps.asciidoc b/docs/reference/search/field-caps.asciidoc index 981c726ce72e0..24e25e4057ca3 100644 --- a/docs/reference/search/field-caps.asciidoc +++ b/docs/reference/search/field-caps.asciidoc @@ -115,7 +115,7 @@ field types are all described as the `keyword` type family. Whether this field is used as a time series dimension. `time_series_metric`:: - Contains metric type if this fields is used as a time series metrics, null if the field is not used as metric. + Contains metric type if this fields is used as a time series metrics, absent if the field is not used as metric. `indices`:: The list of indices where this field has the same type family, or null if all indices @@ -130,8 +130,8 @@ field types are all described as the `keyword` type family. indices have the same definition for the field. `non_dimension_indices`:: - The list of indices where this field is not marked as a dimension field, or null if all - indices have the same definition for the field. + If this list is present in response then some indices have the field marked as a dimension and other indices, the + ones in this list, do not. `metric_conflicts_indices`:: The list of indices where this field is present if these indices don't have the same `time_series_metric` value for