diff --git a/docs/reference/search/field-caps.asciidoc b/docs/reference/search/field-caps.asciidoc index 80ee6bfc24788..24e25e4057ca3 100644 --- a/docs/reference/search/field-caps.asciidoc +++ b/docs/reference/search/field-caps.asciidoc @@ -111,6 +111,12 @@ field types are all described as the `keyword` type family. `aggregatable`:: Whether this field can be aggregated on all indices. +`time_series_dimension`:: + Whether this field is used as a time series dimension. + +`time_series_metric`:: + Contains metric type if this fields is used as a time series metrics, absent if the field is not used as metric. + `indices`:: The list of indices where this field has the same type family, or null if all indices have the same type family for the field. @@ -123,6 +129,14 @@ field types are all described as the `keyword` type family. The list of indices where this field is not aggregatable, or null if all indices have the same definition for the field. +`non_dimension_indices`:: + If this list is present in response then some indices have the field marked as a dimension and other indices, the + ones in this list, do not. + +`metric_conflicts_indices`:: + The list of indices where this field is present if these indices don't have the same `time_series_metric` value for + this field. + `meta`:: Merged metadata across all indices as a map of string keys to arrays of values. A value length of 1 indicates that all indices had the same value for this key, @@ -179,7 +193,6 @@ The API returns the following response: "metadata_field": false, "searchable": true, "aggregatable": false - } } } diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/field_caps/40_time_series.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/field_caps/40_time_series.yml new file mode 100644 index 0000000000000..79606d7957c6c --- /dev/null +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/field_caps/40_time_series.yml @@ -0,0 +1,231 @@ +--- +setup: + - skip: + version: " - 7.99.99" + reason: introduced in 8.0.0 + + - do: + indices.create: + index: tsdb_index1 + body: + settings: + index: + number_of_replicas: 0 + number_of_shards: 2 + mappings: + properties: + "@timestamp": + type: date + metricset: + type: keyword + time_series_dimension: true + non_tsdb_field: + type: keyword + k8s: + properties: + pod: + properties: + availability_zone: + type: short + time_series_dimension: true + uid: + type: keyword + time_series_dimension: true + name: + type: keyword + ip: + type: ip + time_series_dimension: true + network: + properties: + tx: + type: long + time_series_metric: counter + rx: + type: integer + time_series_metric: gauge + packets_dropped: + type: long + time_series_metric: gauge + latency: + type: double + time_series_metric: gauge + + - do: + indices.create: + index: tsdb_index2 + body: + settings: + index: + number_of_replicas: 0 + number_of_shards: 2 + mappings: + properties: + "@timestamp": + type: date + metricset: + type: keyword + non_tsdb_field: + type: keyword + k8s: + properties: + pod: + properties: + availability_zone: + type: short + time_series_dimension: true + uid: + type: keyword + time_series_dimension: true + name: + type: keyword + ip: + type: ip + time_series_dimension: true + network: + properties: + tx: + type: long + time_series_metric: gauge + rx: + type: integer + packets_dropped: + type: long + time_series_metric: gauge + latency: + type: double + time_series_metric: gauge + +--- +"Get simple time series field caps": + + - skip: + version: " - 7.99.99" + reason: introduced in 8.0.0 + + - do: + field_caps: + index: 'tsdb_index1' + fields: [ "metricset", "non_tsdb_field", "k8s.pod.*" ] + + - match: {fields.metricset.keyword.searchable: true} + - match: {fields.metricset.keyword.aggregatable: true} + - match: {fields.metricset.keyword.time_series_dimension: true} + - is_false: fields.metricset.keyword.time_series_metric + - is_false: fields.metricset.keyword.indices + - is_false: fields.metricset.keyword.non_searchable_indices + - is_false: fields.metricset.keyword.non_aggregatable_indices + - is_false: fields.metricset.keyword.non_dimension_indices + + - match: {fields.non_tsdb_field.keyword.searchable: true} + - match: {fields.non_tsdb_field.keyword.aggregatable: true} + - is_false: fields.non_tsdb_field.keyword.time_series_dimension + - is_false: fields.non_tsdb_field.keyword.time_series_metric + - is_false: fields.non_tsdb_field.keyword.indices + - is_false: fields.non_tsdb_field.keyword.non_searchable_indices + - is_false: fields.non_tsdb_field.keyword.non_aggregatable_indices + - is_false: fields.non_tsdb_field.keyword.non_dimension_indices + + - match: {fields.k8s\.pod\.availability_zone.short.time_series_dimension: true} + - is_false: fields.k8s\.pod\.availability_zone.short.time_series_metric + - is_false: fields.k8s\.pod\.availability_zone.short.non_dimension_indices + + - match: {fields.k8s\.pod\.uid.keyword.time_series_dimension: true} + - is_false: fields.k8s\.pod\.uid.keyword.time_series_metric + - is_false: fields.k8s\.pod\.uid.keyword.non_dimension_indices + + - is_false: fields.k8s\.pod\.name.keyword.time_series_dimension + - is_false: fields.k8s\.pod\.name.keyword.time_series_metric + - is_false: fields.k8s\.pod\.name.keyword.non_dimension_indices + + - match: {fields.k8s\.pod\.ip.ip.time_series_dimension: true} + - is_false: fields.k8s\.pod\.ip.ip.time_series_metric + - is_false: fields.k8s\.pod\.ip.ip.non_dimension_indices + + - is_false: fields.k8s\.pod\.network\.tx.long.time_series_dimension + - match: {fields.k8s\.pod\.network\.tx.long.time_series_metric: counter} + - is_false: fields.k8s\.pod\.network\.tx.long.non_dimension_indices + + - is_false: fields.k8s\.pod\.network\.rx.integer.time_series_dimension + - match: {fields.k8s\.pod\.network\.rx.integer.time_series_metric: gauge} + - is_false: fields.k8s\.pod\.network\.rx.integer.non_dimension_indices + + - is_false: fields.k8s\.pod\.network\.packets_dropped.long.time_series_dimension + - match: {fields.k8s\.pod\.network\.packets_dropped.long.time_series_metric: gauge} + - is_false: fields.k8s\.pod\.network\.packets_dropped.long.non_dimension_indices + + - is_false: fields.k8s\.pod\.network\.latency.double.time_series_dimension + - match: {fields.k8s\.pod\.network\.latency.double.time_series_metric: gauge} + - is_false: fields.k8s\.pod\.network\.latency.double.non_dimension_indices + +--- +"Get time series field caps with conflicts": + + - skip: + version: " - 7.99.99" + reason: introduced in 8.0.0 + + - do: + field_caps: + index: tsdb_index1,tsdb_index2 + fields: [ "metricset", "non_tsdb_field", "k8s.pod.*" ] + + - match: {fields.metricset.keyword.searchable: true} + - match: {fields.metricset.keyword.aggregatable: true} + - is_false: fields.metricset.keyword.time_series_dimension + - is_false: fields.metricset.keyword.time_series_metric + - is_false: fields.metricset.keyword.indices + - is_false: fields.metricset.keyword.non_searchable_indices + - is_false: fields.metricset.keyword.non_aggregatable_indices + - match: {fields.metricset.keyword.non_dimension_indices: ["tsdb_index2"]} + - is_false: fields.metricset.keyword.mertric_conflicts_indices + + - match: {fields.non_tsdb_field.keyword.searchable: true} + - match: {fields.non_tsdb_field.keyword.aggregatable: true} + - is_false: fields.non_tsdb_field.keyword.time_series_dimension + - is_false: fields.non_tsdb_field.keyword.time_series_metric + - is_false: fields.non_tsdb_field.keyword.indices + - is_false: fields.non_tsdb_field.keyword.non_searchable_indices + - is_false: fields.non_tsdb_field.keyword.non_aggregatable_indices + - is_false: fields.non_tsdb_field.keyword.non_dimension_indices + - is_false: fields.non_tsdb_field.keyword.mertric_conflicts_indices + + - match: {fields.k8s\.pod\.availability_zone.short.time_series_dimension: true} + - is_false: fields.k8s\.pod\.availability_zone.short.time_series_metric + - is_false: fields.k8s\.pod\.availability_zone.short.non_dimension_indices + - is_false: fields.k8s\.pod\.availability_zone.short.mertric_conflicts_indices + + - match: {fields.k8s\.pod\.uid.keyword.time_series_dimension: true} + - is_false: fields.k8s\.pod\.uid.keyword.time_series_metric + - is_false: fields.k8s\.pod\.uid.keyword.non_dimension_indices + - is_false: fields.k8s\.pod\.uid.keyword.mertric_conflicts_indices + + - is_false: fields.k8s\.pod\.name.keyword.time_series_dimension + - is_false: fields.k8s\.pod\.name.keyword.time_series_metric + - is_false: fields.k8s\.pod\.name.keyword.non_dimension_indices + - is_false: fields.k8s\.pod\.name.keyword.mertric_conflicts_indices + + - match: {fields.k8s\.pod\.ip.ip.time_series_dimension: true} + - is_false: fields.k8s\.pod\.ip.ip.time_series_metric + - is_false: fields.k8s\.pod\.ip.ip.non_dimension_indices + - is_false: fields.k8s\.pod\.ip.ip.mertric_conflicts_indices + + - is_false: fields.k8s\.pod\.network\.tx.long.time_series_dimension + - is_false: fields.k8s\.pod\.network\.tx.long.time_series_metric + - is_false: fields.k8s\.pod\.network\.tx.long.non_dimension_indices + - match: {fields.k8s\.pod\.network\.tx.long.mertric_conflicts_indices: ["tsdb_index1", "tsdb_index2"]} + + - is_false: fields.k8s\.pod\.network\.rx.integer.time_series_dimension + - is_false: fields.k8s\.pod\.network\.rx.integer.time_series_metric + - is_false: fields.k8s\.pod\.network\.rx.integer.non_dimension_indices + - match: {fields.k8s\.pod\.network\.rx.integer.mertric_conflicts_indices: ["tsdb_index1", "tsdb_index2"]} + + - is_false: fields.k8s\.pod\.network\.packets_dropped.long.time_series_dimension + - match: {fields.k8s\.pod\.network\.packets_dropped.long.time_series_metric: gauge} + - is_false: fields.k8s\.pod\.network\.packets_dropped.long.non_dimension_indices + - is_false: fields.k8s\.pod\.network\.packets_dropped.long.mertric_conflicts_indices + + - is_false: fields.k8s\.pod\.network\.latency.double.time_series_dimension + - match: {fields.k8s\.pod\.network\.latency.double.time_series_metric: gauge} + - is_false: fields.k8s\.pod\.network\.latency.double.non_dimension_indices + - is_false: fields.k8s\.pod\.network\.latency.double.mertric_conflicts_indices diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java index c614f5f50f28e..766850555d924 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java @@ -15,11 +15,10 @@ import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.xcontent.XContentBuilder; -import org.elasticsearch.xcontent.XContentFactory; import org.elasticsearch.index.mapper.DocumentParserContext; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MetadataFieldMapper; +import org.elasticsearch.index.mapper.TimeSeriesParams; import org.elasticsearch.index.query.AbstractQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; @@ -30,6 +29,8 @@ import org.elasticsearch.plugins.SearchPlugin; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.transport.RemoteTransportException; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentFactory; import org.junit.Before; import java.io.IOException; @@ -45,8 +46,10 @@ import static java.util.Collections.singletonList; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.hamcrest.Matchers.array; import static org.hamcrest.Matchers.arrayContainingInAnyOrder; import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.equalTo; public class FieldCapabilitiesIT extends ESIntegTestCase { @@ -69,6 +72,14 @@ public void setUp() throws Exception { .startObject("playlist") .field("type", "text") .endObject() + .startObject("some_dimension") + .field("type", "keyword") + .field("time_series_dimension", true) + .endObject() + .startObject("some_metric") + .field("type", "long") + .field("time_series_metric", TimeSeriesParams.MetricType.counter) + .endObject() .startObject("secret_soundtrack") .field("type", "alias") .field("path", "playlist") @@ -98,6 +109,13 @@ public void setUp() throws Exception { .startObject("new_field") .field("type", "long") .endObject() + .startObject("some_dimension") + .field("type", "keyword") + .endObject() + .startObject("some_metric") + .field("type", "long") + .field("time_series_metric", TimeSeriesParams.MetricType.gauge) + .endObject() .endObject() .endObject() .endObject(); @@ -285,6 +303,25 @@ public void testWithRunntimeMappings() throws InterruptedException { assertTrue(runtimeField.get("keyword").isAggregatable()); } + public void testFieldMetricsAndDimensions() { + FieldCapabilitiesResponse response = client().prepareFieldCaps("old_index").setFields("some_dimension", "some_metric").get(); + assertIndices(response, "old_index"); + assertEquals(2, response.get().size()); + assertTrue(response.get().containsKey("some_dimension")); + assertTrue(response.get().get("some_dimension").get("keyword").isDimension()); + assertNull(response.get().get("some_dimension").get("keyword").nonDimensionIndices()); + assertTrue(response.get().containsKey("some_metric")); + assertEquals(TimeSeriesParams.MetricType.counter, response.get().get("some_metric").get("long").getMetricType()); + assertNull(response.get().get("some_metric").get("long").metricConflictsIndices()); + + response = client().prepareFieldCaps("old_index", "new_index").setFields("some_dimension", "some_metric").get(); + assertIndices(response, "old_index", "new_index"); + assertEquals(2, response.get().size()); + assertTrue(response.get().containsKey("some_dimension")); + assertFalse(response.get().get("some_dimension").get("keyword").isDimension()); + assertThat(response.get().get("some_dimension").get("keyword").nonDimensionIndices(), array(equalTo("new_index"))); + } + public void testFailures() throws InterruptedException { // in addition to the existing "old_index" and "new_index", create two where the test query throws an error on rewrite assertAcked(prepareCreate("index1-error")); diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java index 2d0f5e395ba1c..f0a7532dc80ac 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java @@ -8,6 +8,8 @@ package org.elasticsearch.action.fieldcaps; +import org.elasticsearch.Version; +import org.elasticsearch.index.mapper.TimeSeriesParams; import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; @@ -32,6 +34,9 @@ import java.util.function.Function; import java.util.stream.Collectors; +import static org.elasticsearch.index.mapper.TimeSeriesParams.TIME_SERIES_DIMENSION_PARAM; +import static org.elasticsearch.index.mapper.TimeSeriesParams.TIME_SERIES_METRIC_PARAM; + /** * Describes the capabilities of a field optionally merged across multiple indices. */ @@ -41,9 +46,13 @@ public class FieldCapabilities implements Writeable, ToXContentObject { private static final ParseField IS_METADATA_FIELD = new ParseField("metadata_field"); private static final ParseField SEARCHABLE_FIELD = new ParseField("searchable"); private static final ParseField AGGREGATABLE_FIELD = new ParseField("aggregatable"); + private static final ParseField TIME_SERIES_DIMENSION_FIELD = new ParseField(TIME_SERIES_DIMENSION_PARAM); + private static final ParseField TIME_SERIES_METRIC_FIELD = new ParseField(TIME_SERIES_METRIC_PARAM); private static final ParseField INDICES_FIELD = new ParseField("indices"); private static final ParseField NON_SEARCHABLE_INDICES_FIELD = new ParseField("non_searchable_indices"); private static final ParseField NON_AGGREGATABLE_INDICES_FIELD = new ParseField("non_aggregatable_indices"); + private static final ParseField NON_DIMENSION_INDICES_FIELD = new ParseField("non_dimension_indices"); + private static final ParseField METRIC_CONFLICTS_INDICES_FIELD = new ParseField("mertric_conflicts_indices"); private static final ParseField META_FIELD = new ParseField("meta"); private final String name; @@ -51,10 +60,14 @@ public class FieldCapabilities implements Writeable, ToXContentObject { private final boolean isMetadataField; private final boolean isSearchable; private final boolean isAggregatable; + private final boolean isDimension; + private final TimeSeriesParams.MetricType metricType; private final String[] indices; private final String[] nonSearchableIndices; private final String[] nonAggregatableIndices; + private final String[] nonDimensionIndices; + private final String[] metricConflictsIndices; private final Map> meta; @@ -65,42 +78,110 @@ public class FieldCapabilities implements Writeable, ToXContentObject { * @param isMetadataField Whether this field is a metadata field. * @param isSearchable Whether this field is indexed for search. * @param isAggregatable Whether this field can be aggregated on. + * @param isDimension Whether this field can be used as dimension + * @param metricType If this field is a metric field, returns the metric's type or null for non-metrics fields * @param indices The list of indices where this field name is defined as {@code type}, * or null if all indices have the same {@code type} for the field. * @param nonSearchableIndices The list of indices where this field is not searchable, * or null if the field is searchable in all indices. * @param nonAggregatableIndices The list of indices where this field is not aggregatable, * or null if the field is aggregatable in all indices. + * @param nonDimensionIndices The list of indices where this field is not a dimension + * @param metricConflictsIndices The list of indices where this field is has different metric types or not mark as a metric * @param meta Merged metadata across indices. */ public FieldCapabilities(String name, String type, boolean isMetadataField, boolean isSearchable, boolean isAggregatable, + boolean isDimension, + TimeSeriesParams.MetricType metricType, String[] indices, String[] nonSearchableIndices, String[] nonAggregatableIndices, + String[] nonDimensionIndices, + String[] metricConflictsIndices, Map> meta) { this.name = name; this.type = type; this.isMetadataField = isMetadataField; this.isSearchable = isSearchable; this.isAggregatable = isAggregatable; + this.isDimension = isDimension; + this.metricType = metricType; this.indices = indices; this.nonSearchableIndices = nonSearchableIndices; this.nonAggregatableIndices = nonAggregatableIndices; + this.nonDimensionIndices = nonDimensionIndices; + this.metricConflictsIndices = metricConflictsIndices; this.meta = Objects.requireNonNull(meta); } + /** + * Constructor for non-timeseries field caps. Useful for testing + * Constructor for a set of indices. + * @param name The name of the field + * @param type The type associated with the field. + * @param isMetadataField Whether this field is a metadata field. + * @param isSearchable Whether this field is indexed for search. + * @param isAggregatable Whether this field can be aggregated on. + * @param indices The list of indices where this field name is defined as {@code type}, + * or null if all indices have the same {@code type} for the field. + * @param nonSearchableIndices The list of indices where this field is not searchable, + * or null if the field is searchable in all indices. + * @param nonAggregatableIndices The list of indices where this field is not aggregatable, + * or null if the field is aggregatable in all indices. + * @param meta Merged metadata across indices. + */ + public FieldCapabilities(String name, String type, + boolean isMetadataField, + boolean isSearchable, + boolean isAggregatable, + String[] indices, + String[] nonSearchableIndices, + String[] nonAggregatableIndices, + Map> meta) { + this( + name, + type, + isMetadataField, + isSearchable, + isAggregatable, + false, + null, + indices, + nonSearchableIndices, + nonAggregatableIndices, + null, + null, + meta + ); + + } + FieldCapabilities(StreamInput in) throws IOException { this.name = in.readString(); this.type = in.readString(); this.isMetadataField = in.readBoolean(); this.isSearchable = in.readBoolean(); this.isAggregatable = in.readBoolean(); + if (in.getVersion().onOrAfter(Version.V_8_0_0)) { + this.isDimension = in.readBoolean(); + this.metricType = in.readOptionalEnum(TimeSeriesParams.MetricType.class); + } else { + this.isDimension = false; + this.metricType = null; + } this.indices = in.readOptionalStringArray(); this.nonSearchableIndices = in.readOptionalStringArray(); this.nonAggregatableIndices = in.readOptionalStringArray(); + if (in.getVersion().onOrAfter(Version.V_8_0_0)) { + this.nonDimensionIndices = in.readOptionalStringArray(); + this.metricConflictsIndices = in.readOptionalStringArray(); + } else { + this.nonDimensionIndices = null; + this.metricConflictsIndices = null; + } meta = in.readMap(StreamInput::readString, i -> i.readSet(StreamInput::readString)); } @@ -111,9 +192,17 @@ public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(isMetadataField); out.writeBoolean(isSearchable); out.writeBoolean(isAggregatable); + if (out.getVersion().onOrAfter(Version.V_8_0_0)) { + out.writeBoolean(isDimension); + out.writeOptionalEnum(metricType); + } out.writeOptionalStringArray(indices); out.writeOptionalStringArray(nonSearchableIndices); out.writeOptionalStringArray(nonAggregatableIndices); + if (out.getVersion().onOrAfter(Version.V_8_0_0)) { + out.writeOptionalStringArray(nonDimensionIndices); + out.writeOptionalStringArray(metricConflictsIndices); + } out.writeMap(meta, StreamOutput::writeString, (o, set) -> o.writeCollection(set, StreamOutput::writeString)); } @@ -124,6 +213,12 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(IS_METADATA_FIELD.getPreferredName(), isMetadataField); builder.field(SEARCHABLE_FIELD.getPreferredName(), isSearchable); builder.field(AGGREGATABLE_FIELD.getPreferredName(), isAggregatable); + if (isDimension) { + builder.field(TIME_SERIES_DIMENSION_FIELD.getPreferredName(), isDimension); + } + if (metricType != null) { + builder.field(TIME_SERIES_METRIC_FIELD.getPreferredName(), metricType); + } if (indices != null) { builder.array(INDICES_FIELD.getPreferredName(), indices); } @@ -133,6 +228,12 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (nonAggregatableIndices != null) { builder.array(NON_AGGREGATABLE_INDICES_FIELD.getPreferredName(), nonAggregatableIndices); } + if (nonDimensionIndices != null) { + builder.field(NON_DIMENSION_INDICES_FIELD.getPreferredName(), nonDimensionIndices); + } + if (metricConflictsIndices != null) { + builder.field(METRIC_CONFLICTS_INDICES_FIELD.getPreferredName(), metricConflictsIndices); + } if (meta.isEmpty() == false) { builder.startObject("meta"); List>> entries = new ArrayList<>(meta.entrySet()); @@ -156,26 +257,40 @@ public static FieldCapabilities fromXContent(String name, XContentParser parser) private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( "field_capabilities", true, - (a, name) -> new FieldCapabilities(name, + (a, name) -> new FieldCapabilities( + name, (String) a[0], a[3] == null ? false : (boolean) a[3], (boolean) a[1], (boolean) a[2], - a[4] != null ? ((List) a[4]).toArray(new String[0]) : null, - a[5] != null ? ((List) a[5]).toArray(new String[0]) : null, + a[4] == null ? false : (boolean) a[4], + a[5] != null ? Enum.valueOf(TimeSeriesParams.MetricType.class, (String) a[5]) : null, a[6] != null ? ((List) a[6]).toArray(new String[0]) : null, - a[7] != null ? ((Map>) a[7]) : Collections.emptyMap())); + a[7] != null ? ((List) a[7]).toArray(new String[0]) : null, + a[8] != null ? ((List) a[8]).toArray(new String[0]) : null, + a[9] != null ? ((List) a[9]).toArray(new String[0]) : null, + a[10] != null ? ((List) a[10]).toArray(new String[0]) : null, + a[11] != null ? ((Map>) a[11]) : Collections.emptyMap() + ) + ); static { - PARSER.declareString(ConstructingObjectParser.constructorArg(), TYPE_FIELD); - PARSER.declareBoolean(ConstructingObjectParser.constructorArg(), SEARCHABLE_FIELD); - PARSER.declareBoolean(ConstructingObjectParser.constructorArg(), AGGREGATABLE_FIELD); - PARSER.declareBoolean(ConstructingObjectParser.optionalConstructorArg(), IS_METADATA_FIELD); - PARSER.declareStringArray(ConstructingObjectParser.optionalConstructorArg(), INDICES_FIELD); - PARSER.declareStringArray(ConstructingObjectParser.optionalConstructorArg(), NON_SEARCHABLE_INDICES_FIELD); - PARSER.declareStringArray(ConstructingObjectParser.optionalConstructorArg(), NON_AGGREGATABLE_INDICES_FIELD); - PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), - (parser, context) -> parser.map(HashMap::new, p -> Set.copyOf(p.list())), META_FIELD); + PARSER.declareString(ConstructingObjectParser.constructorArg(), TYPE_FIELD); // 0 + PARSER.declareBoolean(ConstructingObjectParser.constructorArg(), SEARCHABLE_FIELD); // 1 + PARSER.declareBoolean(ConstructingObjectParser.constructorArg(), AGGREGATABLE_FIELD); // 2 + PARSER.declareBoolean(ConstructingObjectParser.optionalConstructorArg(), IS_METADATA_FIELD); // 3 + PARSER.declareBoolean(ConstructingObjectParser.optionalConstructorArg(), TIME_SERIES_DIMENSION_FIELD); // 4 + PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), TIME_SERIES_METRIC_FIELD); // 5 + PARSER.declareStringArray(ConstructingObjectParser.optionalConstructorArg(), INDICES_FIELD); // 6 + PARSER.declareStringArray(ConstructingObjectParser.optionalConstructorArg(), NON_SEARCHABLE_INDICES_FIELD); // 7 + PARSER.declareStringArray(ConstructingObjectParser.optionalConstructorArg(), NON_AGGREGATABLE_INDICES_FIELD); // 8 + PARSER.declareStringArray(ConstructingObjectParser.optionalConstructorArg(), NON_DIMENSION_INDICES_FIELD); // 9 + PARSER.declareStringArray(ConstructingObjectParser.optionalConstructorArg(), METRIC_CONFLICTS_INDICES_FIELD); // 10 + PARSER.declareObject( + ConstructingObjectParser.optionalConstructorArg(), + (parser, context) -> parser.map(HashMap::new, p -> Set.copyOf(p.list())), + META_FIELD + ); // 11 } /** @@ -206,6 +321,20 @@ public boolean isSearchable() { return isSearchable; } + /** + * Whether this field is a dimension in any indices. + */ + public boolean isDimension() { + return isDimension; + } + + /** + * The metric type + */ + public TimeSeriesParams.MetricType getMetricType() { + return metricType; + } + /** * The type of the field. */ @@ -237,6 +366,21 @@ public String[] nonAggregatableIndices() { return nonAggregatableIndices; } + + /** + * The list of indices where this field has different dimension or metric flag + */ + public String[] nonDimensionIndices() { + return nonDimensionIndices; + } + + /** + * The list of indices where this field has different dimension or metric flag + */ + public String[] metricConflictsIndices() { + return metricConflictsIndices; + } + /** * Return merged metadata across indices. */ @@ -252,20 +396,26 @@ public boolean equals(Object o) { return isMetadataField == that.isMetadataField && isSearchable == that.isSearchable && isAggregatable == that.isAggregatable && + isDimension == that.isDimension && + Objects.equals(metricType, that.metricType) && Objects.equals(name, that.name) && Objects.equals(type, that.type) && Arrays.equals(indices, that.indices) && Arrays.equals(nonSearchableIndices, that.nonSearchableIndices) && Arrays.equals(nonAggregatableIndices, that.nonAggregatableIndices) && + Arrays.equals(nonDimensionIndices, that.nonDimensionIndices) && + Arrays.equals(metricConflictsIndices, that.metricConflictsIndices) && Objects.equals(meta, that.meta); } @Override public int hashCode() { - int result = Objects.hash(name, type, isMetadataField, isSearchable, isAggregatable, meta); + int result = Objects.hash(name, type, isMetadataField, isSearchable, isAggregatable, isDimension, metricType, meta); result = 31 * result + Arrays.hashCode(indices); result = 31 * result + Arrays.hashCode(nonSearchableIndices); result = 31 * result + Arrays.hashCode(nonAggregatableIndices); + result = 31 * result + Arrays.hashCode(nonDimensionIndices); + result = 31 * result + Arrays.hashCode(metricConflictsIndices); return result; } @@ -280,6 +430,9 @@ static class Builder { private boolean isMetadataField; private boolean isSearchable; private boolean isAggregatable; + private boolean isDimension; + private TimeSeriesParams.MetricType metricType; + private boolean mertricTypeIsSet; private List indiceList; private Map> meta; @@ -288,6 +441,9 @@ static class Builder { this.type = type; this.isSearchable = true; this.isAggregatable = true; + this.isDimension = true; + this.metricType = null; + this.mertricTypeIsSet = false; this.indiceList = new ArrayList<>(); this.meta = new HashMap<>(); } @@ -295,12 +451,31 @@ static class Builder { /** * Collect the field capabilities for an index. */ - void add(String index, boolean isMetadataField, boolean search, boolean agg, Map meta) { - IndexCaps indexCaps = new IndexCaps(index, search, agg); + void add( + String index, + boolean isMetadataField, + boolean search, + boolean agg, + boolean isDimension, + TimeSeriesParams.MetricType metricType, + Map meta + ) { + IndexCaps indexCaps = new IndexCaps(index, search, agg, isDimension, metricType); indiceList.add(indexCaps); this.isSearchable &= search; this.isAggregatable &= agg; this.isMetadataField |= isMetadataField; + this.isDimension &= isDimension; + // If we have discrepancy in metric types or in some indices this field is not marked as a metric field - we will + // treat is a non-metric field and report this discrepancy in metricConflictsIndices + if (this.mertricTypeIsSet) { + if (this.metricType != metricType) { + this.metricType = null; + } + } else { + this.mertricTypeIsSet = true; + this.metricType = metricType; + } for (Map.Entry entry : meta.entrySet()) { this.meta.computeIfAbsent(entry.getKey(), key -> new HashSet<>()) .add(entry.getValue()); @@ -347,12 +522,49 @@ FieldCapabilities build(boolean withIndices) { } else { nonAggregatableIndices = null; } + + final String[] nonDimensionIndices; + if (isDimension == false && indiceList.stream().anyMatch((caps) -> caps.isDimension)) { + // Collect all indices that have dimension == false if this field is marked as a dimension in at least one index + nonDimensionIndices = indiceList.stream() + .filter((caps) -> caps.isDimension == false) + .map(caps -> caps.name) + .toArray(String[]::new); + } else { + nonDimensionIndices = null; + } + + final String[] metricConflictsIndices; + if (indiceList.stream().anyMatch((caps) -> caps.metricType != metricType)) { + // Collect all indices that have this field. If it is marked differently in different indices, we cannot really + // make a decisions which index is "right" and which index is "wrong" so collecting all indices where this field + // is present is probably the only sensible thing to do here + metricConflictsIndices = indiceList.stream() + .map(caps -> caps.name) + .toArray(String[]::new); + } else { + metricConflictsIndices = null; + } + final Function>, Set> entryValueFunction = Map.Entry::getValue; Map> immutableMeta = meta.entrySet().stream() .collect(Collectors.toUnmodifiableMap( Map.Entry::getKey, entryValueFunction.andThen(Set::copyOf))); - return new FieldCapabilities(name, type, isMetadataField, isSearchable, isAggregatable, - indices, nonSearchableIndices, nonAggregatableIndices, immutableMeta); + return new FieldCapabilities( + name, + type, + isMetadataField, + isSearchable, + isAggregatable, + isDimension, + metricType, + indices, + nonSearchableIndices, + nonAggregatableIndices, + nonDimensionIndices, + metricConflictsIndices, + immutableMeta + ); } } @@ -360,11 +572,15 @@ private static class IndexCaps { final String name; final boolean isSearchable; final boolean isAggregatable; + final boolean isDimension; + final TimeSeriesParams.MetricType metricType; - IndexCaps(String name, boolean isSearchable, boolean isAggregatable) { + IndexCaps(String name, boolean isSearchable, boolean isAggregatable, boolean isDimension, TimeSeriesParams.MetricType metricType) { this.name = name; this.isSearchable = isSearchable; this.isAggregatable = isAggregatable; + this.isDimension = isDimension; + this.metricType = metricType; } } } diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/IndexFieldCapabilities.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/IndexFieldCapabilities.java index a59a75a4a5e09..e0164196bedc1 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/IndexFieldCapabilities.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/IndexFieldCapabilities.java @@ -8,10 +8,12 @@ package org.elasticsearch.action.fieldcaps; +import org.elasticsearch.Version; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.util.StringLiteralDeduplicator; +import org.elasticsearch.index.mapper.TimeSeriesParams; import java.io.IOException; import java.util.Map; @@ -29,6 +31,8 @@ public class IndexFieldCapabilities implements Writeable { private final boolean isMetadatafield; private final boolean isSearchable; private final boolean isAggregatable; + private final boolean isDimension; + private final TimeSeriesParams.MetricType metricType; private final Map meta; /** @@ -41,6 +45,8 @@ public class IndexFieldCapabilities implements Writeable { IndexFieldCapabilities(String name, String type, boolean isMetadatafield, boolean isSearchable, boolean isAggregatable, + boolean isDimension, + TimeSeriesParams.MetricType metricType, Map meta) { this.name = name; @@ -48,6 +54,8 @@ public class IndexFieldCapabilities implements Writeable { this.isMetadatafield = isMetadatafield; this.isSearchable = isSearchable; this.isAggregatable = isAggregatable; + this.isDimension = isDimension; + this.metricType = metricType; this.meta = meta; } @@ -57,6 +65,13 @@ public class IndexFieldCapabilities implements Writeable { this.isMetadatafield = in.readBoolean(); this.isSearchable = in.readBoolean(); this.isAggregatable = in.readBoolean(); + if (in.getVersion().onOrAfter(Version.V_8_0_0)) { + this.isDimension = in.readBoolean(); + this.metricType = in.readOptionalEnum(TimeSeriesParams.MetricType.class); + } else { + this.isDimension = false; + this.metricType = null; + } this.meta = in.readMap(StreamInput::readString, StreamInput::readString); } @@ -67,6 +82,10 @@ public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(isMetadatafield); out.writeBoolean(isSearchable); out.writeBoolean(isAggregatable); + if (out.getVersion().onOrAfter(Version.V_8_0_0)) { + out.writeBoolean(isDimension); + out.writeOptionalEnum(metricType); + } out.writeMap(meta, StreamOutput::writeString, StreamOutput::writeString); } @@ -90,6 +109,14 @@ public boolean isSearchable() { return isSearchable; } + public boolean isDimension() { + return isDimension; + } + + public TimeSeriesParams.MetricType getMetricType() { + return metricType; + } + public Map meta() { return meta; } @@ -102,6 +129,8 @@ public boolean equals(Object o) { return isMetadatafield == that.isMetadatafield && isSearchable == that.isSearchable && isAggregatable == that.isAggregatable && + isDimension == that.isDimension && + Objects.equals(metricType, that.metricType) && Objects.equals(name, that.name) && Objects.equals(type, that.type) && Objects.equals(meta, that.meta); @@ -109,6 +138,6 @@ public boolean equals(Object o) { @Override public int hashCode() { - return Objects.hash(name, type, isMetadatafield, isSearchable, isAggregatable, meta); + return Objects.hash(name, type, isMetadatafield, isSearchable, isAggregatable, isDimension, metricType, meta); } } diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java index e997ca1af0a56..3e620b70ba9aa 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java @@ -275,7 +275,7 @@ private void addUnmappedFields(String[] indices, String field, Map> resp Map typeMap = responseMapBuilder.computeIfAbsent(field, f -> new HashMap<>()); FieldCapabilities.Builder builder = typeMap.computeIfAbsent(fieldCap.getType(), key -> new FieldCapabilities.Builder(field, key)); - builder.add(response.getIndexName(), isMetadataField, fieldCap.isSearchable(), fieldCap.isAggregatable(), fieldCap.meta()); + builder.add( + response.getIndexName(), + isMetadataField, + fieldCap.isSearchable(), + fieldCap.isAggregatable(), + fieldCap.isDimension(), + fieldCap.getMetricType(), + fieldCap.meta() + ); } } @@ -350,8 +358,16 @@ private FieldCapabilitiesIndexResponse shardOperation(final FieldCapabilitiesInd MappedFieldType ft = searchExecutionContext.getFieldType(field); boolean isMetadataField = searchExecutionContext.isMetadataField(field); if (isMetadataField || fieldPredicate.test(ft.name())) { - IndexFieldCapabilities fieldCap = new IndexFieldCapabilities(field, - ft.familyTypeName(), isMetadataField, ft.isSearchable(), ft.isAggregatable(), ft.meta()); + IndexFieldCapabilities fieldCap = new IndexFieldCapabilities( + field, + ft.familyTypeName(), + isMetadataField, + ft.isSearchable(), + ft.isAggregatable(), + ft.isDimension(), + ft.getMetricType(), + ft.meta() + ); responseMap.put(field, fieldCap); } else { continue; @@ -376,7 +392,7 @@ private FieldCapabilitiesIndexResponse shardOperation(final FieldCapabilitiesInd if (mapper != null) { String type = mapper.isNested() ? "nested" : "object"; IndexFieldCapabilities fieldCap = new IndexFieldCapabilities(parentField, type, - false, false, false, Collections.emptyMap()); + false, false, false, false, null, Collections.emptyMap()); responseMap.put(parentField, fieldCap); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java index 5e13e3cc3f10a..6d2970b9ec656 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java @@ -172,6 +172,13 @@ public boolean isDimension() { return false; } + /** + * @return metric type or null if the field is not a metric field + */ + public TimeSeriesParams.MetricType getMetricType() { + return null; + } + /** Generates a query that will only match documents that contain the given value. * The default implementation returns a {@link TermQuery} over the value bytes * @throws IllegalArgumentException if {@code value} cannot be converted to the expected data type or if the field is not searchable diff --git a/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponseTests.java b/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponseTests.java index 1e47792561dfe..408397a0c02be 100644 --- a/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponseTests.java @@ -11,6 +11,7 @@ import org.elasticsearch.ElasticsearchExceptionTests; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.index.mapper.TimeSeriesParams; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentType; @@ -73,7 +74,7 @@ private static IndexFieldCapabilities randomFieldCaps(String fieldName) { } return new IndexFieldCapabilities(fieldName, randomAlphaOfLengthBetween(5, 20), - randomBoolean(), randomBoolean(), randomBoolean(), meta); + randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomFrom(TimeSeriesParams.MetricType.values()), meta); } @Override diff --git a/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesTests.java b/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesTests.java index e162728081582..42f9d5fefddfb 100644 --- a/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesTests.java +++ b/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesTests.java @@ -9,6 +9,7 @@ package org.elasticsearch.action.fieldcaps; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.index.mapper.TimeSeriesParams; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.test.AbstractSerializingTestCase; @@ -40,72 +41,119 @@ protected Writeable.Reader instanceReader() { public void testBuilder() { FieldCapabilities.Builder builder = new FieldCapabilities.Builder("field", "type"); - builder.add("index1", false, true, false, Collections.emptyMap()); - builder.add("index2", false, true, false, Collections.emptyMap()); - builder.add("index3", false, true, false, Collections.emptyMap()); + builder.add("index1", false, true, false, false, null, Collections.emptyMap()); + builder.add("index2", false, true, false, false, null, Collections.emptyMap()); + builder.add("index3", false, true, false, false, null, Collections.emptyMap()); { FieldCapabilities cap1 = builder.build(false); assertThat(cap1.isSearchable(), equalTo(true)); assertThat(cap1.isAggregatable(), equalTo(false)); + assertThat(cap1.isDimension(), equalTo(false)); + assertNull(cap1.getMetricType()); assertNull(cap1.indices()); assertNull(cap1.nonSearchableIndices()); assertNull(cap1.nonAggregatableIndices()); + assertNull(cap1.nonDimensionIndices()); assertEquals(Collections.emptyMap(), cap1.meta()); FieldCapabilities cap2 = builder.build(true); assertThat(cap2.isSearchable(), equalTo(true)); assertThat(cap2.isAggregatable(), equalTo(false)); + assertThat(cap2.isDimension(), equalTo(false)); + assertNull(cap2.getMetricType()); assertThat(cap2.indices().length, equalTo(3)); assertThat(cap2.indices(), equalTo(new String[]{"index1", "index2", "index3"})); assertNull(cap2.nonSearchableIndices()); assertNull(cap2.nonAggregatableIndices()); + assertNull(cap2.nonDimensionIndices()); assertEquals(Collections.emptyMap(), cap2.meta()); } builder = new FieldCapabilities.Builder("field", "type"); - builder.add("index1", false, false, true, Collections.emptyMap()); - builder.add("index2", false, true, false, Collections.emptyMap()); - builder.add("index3", false, false, false, Collections.emptyMap()); + builder.add("index1", false, false, true, true, null, Collections.emptyMap()); + builder.add("index2", false, true, false, false, TimeSeriesParams.MetricType.counter, Collections.emptyMap()); + builder.add("index3", false, false, false, false, null, Collections.emptyMap()); { FieldCapabilities cap1 = builder.build(false); assertThat(cap1.isSearchable(), equalTo(false)); assertThat(cap1.isAggregatable(), equalTo(false)); + assertThat(cap1.isDimension(), equalTo(false)); + assertNull(cap1.getMetricType()); assertNull(cap1.indices()); assertThat(cap1.nonSearchableIndices(), equalTo(new String[]{"index1", "index3"})); assertThat(cap1.nonAggregatableIndices(), equalTo(new String[]{"index2", "index3"})); + assertThat(cap1.nonDimensionIndices(), equalTo(new String[]{"index2", "index3"})); assertEquals(Collections.emptyMap(), cap1.meta()); FieldCapabilities cap2 = builder.build(true); assertThat(cap2.isSearchable(), equalTo(false)); assertThat(cap2.isAggregatable(), equalTo(false)); + assertThat(cap2.isDimension(), equalTo(false)); + assertNull(cap2.getMetricType()); assertThat(cap2.indices().length, equalTo(3)); assertThat(cap2.indices(), equalTo(new String[]{"index1", "index2", "index3"})); assertThat(cap2.nonSearchableIndices(), equalTo(new String[]{"index1", "index3"})); assertThat(cap2.nonAggregatableIndices(), equalTo(new String[]{"index2", "index3"})); + assertThat(cap2.nonDimensionIndices(), equalTo(new String[]{"index2", "index3"})); assertEquals(Collections.emptyMap(), cap2.meta()); } builder = new FieldCapabilities.Builder("field", "type"); - builder.add("index1", false, true, true, Collections.emptyMap()); - builder.add("index2", false, true, true, Map.of("foo", "bar")); - builder.add("index3", false, true, true, Map.of("foo", "quux")); + builder.add("index1", false, true, true, true, TimeSeriesParams.MetricType.counter, Collections.emptyMap()); + builder.add("index2", false, true, true, true, TimeSeriesParams.MetricType.counter, Map.of("foo", "bar")); + builder.add("index3", false, true, true, true, TimeSeriesParams.MetricType.counter, Map.of("foo", "quux")); { FieldCapabilities cap1 = builder.build(false); assertThat(cap1.isSearchable(), equalTo(true)); assertThat(cap1.isAggregatable(), equalTo(true)); + assertThat(cap1.isDimension(), equalTo(true)); + assertThat(cap1.getMetricType(), equalTo(TimeSeriesParams.MetricType.counter)); assertNull(cap1.indices()); assertNull(cap1.nonSearchableIndices()); assertNull(cap1.nonAggregatableIndices()); + assertNull(cap1.nonDimensionIndices()); assertEquals(Map.of("foo", Set.of("bar", "quux")), cap1.meta()); FieldCapabilities cap2 = builder.build(true); assertThat(cap2.isSearchable(), equalTo(true)); assertThat(cap2.isAggregatable(), equalTo(true)); + assertThat(cap2.isDimension(), equalTo(true)); + assertThat(cap2.getMetricType(), equalTo(TimeSeriesParams.MetricType.counter)); assertThat(cap2.indices().length, equalTo(3)); assertThat(cap2.indices(), equalTo(new String[]{"index1", "index2", "index3"})); assertNull(cap2.nonSearchableIndices()); assertNull(cap2.nonAggregatableIndices()); + assertNull(cap2.nonDimensionIndices()); + assertEquals(Map.of("foo", Set.of("bar", "quux")), cap2.meta()); + } + + builder = new FieldCapabilities.Builder("field", "type"); + builder.add("index1", false, true, true, true, TimeSeriesParams.MetricType.counter, Collections.emptyMap()); + builder.add("index2", false, true, true, true, TimeSeriesParams.MetricType.gauge, Map.of("foo", "bar")); + builder.add("index3", false, true, true, true, TimeSeriesParams.MetricType.counter, Map.of("foo", "quux")); + { + FieldCapabilities cap1 = builder.build(false); + assertThat(cap1.isSearchable(), equalTo(true)); + assertThat(cap1.isAggregatable(), equalTo(true)); + assertThat(cap1.isDimension(), equalTo(true)); + assertNull(cap1.getMetricType()); + assertNull(cap1.indices()); + assertNull(cap1.nonSearchableIndices()); + assertNull(cap1.nonAggregatableIndices()); + assertNull(cap1.nonDimensionIndices()); + assertEquals(Map.of("foo", Set.of("bar", "quux")), cap1.meta()); + + FieldCapabilities cap2 = builder.build(true); + assertThat(cap2.isSearchable(), equalTo(true)); + assertThat(cap2.isAggregatable(), equalTo(true)); + assertThat(cap2.isDimension(), equalTo(true)); + assertNull(cap2.getMetricType()); + assertThat(cap2.indices().length, equalTo(3)); + assertThat(cap2.indices(), equalTo(new String[]{"index1", "index2", "index3"})); + assertNull(cap2.nonSearchableIndices()); + assertNull(cap2.nonAggregatableIndices()); + assertNull(cap2.nonDimensionIndices()); assertEquals(Map.of("foo", Set.of("bar", "quux")), cap2.meta()); } } @@ -133,6 +181,22 @@ static FieldCapabilities randomFieldCaps(String fieldName) { } } + String[] nonDimensionIndices = null; + if (randomBoolean()) { + nonDimensionIndices = new String[randomIntBetween(0, 5)]; + for (int i = 0; i < nonDimensionIndices.length; i++) { + nonDimensionIndices[i] = randomAlphaOfLengthBetween(5, 20); + } + } + + String[] metricConflictsIndices = null; + if (randomBoolean()) { + metricConflictsIndices = new String[randomIntBetween(0, 5)]; + for (int i = 0; i < metricConflictsIndices.length; i++) { + metricConflictsIndices[i] = randomAlphaOfLengthBetween(5, 20); + } + } + Map> meta; switch (randomInt(2)) { case 0: @@ -146,9 +210,21 @@ static FieldCapabilities randomFieldCaps(String fieldName) { break; } - return new FieldCapabilities(fieldName, - randomAlphaOfLengthBetween(5, 20), randomBoolean(), randomBoolean(), randomBoolean(), - indices, nonSearchableIndices, nonAggregatableIndices, meta); + return new FieldCapabilities( + fieldName, + randomAlphaOfLengthBetween(5, 20), + randomBoolean(), + randomBoolean(), + randomBoolean(), + randomBoolean(), + randomFrom(TimeSeriesParams.MetricType.values()), + indices, + nonSearchableIndices, + nonAggregatableIndices, + nonDimensionIndices, + metricConflictsIndices, + meta + ); } @Override @@ -158,11 +234,15 @@ protected FieldCapabilities mutateInstance(FieldCapabilities instance) { boolean isMetadataField = instance.isMetadataField(); boolean isSearchable = instance.isSearchable(); boolean isAggregatable = instance.isAggregatable(); + boolean isDimension = instance.isDimension(); + TimeSeriesParams.MetricType metricType = instance.getMetricType(); String[] indices = instance.indices(); String[] nonSearchableIndices = instance.nonSearchableIndices(); String[] nonAggregatableIndices = instance.nonAggregatableIndices(); + String[] nonDimensionIndices = instance.nonDimensionIndices(); + String[] metricConflictsIndices = instance.metricConflictsIndices(); Map> meta = instance.meta(); - switch (between(0, 8)) { + switch (between(0, 12)) { case 0: name += randomAlphaOfLengthBetween(1, 10); break; @@ -229,10 +309,67 @@ protected FieldCapabilities mutateInstance(FieldCapabilities instance) { case 8: isMetadataField = isMetadataField == false; break; + case 9: + isDimension = isDimension == false; + break; + case 10: + if (metricType == null) { + metricType = randomFrom(TimeSeriesParams.MetricType.values()); + } else { + if (randomBoolean()) { + metricType = null; + } else { + metricType = randomValueOtherThan(metricType, () -> randomFrom(TimeSeriesParams.MetricType.values())); + } + } + break; + case 11: + String[] newTimeSeriesDimensionsConflictsIndices; + int startTimeSeriesDimensionsConflictsPos = 0; + if (nonDimensionIndices == null) { + newTimeSeriesDimensionsConflictsIndices = new String[between(1, 10)]; + } else { + newTimeSeriesDimensionsConflictsIndices = Arrays.copyOf(nonDimensionIndices, + nonDimensionIndices.length + between(1, 10)); + startTimeSeriesDimensionsConflictsPos = nonDimensionIndices.length; + } + for (int i = startTimeSeriesDimensionsConflictsPos; i < newTimeSeriesDimensionsConflictsIndices.length; i++) { + newTimeSeriesDimensionsConflictsIndices[i] = randomAlphaOfLengthBetween(5, 20); + } + nonDimensionIndices = newTimeSeriesDimensionsConflictsIndices; + break; + case 12: + String[] newMetricConflictsIndices; + int startMetricConflictsPos = 0; + if (metricConflictsIndices == null) { + newMetricConflictsIndices = new String[between(1, 10)]; + } else { + newMetricConflictsIndices = Arrays.copyOf(metricConflictsIndices, + metricConflictsIndices.length + between(1, 10)); + startMetricConflictsPos = metricConflictsIndices.length; + } + for (int i = startMetricConflictsPos; i < newMetricConflictsIndices.length; i++) { + newMetricConflictsIndices[i] = randomAlphaOfLengthBetween(5, 20); + } + metricConflictsIndices = newMetricConflictsIndices; + break; default: throw new AssertionError(); } - return new FieldCapabilities(name, type, isMetadataField, isSearchable, isAggregatable, - indices, nonSearchableIndices, nonAggregatableIndices, meta); + return new FieldCapabilities( + name, + type, + isMetadataField, + isSearchable, + isAggregatable, + isDimension, + metricType, + indices, + nonSearchableIndices, + nonAggregatableIndices, + nonDimensionIndices, + metricConflictsIndices, + meta + ); } } diff --git a/server/src/test/java/org/elasticsearch/action/fieldcaps/MergedFieldCapabilitiesResponseTests.java b/server/src/test/java/org/elasticsearch/action/fieldcaps/MergedFieldCapabilitiesResponseTests.java index 576d8ca9bd940..1c20a8066b54f 100644 --- a/server/src/test/java/org/elasticsearch/action/fieldcaps/MergedFieldCapabilitiesResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/fieldcaps/MergedFieldCapabilitiesResponseTests.java @@ -10,6 +10,7 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.index.mapper.TimeSeriesParams; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentFactory; @@ -115,6 +116,7 @@ public void testToXContent() throws IOException { " \"metadata_field\": false," + " \"searchable\": false," + " \"aggregatable\": true," + + " \"time_series_dimension\": true," + " \"indices\": [\"index3\", \"index4\"]," + " \"non_searchable_indices\": [\"index4\"] " + " }," + @@ -123,8 +125,10 @@ public void testToXContent() throws IOException { " \"metadata_field\": false," + " \"searchable\": true," + " \"aggregatable\": false," + + " \"time_series_metric\": \"counter\"," + " \"indices\": [\"index1\", \"index2\"]," + - " \"non_aggregatable_indices\": [\"index1\"] " + + " \"non_aggregatable_indices\": [\"index1\"]," + + " \"non_dimension_indices\":[\"index4\"] " + " }" + " }," + " \"title\": { " + @@ -147,20 +151,20 @@ public void testToXContent() throws IOException { private static FieldCapabilitiesResponse createSimpleResponse() { Map titleCapabilities = new HashMap<>(); - titleCapabilities.put("text", new FieldCapabilities("title", "text", false, true, false, - null, null, null, Collections.emptyMap())); + titleCapabilities.put("text", new FieldCapabilities("title", "text", false, true, false, false, null, + null, null, null, null, null, Collections.emptyMap())); Map ratingCapabilities = new HashMap<>(); ratingCapabilities.put("long", new FieldCapabilities("rating", "long", - false, true, false, - new String[]{"index1", "index2"}, - null, - new String[]{"index1"}, Collections.emptyMap())); + false, true, false, false, TimeSeriesParams.MetricType.counter, + new String[]{"index1", "index2"}, null, new String[]{"index1"}, new String[]{"index4"}, + null, Collections.emptyMap() + )); ratingCapabilities.put("keyword", new FieldCapabilities("rating", "keyword", - false, false, true, - new String[]{"index3", "index4"}, - new String[]{"index4"}, - null, Collections.emptyMap())); + false, false, true, true, null, + new String[]{"index3", "index4"}, new String[]{"index4"}, null, null, null, + Collections.emptyMap() + )); Map> responses = new HashMap<>(); responses.put("title", titleCapabilities); diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java index af11af45c72eb..8eb425bb72007 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java @@ -248,9 +248,11 @@ public void testMergeIncompatibleCapabilitiesOfObjectFields() throws Exception { Map multi = new HashMap<>(); multi.put("long", new FieldCapabilities(fieldName, "long", false, true, true, new String[] { "one-index" }, null, null, - Collections.emptyMap())); + Collections.emptyMap() + )); multi.put("text", new FieldCapabilities(fieldName, "text", false, true, false, new String[] { "another-index" }, null, null, - Collections.emptyMap())); + Collections.emptyMap() + )); fieldCaps.put(fieldName, multi); diff --git a/x-pack/qa/runtime-fields/build.gradle b/x-pack/qa/runtime-fields/build.gradle index 2701c52b4a2b4..c8dc8a378dda2 100644 --- a/x-pack/qa/runtime-fields/build.gradle +++ b/x-pack/qa/runtime-fields/build.gradle @@ -98,6 +98,7 @@ subprojects { 'search/330_fetch_fields/error includes glob pattern', // we need a @timestamp field to be defined in index mapping 'search/380_sort_segments_on_timestamp/*', + 'field_caps/40_time_series/*', /////// NOT SUPPORTED /////// ].join(',') }