diff --git a/docs/changelog/99054.yaml b/docs/changelog/99054.yaml new file mode 100644 index 0000000000000..a9e4128e7ae97 --- /dev/null +++ b/docs/changelog/99054.yaml @@ -0,0 +1,5 @@ +pr: 99054 +summary: "ESQL: Mark counter fields as unsupported" +area: ES|QL +type: enhancement +issues: [] diff --git a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/40_tsdb.yml b/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/40_tsdb.yml index f37c897d77b4b..a72205b3af064 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/40_tsdb.yml +++ b/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/40_tsdb.yml @@ -32,8 +32,10 @@ setup: properties: tx: type: long + time_series_metric: counter rx: type: long + time_series_metric: counter - do: bulk: refresh: true @@ -70,9 +72,9 @@ load everything: - match: {columns.2.name: "k8s.pod.name"} - match: {columns.2.type: "keyword"} - match: {columns.3.name: "k8s.pod.network.rx"} - - match: {columns.3.type: "long"} + - match: {columns.3.type: "unsupported"} - match: {columns.4.name: "k8s.pod.network.tx"} - - match: {columns.4.type: "long"} + - match: {columns.4.type: "unsupported"} - match: {columns.5.name: "k8s.pod.uid"} - match: {columns.5.type: "keyword"} - match: {columns.6.name: "metricset"} @@ -84,14 +86,22 @@ load a document: - do: esql.query: body: - query: 'from test | where k8s.pod.network.tx == 1434577921' + query: 'from test | where @timestamp == "2021-04-28T18:50:23.142Z"' - length: {values: 1} - length: {values.0: 7} - match: {values.0.0: "2021-04-28T18:50:23.142Z"} - match: {values.0.1: "10.10.55.3"} - match: {values.0.2: "dog"} - - match: {values.0.3: 530600088} - - match: {values.0.4: 1434577921} + - match: {values.0.3: ""} + - match: {values.0.4: ""} - match: {values.0.5: "df3145b3-0563-4d3b-a0f7-897eb2876ea9"} - match: {values.0.6: "pod"} + +--- +filter on counter: + - do: + catch: /Cannot use field \[k8s.pod.network.tx\] with unsupported type \[counter\]/ + esql.query: + body: + query: 'from test | where k8s.pod.network.tx == 1434577921' diff --git a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/45_non_tsdb_counter.yml b/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/45_non_tsdb_counter.yml new file mode 100644 index 0000000000000..a4344946aea0d --- /dev/null +++ b/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/45_non_tsdb_counter.yml @@ -0,0 +1,112 @@ +setup: + - do: + indices.create: + index: test + body: + settings: + index: + mode: standard + mappings: + properties: + "@timestamp": + type: date + metricset: + type: keyword + time_series_dimension: true + k8s: + properties: + pod: + properties: + uid: + type: keyword + time_series_dimension: true + name: + type: keyword + ip: + type: ip + network: + properties: + tx: + type: long + time_series_metric: counter + rx: + type: long + time_series_metric: counter + - do: + bulk: + refresh: true + index: test + body: + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:50:04.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2001818691, "rx": 802133794}}}}' + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:50:24.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2005177954, "rx": 801479970}}}}' + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:50:44.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2006223737, "rx": 802337279}}}}' + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:51:04.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.2", "network": {"tx": 2012916202, "rx": 803685721}}}}' + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:50:03.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 1434521831, "rx": 530575198}}}}' + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:50:23.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 1434577921, "rx": 530600088}}}}' + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:50:53.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 1434587694, "rx": 530604797}}}}' + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:51:03.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 1434595272, "rx": 530605511}}}}' + +--- +load everything: + - do: + esql.query: + body: + query: 'from test' + + - match: {columns.0.name: "@timestamp"} + - match: {columns.0.type: "date"} + - match: {columns.1.name: "k8s.pod.ip"} + - match: {columns.1.type: "ip"} + - match: {columns.2.name: "k8s.pod.name"} + - match: {columns.2.type: "keyword"} + - match: {columns.3.name: "k8s.pod.network.rx"} + - match: {columns.3.type: "long"} + - match: {columns.4.name: "k8s.pod.network.tx"} + - match: {columns.4.type: "long"} + - match: {columns.5.name: "k8s.pod.uid"} + - match: {columns.5.type: "keyword"} + - match: {columns.6.name: "metricset"} + - match: {columns.6.type: "keyword"} + - length: {values: 8} + +--- +load a document: + - do: + esql.query: + body: + query: 'from test | where @timestamp == "2021-04-28T18:50:23.142Z"' + + - length: {values: 1} + - length: {values.0: 7} + - match: {values.0.0: "2021-04-28T18:50:23.142Z"} + - match: {values.0.1: "10.10.55.3"} + - match: {values.0.2: "dog"} + - match: {values.0.3: 530600088} + - match: {values.0.4: 1434577921} + - match: {values.0.5: "df3145b3-0563-4d3b-a0f7-897eb2876ea9"} + - match: {values.0.6: "pod"} + +--- +filter on counter: + - do: + esql.query: + body: + query: 'from test | where k8s.pod.network.tx == 1434577921' + + - length: {values: 1} + - length: {values.0: 7} + - match: {values.0.0: "2021-04-28T18:50:23.142Z"} + - match: {values.0.1: "10.10.55.3"} + - match: {values.0.2: "dog"} + - match: {values.0.3: 530600088} + - match: {values.0.4: 1434577921} + - match: {values.0.5: "df3145b3-0563-4d3b-a0f7-897eb2876ea9"} + - match: {values.0.6: "pod"} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponse.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponse.java index 774208480c6ff..2b029a03fa9f9 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponse.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponse.java @@ -247,7 +247,7 @@ private static Object valueAt(String dataType, Block block, int offset, BytesRef */ private static Page valuesToPage(List dataTypes, List> values) { List results = dataTypes.stream() - .map(c -> LocalExecutionPlanner.toElementType(EsqlDataTypes.fromEs(c)).newBlockBuilder(values.size())) + .map(c -> LocalExecutionPlanner.toElementType(EsqlDataTypes.fromName(c)).newBlockBuilder(values.size())) .toList(); for (List row : values) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistry.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistry.java index a0883fce4b79f..ff6e7f4aa2736 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistry.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistry.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.esql.type; +import org.elasticsearch.index.mapper.TimeSeriesParams; import org.elasticsearch.xpack.ql.type.DataType; import org.elasticsearch.xpack.ql.type.DataTypeConverter; import org.elasticsearch.xpack.ql.type.DataTypeRegistry; @@ -29,8 +30,12 @@ public Collection dataTypes() { } @Override - public DataType fromEs(String typeName) { - return EsqlDataTypes.fromEs(typeName); + public DataType fromEs(String typeName, TimeSeriesParams.MetricType metricType) { + if (metricType == TimeSeriesParams.MetricType.COUNTER) { + // Counter fields will be a counter type, for now they are unsupported + return DataTypes.UNSUPPORTED; + } + return EsqlDataTypes.fromName(typeName); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypes.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypes.java index 64358a3435e1f..b8ba722b989ad 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypes.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypes.java @@ -87,7 +87,7 @@ public static DataType fromTypeName(String name) { return NAME_TO_TYPE.get(name.toLowerCase(Locale.ROOT)); } - public static DataType fromEs(String name) { + public static DataType fromName(String name) { DataType type = ES_TO_TYPE.get(name); return type != null ? type : UNSUPPORTED; } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java index 07899000cba4a..5bf8df1c3fd0b 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java @@ -72,7 +72,7 @@ private ColumnInfo randomColumnInfo() { private Page randomPage(List columns) { return new Page(columns.stream().map(c -> { - Block.Builder builder = LocalExecutionPlanner.toElementType(EsqlDataTypes.fromEs(c.type())).newBlockBuilder(1); + Block.Builder builder = LocalExecutionPlanner.toElementType(EsqlDataTypes.fromName(c.type())).newBlockBuilder(1); switch (c.type()) { case "unsigned_long", "long" -> ((LongBlock.Builder) builder).appendLong(randomLong()); case "integer" -> ((IntBlock.Builder) builder).appendInt(randomInt()); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistryTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistryTests.java new file mode 100644 index 0000000000000..3620fcc8c5926 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistryTests.java @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.esql.type; + +import org.elasticsearch.action.fieldcaps.FieldCapabilities; +import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse; +import org.elasticsearch.index.mapper.TimeSeriesParams; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.ql.index.IndexResolution; +import org.elasticsearch.xpack.ql.index.IndexResolver; +import org.elasticsearch.xpack.ql.type.DataType; +import org.elasticsearch.xpack.ql.type.DataTypes; +import org.elasticsearch.xpack.ql.type.EsField; + +import java.util.Map; + +import static org.hamcrest.Matchers.equalTo; + +public class EsqlDataTypeRegistryTests extends ESTestCase { + public void testCounter() { + resolve("long", TimeSeriesParams.MetricType.COUNTER, DataTypes.UNSUPPORTED); + } + + public void testGauge() { + resolve("long", TimeSeriesParams.MetricType.GAUGE, DataTypes.LONG); + } + + public void testLong() { + resolve("long", null, DataTypes.LONG); + } + + private void resolve(String esTypeName, TimeSeriesParams.MetricType metricType, DataType expected) { + String[] indices = new String[] { "idx-" + randomAlphaOfLength(5) }; + FieldCapabilities fieldCap = new FieldCapabilities( + randomAlphaOfLength(3), + esTypeName, + false, + true, + true, + false, + metricType, + indices, + null, + null, + null, + null, + Map.of() + ); + FieldCapabilitiesResponse caps = new FieldCapabilitiesResponse(indices, Map.of(fieldCap.getName(), Map.of(esTypeName, fieldCap))); + IndexResolution resolution = IndexResolver.mergedMappings(EsqlDataTypeRegistry.INSTANCE, "idx-*", caps); + + EsField f = resolution.get().mapping().get(fieldCap.getName()); + assertThat(f.getDataType(), equalTo(expected)); + } +} diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java index 16d245bb93de5..c0c3068b4d98b 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java @@ -24,6 +24,7 @@ import org.elasticsearch.common.util.Maps; import org.elasticsearch.core.Tuple; import org.elasticsearch.index.IndexNotFoundException; +import org.elasticsearch.index.mapper.TimeSeriesParams; import org.elasticsearch.transport.NoSuchRemoteClusterException; import org.elasticsearch.xpack.ql.QlIllegalArgumentException; import org.elasticsearch.xpack.ql.type.DataType; @@ -453,7 +454,7 @@ private static EsField createField( // lack of parent implies the field is an alias if (map == null) { // as such, create the field manually, marking the field to also be an alias - fieldFunction = s -> createField(typeRegistry, s, OBJECT.esType(), new TreeMap<>(), false, true); + fieldFunction = s -> createField(typeRegistry, s, OBJECT.esType(), null, new TreeMap<>(), false, true); } else { Iterator iterator = map.values().iterator(); FieldCapabilities parentCap = iterator.next(); @@ -461,7 +462,15 @@ private static EsField createField( parentCap = iterator.next(); } final FieldCapabilities parentC = parentCap; - fieldFunction = s -> createField(typeRegistry, s, parentC.getType(), new TreeMap<>(), parentC.isAggregatable(), false); + fieldFunction = s -> createField( + typeRegistry, + s, + parentC.getType(), + parentC.getMetricType(), + new TreeMap<>(), + parentC.isAggregatable(), + false + ); } parent = createField(typeRegistry, parentName, globalCaps, hierarchicalMapping, flattedMapping, fieldFunction); @@ -495,11 +504,12 @@ private static EsField createField( DataTypeRegistry typeRegistry, String fieldName, String typeName, + TimeSeriesParams.MetricType metricType, Map props, boolean isAggregateable, boolean isAlias ) { - DataType esType = typeRegistry.fromEs(typeName); + DataType esType = typeRegistry.fromEs(typeName, metricType); if (esType == TEXT) { return new TextEsField(fieldName, props, false, isAlias); @@ -514,7 +524,8 @@ private static EsField createField( return DateEsField.dateEsField(fieldName, props, isAggregateable); } if (esType == UNSUPPORTED) { - return new UnsupportedEsField(fieldName, typeName, null, props); + String originalType = metricType == TimeSeriesParams.MetricType.COUNTER ? "counter" : typeName; + return new UnsupportedEsField(fieldName, originalType, null, props); } return new EsField(fieldName, esType, props, isAggregateable, isAlias); @@ -727,7 +738,15 @@ private static void createField( indexFields.flattedMapping, s -> invalidField != null ? invalidField - : createField(typeRegistry, s, typeCap.getType(), emptyMap(), typeCap.isAggregatable(), isAliasFieldType.get()) + : createField( + typeRegistry, + s, + typeCap.getType(), + typeCap.getMetricType(), + emptyMap(), + typeCap.isAggregatable(), + isAliasFieldType.get() + ) ); } diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/DataTypeRegistry.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/DataTypeRegistry.java index 6989d4572a974..712b7df103947 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/DataTypeRegistry.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/DataTypeRegistry.java @@ -7,6 +7,8 @@ package org.elasticsearch.xpack.ql.type; +import org.elasticsearch.index.mapper.TimeSeriesParams; + import java.util.Collection; /** @@ -19,7 +21,7 @@ public interface DataTypeRegistry { // Collection dataTypes(); - DataType fromEs(String typeName); + DataType fromEs(String typeName, TimeSeriesParams.MetricType metricType); DataType fromJava(Object value); diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/DefaultDataTypeRegistry.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/DefaultDataTypeRegistry.java index 3c37998191eaa..472f51870d01c 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/DefaultDataTypeRegistry.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/DefaultDataTypeRegistry.java @@ -7,6 +7,8 @@ package org.elasticsearch.xpack.ql.type; +import org.elasticsearch.index.mapper.TimeSeriesParams; + import java.util.Collection; public class DefaultDataTypeRegistry implements DataTypeRegistry { @@ -21,7 +23,7 @@ public Collection dataTypes() { } @Override - public DataType fromEs(String typeName) { + public DataType fromEs(String typeName, TimeSeriesParams.MetricType metricType) { return DataTypes.fromEs(typeName); } diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/Types.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/Types.java index c202e797b7566..a19f4c634f77c 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/Types.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/Types.java @@ -8,6 +8,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.core.Booleans; +import org.elasticsearch.index.mapper.TimeSeriesParams; import java.util.Collections; import java.util.LinkedHashMap; @@ -52,8 +53,9 @@ private static DataType getType(DataTypeRegistry typeRegistry, Map dataTypes() { } @Override - public DataType fromEs(String typeName) { + public DataType fromEs(String typeName, TimeSeriesParams.MetricType metricType) { return SqlDataTypes.fromEs(typeName); }