diff --git a/docs/reference/data-streams/tsds.asciidoc b/docs/reference/data-streams/tsds.asciidoc index 1f9a35a3df823..3f49a7ab8c700 100644 --- a/docs/reference/data-streams/tsds.asciidoc +++ b/docs/reference/data-streams/tsds.asciidoc @@ -155,7 +155,7 @@ of aggregations (for example `sum`) compute results that don't make sense for a + Only numeric and `aggregate_metric_double` fields support the `counter` metric type. -NOTE: Due to the cumulative nature of counter fields, only the following aggregations are allowed with the `counter` field: `rate`, `histogram`, `range`, `min`, `max`, `top_metrics` and `variable_width_histogram`. +NOTE: Due to the cumulative nature of counter fields, the following aggregations are supported and expected to provide meaningful results with the `counter` field: `rate`, `histogram`, `range`, `min`, `max`, `top_metrics` and `variable_width_histogram`. In order to prevent issues with existing integrations and custom dashboards, we also allow the following aggregations, even if the result might be meaningless on counters: `avg`, `box plot`, `cardinality`, `extended stats`, `median absolute deviation`, `percentile ranks`, `percentiles`, `stats`, `sum` and `value count`. // tag::time-series-metric-gauge[] `gauge`:: A metric that represents a single numeric that can arbitrarily increase or decrease. For example, a temperature or diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/120_counter_fields.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/120_counter_fields.yml index f993b18ddd8ab..5767252060bd4 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/120_counter_fields.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/120_counter_fields.yml @@ -1,53 +1,324 @@ --- -"avg aggregation on counter field": +setup: - skip: version: " - 8.9.99" - reason: "counter field support added in 8.7, but exception message changed in 8.10.0" + reason: "counter field support added in 8.10" + features: close_to - do: indices.create: - index: myindex1 - body: - mappings: - properties: - counter_field: - type : long - time_series_metric: counter - - do: - indices.create: - index: myindex2 + index: test_counter body: settings: index: mode: time_series - routing_path: [ keyword_field ] + routing_path: [ key ] time_series: - start_time: 2023-01-01T00:00:00Z - end_time: 2024-01-01T00:00:00Z + start_time: 2021-01-01T00:00:00Z + end_time: 2021-01-31T00:00:00Z + number_of_shards: 1 mappings: properties: - keyword_field: + "@timestamp": + type: date + key: type: keyword time_series_dimension: true - counter_field: - type : long + counter: + type: long time_series_metric: counter + weight: + type: integer + + - do: + bulk: + index: test_counter + refresh: true + body: + - '{ "index": {} }' + - '{ "@timestamp": "2021-01-01T00:01:00Z", "key": "bar", "counter": 10, "weight": 2 }' + - '{ "index": {} }' + - '{ "@timestamp": "2021-01-01T00:02:00Z", "key": "bar", "counter": 20, "weight": 1 }' + - '{ "index": {} }' + - '{ "@timestamp": "2021-01-01T00:03:00Z", "key": "bar", "counter": 22, "weight": 2 }' + - '{ "index": {} }' + - '{ "@timestamp": "2021-01-01T00:04:00Z", "key": "bar", "counter": 28, "weight": 1 }' + +--- +"avg": - do: search: - index: myindex1 + index: test_counter body: aggs: - the_counter_avg: + counter_avg: avg: - field: counter_field - - match: { aggregations.the_counter_avg.value: null } + field: counter + + - match: { hits.total.value: 4 } + - length: { hits.hits: 4 } + - close_to: { aggregations.counter_avg.value: { value: 20.00, error: 0.01 } } +--- +"cardinality": - do: - catch: /Field \[counter_field\] of type \[long\]\[counter\] is not supported for aggregation \[avg\]/ search: - index: myindex2 + index: test_counter body: aggs: - the_counter_avg: - avg: - field: counter_field + counter_cardinality: + cardinality: + field: counter + + - match: { hits.total.value: 4 } + - length: { hits.hits: 4 } + - match: { aggregations.counter_cardinality.value: 4 } + +--- +"extended stats": + - do: + search: + index: test_counter + body: + aggs: + counter_extended_stats: + extended_stats: + field: counter + + - match: { hits.total.value: 4 } + - length: { hits.hits: 4 } + - match: { aggregations.counter_extended_stats.count: 4 } + - close_to: { aggregations.counter_extended_stats.min: { value: 10.00, error: 0.01 } } + - close_to: { aggregations.counter_extended_stats.max: { value: 28.00, error: 0.01 } } + - close_to: { aggregations.counter_extended_stats.avg: { value: 20.00, error: 0.01 } } + - close_to: { aggregations.counter_extended_stats.sum: { value: 80.00, error: 0.01 } } + - close_to: { aggregations.counter_extended_stats.sum_of_squares: { value: 1768.00, error: 0.01 } } + - close_to: { aggregations.counter_extended_stats.std_deviation: { value: 6.48, error: 0.01 } } + - close_to: { aggregations.counter_extended_stats.std_deviation_bounds.upper: { value: 32.96, error: 0.01 } } + - close_to: { aggregations.counter_extended_stats.std_deviation_bounds.lower: { value: 7.03, error: 0.01 } } + +--- +"median absolute deviation": + - do: + search: + index: test_counter + body: + aggs: + mad_counter: + median_absolute_deviation: + field: counter + + - match: { hits.total.value: 4 } + - length: { hits.hits: 4 } + - close_to: { aggregations.mad_counter.value: { value: 4.00, error: 0.01 } } + +--- +"percentile ranks hdr": + - do: + search: + index: test_counter + body: + aggs: + counter_percentile_ranks: + percentile_ranks: + field: counter + values: [50, 90] + keyed: false + hdr: + number_of_significant_value_digits: 2 + + - match: { hits.total.value: 4 } + - length: { hits.hits: 4 } + - close_to: { aggregations.counter_percentile_ranks.values.0.value: { value: 100.00, error: 0.01 } } + - close_to: { aggregations.counter_percentile_ranks.values.1.value: { value: 100.00, error: 0.01 } } + +--- +"percentile ranks tdigest": + - do: + search: + index: test_counter + body: + aggs: + counter_percentiles: + percentiles: + field: counter + keyed: false + + - match: { hits.total.value: 4 } + - length: { hits.hits: 4 } + - close_to: { aggregations.counter_percentiles.values.0.value: { value: 10.30, error: 0.01 } } + - close_to: { aggregations.counter_percentiles.values.1.value: { value: 11.50, error: 0.01 } } + - close_to: { aggregations.counter_percentiles.values.2.value: { value: 17.50, error: 0.01 } } + - close_to: { aggregations.counter_percentiles.values.3.value: { value: 21.00, error: 0.01 } } + - close_to: { aggregations.counter_percentiles.values.4.value: { value: 23.50, error: 0.01 } } + - close_to: { aggregations.counter_percentiles.values.5.value: { value: 27.10, error: 0.01 } } + - close_to: { aggregations.counter_percentiles.values.6.value: { value: 27.82, error: 0.01 } } + +--- +"percentiles hdr": + - do: + search: + index: test_counter + body: + aggs: + counter_percentiles: + percentiles: + field: counter + keyed: false + hdr: + number_of_significant_value_digits: 2 + + - match: { hits.total.value: 4 } + - length: { hits.hits: 4 } + - close_to: { aggregations.counter_percentiles.values.0.value: { value: 10.00, error: 0.01 } } + - close_to: { aggregations.counter_percentiles.values.1.value: { value: 10.00, error: 0.01 } } + - close_to: { aggregations.counter_percentiles.values.2.value: { value: 10.00, error: 0.01 } } + - close_to: { aggregations.counter_percentiles.values.3.value: { value: 20.06, error: 0.01 } } + - close_to: { aggregations.counter_percentiles.values.4.value: { value: 22.06, error: 0.01 } } + - close_to: { aggregations.counter_percentiles.values.5.value: { value: 28.06, error: 0.01 } } + - close_to: { aggregations.counter_percentiles.values.6.value: { value: 28.06, error: 0.01 } } + +--- +"percentiles tdigest": + - do: + bulk: + index: test_counter + refresh: true + body: + - '{ "index": {} }' + - '{ "@timestamp": "2021-01-01T00:01:00Z", "key": "bar", "counter": 10 }' + - '{ "index": {} }' + - '{ "@timestamp": "2021-01-01T00:02:00Z", "key": "bar", "counter": 20 }' + - '{ "index": {} }' + - '{ "@timestamp": "2021-01-01T00:03:00Z", "key": "bar", "counter": 22 }' + - '{ "index": {} }' + - '{ "@timestamp": "2021-01-01T00:04:00Z", "key": "bar", "counter": 28 }' + + - do: + search: + index: test_counter + body: + aggs: + counter_percentiles: + percentiles: + field: counter + keyed: false + + - match: { hits.total.value: 4 } + - length: { hits.hits: 4 } + - close_to: { aggregations.counter_percentiles.values.0.value: { value: 10.30, error: 0.01 } } + - close_to: { aggregations.counter_percentiles.values.1.value: { value: 11.50, error: 0.01 } } + - close_to: { aggregations.counter_percentiles.values.2.value: { value: 17.50, error: 0.01 } } + - close_to: { aggregations.counter_percentiles.values.3.value: { value: 21.00, error: 0.01 } } + - close_to: { aggregations.counter_percentiles.values.4.value: { value: 23.50, error: 0.01 } } + - close_to: { aggregations.counter_percentiles.values.5.value: { value: 27.10, error: 0.01 } } + - close_to: { aggregations.counter_percentiles.values.6.value: { value: 27.82, error: 0.01 } } + +--- +"stats": + - do: + bulk: + index: test_counter + refresh: true + body: + - '{ "index": {} }' + - '{ "@timestamp": "2021-01-01T00:01:00Z", "key": "bar", "counter": 10 }' + - '{ "index": {} }' + - '{ "@timestamp": "2021-01-01T00:02:00Z", "key": "bar", "counter": 20 }' + - '{ "index": {} }' + - '{ "@timestamp": "2021-01-01T00:03:00Z", "key": "bar", "counter": 22 }' + - '{ "index": {} }' + - '{ "@timestamp": "2021-01-01T00:04:00Z", "key": "bar", "counter": 28 }' + + - do: + search: + index: test_counter + body: + aggs: + counter_extended_stats: + stats: + field: counter + + - match: { hits.total.value: 4 } + - length: { hits.hits: 4 } + - match: { aggregations.counter_extended_stats.count: 4 } + - close_to: { aggregations.counter_extended_stats.min: { value: 10.00, error: 0.01 } } + - close_to: { aggregations.counter_extended_stats.max: { value: 28.00, error: 0.01 } } + - close_to: { aggregations.counter_extended_stats.avg: { value: 20.00, error: 0.01 } } + - close_to: { aggregations.counter_extended_stats.sum: { value: 80.00, error: 0.01 } } + +--- +"sum": + - do: + bulk: + index: test_counter + refresh: true + body: + - '{ "index": {} }' + - '{ "@timestamp": "2021-01-01T00:01:00Z", "key": "bar", "counter": 10 }' + - '{ "index": {} }' + - '{ "@timestamp": "2021-01-01T00:02:00Z", "key": "bar", "counter": 20 }' + - '{ "index": {} }' + - '{ "@timestamp": "2021-01-01T00:03:00Z", "key": "bar", "counter": 22 }' + - '{ "index": {} }' + - '{ "@timestamp": "2021-01-01T00:04:00Z", "key": "bar", "counter": 28 }' + + - do: + search: + index: test_counter + body: + aggs: + counter_sum: + sum: + field: counter + + - match: { hits.total.value: 4 } + - length: { hits.hits: 4 } + - close_to: { aggregations.counter_sum.value: { value: 80.00, error: 0.01 } } + +--- +"value count": + - do: + bulk: + index: test_counter + refresh: true + body: + - '{ "index": {} }' + - '{ "@timestamp": "2021-01-01T00:01:00Z", "key": "bar", "counter": 10 }' + - '{ "index": {} }' + - '{ "@timestamp": "2021-01-01T00:02:00Z", "key": "bar", "counter": 20 }' + - '{ "index": {} }' + - '{ "@timestamp": "2021-01-01T00:03:00Z", "key": "bar", "counter": 22 }' + - '{ "index": {} }' + - '{ "@timestamp": "2021-01-01T00:04:00Z", "key": "bar", "counter": 28 }' + + - do: + search: + index: test_counter + body: + aggs: + counter_value_count: + value_count: + field: counter + + - match: { hits.total.value: 4 } + - length: { hits.hits: 4 } + - match: { aggregations.counter_value_count.value: 4 } + +--- +"weighted avg": + - do: + search: + index: test_counter + body: + aggs: + counter_weighted_avg: + weighted_avg: + value: + field: counter + weight: + field: weight + + - match: { hits.total.value: 4 } + - length: { hits.hits: 4 } + - close_to: { aggregations.counter_weighted_avg.value: { value: 18.66, error: 0.01 } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AvgAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AvgAggregatorFactory.java index 1ad0e02c003f1..8861467160591 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AvgAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AvgAggregatorFactory.java @@ -14,6 +14,7 @@ import org.elasticsearch.search.aggregations.CardinalityUpperBound; import org.elasticsearch.search.aggregations.support.AggregationContext; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.TimeSeriesValuesSourceType; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; @@ -41,7 +42,12 @@ class AvgAggregatorFactory extends ValuesSourceAggregatorFactory { static void registerAggregators(ValuesSourceRegistry.Builder builder) { builder.register( AvgAggregationBuilder.REGISTRY_KEY, - List.of(CoreValuesSourceType.NUMERIC, CoreValuesSourceType.DATE, CoreValuesSourceType.BOOLEAN), + List.of( + CoreValuesSourceType.NUMERIC, + CoreValuesSourceType.DATE, + CoreValuesSourceType.BOOLEAN, + TimeSeriesValuesSourceType.COUNTER + ), AvgAggregator::new, true ); diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/CardinalityAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/CardinalityAggregatorFactory.java index b2c5254bf2b5e..f27efafaf64cf 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/CardinalityAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/CardinalityAggregatorFactory.java @@ -15,6 +15,7 @@ import org.elasticsearch.search.aggregations.CardinalityUpperBound; import org.elasticsearch.search.aggregations.support.AggregationContext; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.TimeSeriesValuesSourceType; import org.elasticsearch.search.aggregations.support.ValuesSource; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; @@ -24,6 +25,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.stream.Stream; public class CardinalityAggregatorFactory extends ValuesSourceAggregatorFactory { @@ -151,7 +153,7 @@ public abstract boolean useGlobalOrdinals(AggregationContext context, ValuesSour public static void registerAggregators(ValuesSourceRegistry.Builder builder) { builder.register( CardinalityAggregationBuilder.REGISTRY_KEY, - CoreValuesSourceType.ALL_CORE, + Stream.concat(CoreValuesSourceType.ALL_CORE.stream(), Stream.of(TimeSeriesValuesSourceType.COUNTER)).toList(), (name, valuesSourceConfig, precision, executionMode, context, parent, metadata) -> { // check global ords if (valuesSourceConfig.hasValues()) { diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsAggregatorFactory.java index e200dabbf325e..13bdeb6e7e72d 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsAggregatorFactory.java @@ -14,6 +14,7 @@ import org.elasticsearch.search.aggregations.CardinalityUpperBound; import org.elasticsearch.search.aggregations.support.AggregationContext; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.TimeSeriesValuesSourceType; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; @@ -46,7 +47,12 @@ class ExtendedStatsAggregatorFactory extends ValuesSourceAggregatorFactory { static void registerAggregators(ValuesSourceRegistry.Builder builder) { builder.register( ExtendedStatsAggregationBuilder.REGISTRY_KEY, - List.of(CoreValuesSourceType.NUMERIC, CoreValuesSourceType.DATE, CoreValuesSourceType.BOOLEAN), + List.of( + CoreValuesSourceType.NUMERIC, + CoreValuesSourceType.DATE, + CoreValuesSourceType.BOOLEAN, + TimeSeriesValuesSourceType.COUNTER + ), ExtendedStatsAggregator::new, true ); diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MedianAbsoluteDeviationAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MedianAbsoluteDeviationAggregatorFactory.java index 91e96b17ff666..a38cafe253257 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MedianAbsoluteDeviationAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MedianAbsoluteDeviationAggregatorFactory.java @@ -14,11 +14,13 @@ import org.elasticsearch.search.aggregations.CardinalityUpperBound; import org.elasticsearch.search.aggregations.support.AggregationContext; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.TimeSeriesValuesSourceType; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; import java.io.IOException; +import java.util.List; import java.util.Map; public class MedianAbsoluteDeviationAggregatorFactory extends ValuesSourceAggregatorFactory { @@ -47,7 +49,7 @@ public class MedianAbsoluteDeviationAggregatorFactory extends ValuesSourceAggreg static void registerAggregators(ValuesSourceRegistry.Builder builder) { builder.register( MedianAbsoluteDeviationAggregationBuilder.REGISTRY_KEY, - CoreValuesSourceType.NUMERIC, + List.of(CoreValuesSourceType.NUMERIC, TimeSeriesValuesSourceType.COUNTER), MedianAbsoluteDeviationAggregator::new, true ); diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/PercentileRanksAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/PercentileRanksAggregatorFactory.java index 1d1da52716250..6dcf3644459fc 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/PercentileRanksAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/PercentileRanksAggregatorFactory.java @@ -14,6 +14,7 @@ import org.elasticsearch.search.aggregations.CardinalityUpperBound; import org.elasticsearch.search.aggregations.support.AggregationContext; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.TimeSeriesValuesSourceType; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; @@ -33,7 +34,12 @@ class PercentileRanksAggregatorFactory extends ValuesSourceAggregatorFactory { static void registerAggregators(ValuesSourceRegistry.Builder builder) { builder.register( PercentileRanksAggregationBuilder.REGISTRY_KEY, - List.of(CoreValuesSourceType.NUMERIC, CoreValuesSourceType.DATE, CoreValuesSourceType.BOOLEAN), + List.of( + CoreValuesSourceType.NUMERIC, + CoreValuesSourceType.DATE, + CoreValuesSourceType.BOOLEAN, + TimeSeriesValuesSourceType.COUNTER + ), (name, config, context, parent, percents, percentilesConfig, keyed, formatter, metadata) -> percentilesConfig .createPercentileRanksAggregator(name, config, context, parent, percents, keyed, formatter, metadata), true diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/PercentilesAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/PercentilesAggregatorFactory.java index 67c44e0dfb66a..673536ec3a1b3 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/PercentilesAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/PercentilesAggregatorFactory.java @@ -14,6 +14,7 @@ import org.elasticsearch.search.aggregations.CardinalityUpperBound; import org.elasticsearch.search.aggregations.support.AggregationContext; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.TimeSeriesValuesSourceType; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; @@ -37,7 +38,12 @@ class PercentilesAggregatorFactory extends ValuesSourceAggregatorFactory { static void registerAggregators(ValuesSourceRegistry.Builder builder) { builder.register( PercentilesAggregationBuilder.REGISTRY_KEY, - List.of(CoreValuesSourceType.NUMERIC, CoreValuesSourceType.DATE, CoreValuesSourceType.BOOLEAN), + List.of( + CoreValuesSourceType.NUMERIC, + CoreValuesSourceType.DATE, + CoreValuesSourceType.BOOLEAN, + TimeSeriesValuesSourceType.COUNTER + ), (name, config, context, parent, percents, percentilesConfig, keyed, formatter, metadata) -> percentilesConfig .createPercentilesAggregator(name, config, context, parent, percents, keyed, formatter, metadata), true diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/StatsAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/StatsAggregatorFactory.java index ffaada37238ff..797a0f69454fe 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/StatsAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/StatsAggregatorFactory.java @@ -14,6 +14,7 @@ import org.elasticsearch.search.aggregations.CardinalityUpperBound; import org.elasticsearch.search.aggregations.support.AggregationContext; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.TimeSeriesValuesSourceType; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; @@ -44,7 +45,12 @@ class StatsAggregatorFactory extends ValuesSourceAggregatorFactory { static void registerAggregators(ValuesSourceRegistry.Builder builder) { builder.register( StatsAggregationBuilder.REGISTRY_KEY, - List.of(CoreValuesSourceType.NUMERIC, CoreValuesSourceType.DATE, CoreValuesSourceType.BOOLEAN), + List.of( + CoreValuesSourceType.NUMERIC, + CoreValuesSourceType.DATE, + CoreValuesSourceType.BOOLEAN, + TimeSeriesValuesSourceType.COUNTER + ), StatsAggregator::new, true ); diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/SumAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/SumAggregatorFactory.java index 875cdf56fac33..7aef869d580b9 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/SumAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/SumAggregatorFactory.java @@ -14,6 +14,7 @@ import org.elasticsearch.search.aggregations.CardinalityUpperBound; import org.elasticsearch.search.aggregations.support.AggregationContext; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.TimeSeriesValuesSourceType; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; @@ -43,7 +44,12 @@ class SumAggregatorFactory extends ValuesSourceAggregatorFactory { static void registerAggregators(ValuesSourceRegistry.Builder builder) { builder.register( SumAggregationBuilder.REGISTRY_KEY, - List.of(CoreValuesSourceType.NUMERIC, CoreValuesSourceType.DATE, CoreValuesSourceType.BOOLEAN), + List.of( + CoreValuesSourceType.NUMERIC, + CoreValuesSourceType.DATE, + CoreValuesSourceType.BOOLEAN, + TimeSeriesValuesSourceType.COUNTER + ), SumAggregator::new, true ); diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ValueCountAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ValueCountAggregatorFactory.java index 9868913d63c2a..10f8bf198b7fe 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ValueCountAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ValueCountAggregatorFactory.java @@ -14,19 +14,26 @@ import org.elasticsearch.search.aggregations.CardinalityUpperBound; import org.elasticsearch.search.aggregations.support.AggregationContext; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.TimeSeriesValuesSourceType; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; import java.io.IOException; import java.util.Map; +import java.util.stream.Stream; class ValueCountAggregatorFactory extends ValuesSourceAggregatorFactory { private final MetricAggregatorSupplier aggregatorSupplier; public static void registerAggregators(ValuesSourceRegistry.Builder builder) { - builder.register(ValueCountAggregationBuilder.REGISTRY_KEY, CoreValuesSourceType.ALL_CORE, ValueCountAggregator::new, true); + builder.register( + ValueCountAggregationBuilder.REGISTRY_KEY, + Stream.concat(CoreValuesSourceType.ALL_CORE.stream(), Stream.of(TimeSeriesValuesSourceType.COUNTER)).toList(), + ValueCountAggregator::new, + true + ); } ValueCountAggregatorFactory( diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/boxplot/BoxplotAggregatorFactory.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/boxplot/BoxplotAggregatorFactory.java index ae057f7a37a75..cf62b07d6c97c 100644 --- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/boxplot/BoxplotAggregatorFactory.java +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/boxplot/BoxplotAggregatorFactory.java @@ -15,6 +15,7 @@ import org.elasticsearch.search.aggregations.metrics.TDigestExecutionHint; import org.elasticsearch.search.aggregations.support.AggregationContext; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.TimeSeriesValuesSourceType; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; @@ -34,7 +35,7 @@ public class BoxplotAggregatorFactory extends ValuesSourceAggregatorFactory { static void registerAggregators(ValuesSourceRegistry.Builder builder) { builder.register( BoxplotAggregationBuilder.REGISTRY_KEY, - List.of(CoreValuesSourceType.NUMERIC, AnalyticsValuesSourceType.HISTOGRAM), + List.of(CoreValuesSourceType.NUMERIC, AnalyticsValuesSourceType.HISTOGRAM, TimeSeriesValuesSourceType.COUNTER), BoxplotAggregator::new, true ); diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/boxplot.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/boxplot.yml index d98b2a92b8106..b65ab8a771c82 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/boxplot.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/boxplot.yml @@ -192,3 +192,66 @@ setup: - match: { aggregations.plot.q3: 13.75 } - match: { aggregations.plot.lower: 2.0 } - match: { aggregations.plot.upper: 15.0 } + +--- +"Counter field": + - skip: + version: " - 8.9.99" + reason: "counter field support added in 8.10" + features: close_to + + - do: + indices.create: + index: test_counter + body: + settings: + index: + mode: time_series + routing_path: [ key ] + time_series: + start_time: 2021-01-01T00:00:00Z + end_time: 2021-01-31T00:00:00Z + number_of_shards: 1 + mappings: + properties: + "@timestamp": + type: date + key: + type: keyword + time_series_dimension: true + counter: + type: long + time_series_metric: counter + + - do: + bulk: + index: test_counter + refresh: true + body: + - '{ "index": {} }' + - '{ "@timestamp": "2021-01-01T00:01:00Z", "key": "bar", "counter": 10 }' + - '{ "index": {} }' + - '{ "@timestamp": "2021-01-01T00:02:00Z", "key": "bar", "counter": 20 }' + - '{ "index": {} }' + - '{ "@timestamp": "2021-01-01T00:03:00Z", "key": "bar", "counter": 22 }' + - '{ "index": {} }' + - '{ "@timestamp": "2021-01-01T00:04:00Z", "key": "bar", "counter": 28 }' + + - do: + search: + index: test_counter + body: + aggs: + counter_boxplot: + boxplot: + field: counter + + - match: { hits.total.value: 4 } + - length: { hits.hits: 4 } + - close_to: { aggregations.counter_boxplot.min: { value: 10.00, error: 0.01 } } + - close_to: { aggregations.counter_boxplot.max: { value: 28.00, error: 0.01 } } + - close_to: { aggregations.counter_boxplot.q1: { value: 17.50, error: 0.01 } } + - close_to: { aggregations.counter_boxplot.q2: { value: 21.00, error: 0.01 } } + - close_to: { aggregations.counter_boxplot.q3: { value: 23.50, error: 0.01 } } + - close_to: { aggregations.counter_boxplot.lower: { value: 10.00, error: 0.01 } } + - close_to: { aggregations.counter_boxplot.upper: { value: 28.00, error: 0.01 } } diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/matrix_stats.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/matrix_stats.yml index 5dfff391e5b4c..4544a9beed307 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/matrix_stats.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/matrix_stats.yml @@ -206,3 +206,78 @@ setup: - close_to: { aggregations.matrix.buckets.0.stats.fields.1.covariance.value2: { value: 0.0, error: 0.0001 } } - close_to: { aggregations.matrix.buckets.0.stats.fields.1.correlation.value1: { value: 1.0, error: 0.0001 } } - close_to: { aggregations.matrix.buckets.0.stats.fields.1.correlation.value2: { value: 0.0, error: 0.0001 } } + +--- +"Counter field": + - skip: + version: " - 8.9.99" + reason: "counter field support added in 8.10" + features: close_to + + - do: + indices.create: + index: test_counter + body: + settings: + index: + mode: time_series + routing_path: [ key ] + time_series: + start_time: 2021-01-01T00:00:00Z + end_time: 2021-01-31T00:00:00Z + number_of_shards: 1 + mappings: + properties: + "@timestamp": + type: date + key: + type: keyword + time_series_dimension: true + counter1: + type: long + time_series_metric: counter + counter2: + type: long + time_series_metric: counter + + - do: + bulk: + index: test_counter + refresh: true + body: + - '{ "index": {} }' + - '{ "@timestamp": "2021-01-01T00:01:00Z", "key": "bar", "counter1": 10, "counter2": 7 }' + - '{ "index": {} }' + - '{ "@timestamp": "2021-01-01T00:02:00Z", "key": "bar", "counter1": 20, "counter2": 10 }' + - '{ "index": {} }' + - '{ "@timestamp": "2021-01-01T00:03:00Z", "key": "bar", "counter1": 22, "counter2": 12 }' + - '{ "index": {} }' + - '{ "@timestamp": "2021-01-01T00:04:00Z", "key": "bar", "counter1": 28, "counter2": 13 }' + + - do: + search: + index: test_counter + body: + aggs: + counter_matrix_stats: + matrix_stats: + fields: [ "counter1", "counter2" ] + + - match: { hits.total.value: 4 } + - length: { hits.hits: 4 } + - match: { aggregations.counter_matrix_stats.fields.0.name: "counter2" } + - match: { aggregations.counter_matrix_stats.fields.0.count: 4 } + - close_to: { aggregations.counter_matrix_stats.fields.0.mean: { value: 10.50, error: 0.01 } } + - close_to: { aggregations.counter_matrix_stats.fields.0.variance: { value: 7.00, error: 0.01 } } + - close_to: { aggregations.counter_matrix_stats.fields.0.covariance.counter2: { value: 7.00, error: 0.01 } } + - close_to: { aggregations.counter_matrix_stats.fields.0.covariance.counter1: { value: 19.33, error: 0.01 } } + - close_to: { aggregations.counter_matrix_stats.fields.0.correlation.counter2: { value: 1.00, error: 0.01 } } + - close_to: { aggregations.counter_matrix_stats.fields.0.correlation.counter1: { value: 0.97, error: 0.01 } } + - match: { aggregations.counter_matrix_stats.fields.1.name: "counter1" } + - match: { aggregations.counter_matrix_stats.fields.1.count: 4 } + - close_to: { aggregations.counter_matrix_stats.fields.1.mean: { value: 20.00, error: 0.01 } } + - close_to: { aggregations.counter_matrix_stats.fields.1.variance: { value: 56.00, error: 0.01 } } + - close_to: { aggregations.counter_matrix_stats.fields.1.covariance.counter2: { value: 19.33, error: 0.01 } } + - close_to: { aggregations.counter_matrix_stats.fields.1.covariance.counter1: { value: 56.00, error: 0.01 } } + - close_to: { aggregations.counter_matrix_stats.fields.1.correlation.counter2: { value: 0.97, error: 0.01 } } + - close_to: { aggregations.counter_matrix_stats.fields.1.correlation.counter1: { value: 1.00, error: 0.01 } }