From d0b2f650df79bd343b396d31fc57575437ed1364 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna <93581129+salvatore-campagna@users.noreply.github.com> Date: Tue, 8 Aug 2023 17:47:47 +0200 Subject: [PATCH] Enable all remaining metric aggregations on counters (#97974) Here we enable aggregations previously not allowed on fields of type counter. The decision of enabling such aggregations even if the result is "meaningless" for counters has been taken to favour TSDB adoption. Aggregations now allowed, other than the existing ones, include: * avg * box plot * cardinality * extended stats * median absolute deviation * percentile ranks * percentiles * stats * sum * value count I included tests for the weighted average and matrix stats aggregations too. Resolves #97882 --- docs/reference/data-streams/tsds.asciidoc | 2 +- .../test/tsdb/120_counter_fields.yml | 325 ++++++++++++++++-- .../metrics/AvgAggregatorFactory.java | 8 +- .../metrics/CardinalityAggregatorFactory.java | 4 +- .../ExtendedStatsAggregatorFactory.java | 8 +- ...ianAbsoluteDeviationAggregatorFactory.java | 4 +- .../PercentileRanksAggregatorFactory.java | 8 +- .../metrics/PercentilesAggregatorFactory.java | 8 +- .../metrics/StatsAggregatorFactory.java | 8 +- .../metrics/SumAggregatorFactory.java | 8 +- .../metrics/ValueCountAggregatorFactory.java | 9 +- .../boxplot/BoxplotAggregatorFactory.java | 3 +- .../rest-api-spec/test/analytics/boxplot.yml | 63 ++++ .../test/analytics/matrix_stats.yml | 75 ++++ 14 files changed, 495 insertions(+), 38 deletions(-) diff --git a/docs/reference/data-streams/tsds.asciidoc b/docs/reference/data-streams/tsds.asciidoc index 1f9a35a3df823..3f49a7ab8c700 100644 --- a/docs/reference/data-streams/tsds.asciidoc +++ b/docs/reference/data-streams/tsds.asciidoc @@ -155,7 +155,7 @@ of aggregations (for example `sum`) compute results that don't make sense for a + Only numeric and `aggregate_metric_double` fields support the `counter` metric type. -NOTE: Due to the cumulative nature of counter fields, only the following aggregations are allowed with the `counter` field: `rate`, `histogram`, `range`, `min`, `max`, `top_metrics` and `variable_width_histogram`. +NOTE: Due to the cumulative nature of counter fields, the following aggregations are supported and expected to provide meaningful results with the `counter` field: `rate`, `histogram`, `range`, `min`, `max`, `top_metrics` and `variable_width_histogram`. In order to prevent issues with existing integrations and custom dashboards, we also allow the following aggregations, even if the result might be meaningless on counters: `avg`, `box plot`, `cardinality`, `extended stats`, `median absolute deviation`, `percentile ranks`, `percentiles`, `stats`, `sum` and `value count`. // tag::time-series-metric-gauge[] `gauge`:: A metric that represents a single numeric that can arbitrarily increase or decrease. For example, a temperature or diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/120_counter_fields.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/120_counter_fields.yml index f993b18ddd8ab..5767252060bd4 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/120_counter_fields.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/120_counter_fields.yml @@ -1,53 +1,324 @@ --- -"avg aggregation on counter field": +setup: - skip: version: " - 8.9.99" - reason: "counter field support added in 8.7, but exception message changed in 8.10.0" + reason: "counter field support added in 8.10" + features: close_to - do: indices.create: - index: myindex1 - body: - mappings: - properties: - counter_field: - type : long - time_series_metric: counter - - do: - indices.create: - index: myindex2 + index: test_counter body: settings: index: mode: time_series - routing_path: [ keyword_field ] + routing_path: [ key ] time_series: - start_time: 2023-01-01T00:00:00Z - end_time: 2024-01-01T00:00:00Z + start_time: 2021-01-01T00:00:00Z + end_time: 2021-01-31T00:00:00Z + number_of_shards: 1 mappings: properties: - keyword_field: + "@timestamp": + type: date + key: type: keyword time_series_dimension: true - counter_field: - type : long + counter: + type: long time_series_metric: counter + weight: + type: integer + + - do: + bulk: + index: test_counter + refresh: true + body: + - '{ "index": {} }' + - '{ "@timestamp": "2021-01-01T00:01:00Z", "key": "bar", "counter": 10, "weight": 2 }' + - '{ "index": {} }' + - '{ "@timestamp": "2021-01-01T00:02:00Z", "key": "bar", "counter": 20, "weight": 1 }' + - '{ "index": {} }' + - '{ "@timestamp": "2021-01-01T00:03:00Z", "key": "bar", "counter": 22, "weight": 2 }' + - '{ "index": {} }' + - '{ "@timestamp": "2021-01-01T00:04:00Z", "key": "bar", "counter": 28, "weight": 1 }' + +--- +"avg": - do: search: - index: myindex1 + index: test_counter body: aggs: - the_counter_avg: + counter_avg: avg: - field: counter_field - - match: { aggregations.the_counter_avg.value: null } + field: counter + + - match: { hits.total.value: 4 } + - length: { hits.hits: 4 } + - close_to: { aggregations.counter_avg.value: { value: 20.00, error: 0.01 } } +--- +"cardinality": - do: - catch: /Field \[counter_field\] of type \[long\]\[counter\] is not supported for aggregation \[avg\]/ search: - index: myindex2 + index: test_counter body: aggs: - the_counter_avg: - avg: - field: counter_field + counter_cardinality: + cardinality: + field: counter + + - match: { hits.total.value: 4 } + - length: { hits.hits: 4 } + - match: { aggregations.counter_cardinality.value: 4 } + +--- +"extended stats": + - do: + search: + index: test_counter + body: + aggs: + counter_extended_stats: + extended_stats: + field: counter + + - match: { hits.total.value: 4 } + - length: { hits.hits: 4 } + - match: { aggregations.counter_extended_stats.count: 4 } + - close_to: { aggregations.counter_extended_stats.min: { value: 10.00, error: 0.01 } } + - close_to: { aggregations.counter_extended_stats.max: { value: 28.00, error: 0.01 } } + - close_to: { aggregations.counter_extended_stats.avg: { value: 20.00, error: 0.01 } } + - close_to: { aggregations.counter_extended_stats.sum: { value: 80.00, error: 0.01 } } + - close_to: { aggregations.counter_extended_stats.sum_of_squares: { value: 1768.00, error: 0.01 } } + - close_to: { aggregations.counter_extended_stats.std_deviation: { value: 6.48, error: 0.01 } } + - close_to: { aggregations.counter_extended_stats.std_deviation_bounds.upper: { value: 32.96, error: 0.01 } } + - close_to: { aggregations.counter_extended_stats.std_deviation_bounds.lower: { value: 7.03, error: 0.01 } } + +--- +"median absolute deviation": + - do: + search: + index: test_counter + body: + aggs: + mad_counter: + median_absolute_deviation: + field: counter + + - match: { hits.total.value: 4 } + - length: { hits.hits: 4 } + - close_to: { aggregations.mad_counter.value: { value: 4.00, error: 0.01 } } + +--- +"percentile ranks hdr": + - do: + search: + index: test_counter + body: + aggs: + counter_percentile_ranks: + percentile_ranks: + field: counter + values: [50, 90] + keyed: false + hdr: + number_of_significant_value_digits: 2 + + - match: { hits.total.value: 4 } + - length: { hits.hits: 4 } + - close_to: { aggregations.counter_percentile_ranks.values.0.value: { value: 100.00, error: 0.01 } } + - close_to: { aggregations.counter_percentile_ranks.values.1.value: { value: 100.00, error: 0.01 } } + +--- +"percentile ranks tdigest": + - do: + search: + index: test_counter + body: + aggs: + counter_percentiles: + percentiles: + field: counter + keyed: false + + - match: { hits.total.value: 4 } + - length: { hits.hits: 4 } + - close_to: { aggregations.counter_percentiles.values.0.value: { value: 10.30, error: 0.01 } } + - close_to: { aggregations.counter_percentiles.values.1.value: { value: 11.50, error: 0.01 } } + - close_to: { aggregations.counter_percentiles.values.2.value: { value: 17.50, error: 0.01 } } + - close_to: { aggregations.counter_percentiles.values.3.value: { value: 21.00, error: 0.01 } } + - close_to: { aggregations.counter_percentiles.values.4.value: { value: 23.50, error: 0.01 } } + - close_to: { aggregations.counter_percentiles.values.5.value: { value: 27.10, error: 0.01 } } + - close_to: { aggregations.counter_percentiles.values.6.value: { value: 27.82, error: 0.01 } } + +--- +"percentiles hdr": + - do: + search: + index: test_counter + body: + aggs: + counter_percentiles: + percentiles: + field: counter + keyed: false + hdr: + number_of_significant_value_digits: 2 + + - match: { hits.total.value: 4 } + - length: { hits.hits: 4 } + - close_to: { aggregations.counter_percentiles.values.0.value: { value: 10.00, error: 0.01 } } + - close_to: { aggregations.counter_percentiles.values.1.value: { value: 10.00, error: 0.01 } } + - close_to: { aggregations.counter_percentiles.values.2.value: { value: 10.00, error: 0.01 } } + - close_to: { aggregations.counter_percentiles.values.3.value: { value: 20.06, error: 0.01 } } + - close_to: { aggregations.counter_percentiles.values.4.value: { value: 22.06, error: 0.01 } } + - close_to: { aggregations.counter_percentiles.values.5.value: { value: 28.06, error: 0.01 } } + - close_to: { aggregations.counter_percentiles.values.6.value: { value: 28.06, error: 0.01 } } + +--- +"percentiles tdigest": + - do: + bulk: + index: test_counter + refresh: true + body: + - '{ "index": {} }' + - '{ "@timestamp": "2021-01-01T00:01:00Z", "key": "bar", "counter": 10 }' + - '{ "index": {} }' + - '{ "@timestamp": "2021-01-01T00:02:00Z", "key": "bar", "counter": 20 }' + - '{ "index": {} }' + - '{ "@timestamp": "2021-01-01T00:03:00Z", "key": "bar", "counter": 22 }' + - '{ "index": {} }' + - '{ "@timestamp": "2021-01-01T00:04:00Z", "key": "bar", "counter": 28 }' + + - do: + search: + index: test_counter + body: + aggs: + counter_percentiles: + percentiles: + field: counter + keyed: false + + - match: { hits.total.value: 4 } + - length: { hits.hits: 4 } + - close_to: { aggregations.counter_percentiles.values.0.value: { value: 10.30, error: 0.01 } } + - close_to: { aggregations.counter_percentiles.values.1.value: { value: 11.50, error: 0.01 } } + - close_to: { aggregations.counter_percentiles.values.2.value: { value: 17.50, error: 0.01 } } + - close_to: { aggregations.counter_percentiles.values.3.value: { value: 21.00, error: 0.01 } } + - close_to: { aggregations.counter_percentiles.values.4.value: { value: 23.50, error: 0.01 } } + - close_to: { aggregations.counter_percentiles.values.5.value: { value: 27.10, error: 0.01 } } + - close_to: { aggregations.counter_percentiles.values.6.value: { value: 27.82, error: 0.01 } } + +--- +"stats": + - do: + bulk: + index: test_counter + refresh: true + body: + - '{ "index": {} }' + - '{ "@timestamp": "2021-01-01T00:01:00Z", "key": "bar", "counter": 10 }' + - '{ "index": {} }' + - '{ "@timestamp": "2021-01-01T00:02:00Z", "key": "bar", "counter": 20 }' + - '{ "index": {} }' + - '{ "@timestamp": "2021-01-01T00:03:00Z", "key": "bar", "counter": 22 }' + - '{ "index": {} }' + - '{ "@timestamp": "2021-01-01T00:04:00Z", "key": "bar", "counter": 28 }' + + - do: + search: + index: test_counter + body: + aggs: + counter_extended_stats: + stats: + field: counter + + - match: { hits.total.value: 4 } + - length: { hits.hits: 4 } + - match: { aggregations.counter_extended_stats.count: 4 } + - close_to: { aggregations.counter_extended_stats.min: { value: 10.00, error: 0.01 } } + - close_to: { aggregations.counter_extended_stats.max: { value: 28.00, error: 0.01 } } + - close_to: { aggregations.counter_extended_stats.avg: { value: 20.00, error: 0.01 } } + - close_to: { aggregations.counter_extended_stats.sum: { value: 80.00, error: 0.01 } } + +--- +"sum": + - do: + bulk: + index: test_counter + refresh: true + body: + - '{ "index": {} }' + - '{ "@timestamp": "2021-01-01T00:01:00Z", "key": "bar", "counter": 10 }' + - '{ "index": {} }' + - '{ "@timestamp": "2021-01-01T00:02:00Z", "key": "bar", "counter": 20 }' + - '{ "index": {} }' + - '{ "@timestamp": "2021-01-01T00:03:00Z", "key": "bar", "counter": 22 }' + - '{ "index": {} }' + - '{ "@timestamp": "2021-01-01T00:04:00Z", "key": "bar", "counter": 28 }' + + - do: + search: + index: test_counter + body: + aggs: + counter_sum: + sum: + field: counter + + - match: { hits.total.value: 4 } + - length: { hits.hits: 4 } + - close_to: { aggregations.counter_sum.value: { value: 80.00, error: 0.01 } } + +--- +"value count": + - do: + bulk: + index: test_counter + refresh: true + body: + - '{ "index": {} }' + - '{ "@timestamp": "2021-01-01T00:01:00Z", "key": "bar", "counter": 10 }' + - '{ "index": {} }' + - '{ "@timestamp": "2021-01-01T00:02:00Z", "key": "bar", "counter": 20 }' + - '{ "index": {} }' + - '{ "@timestamp": "2021-01-01T00:03:00Z", "key": "bar", "counter": 22 }' + - '{ "index": {} }' + - '{ "@timestamp": "2021-01-01T00:04:00Z", "key": "bar", "counter": 28 }' + + - do: + search: + index: test_counter + body: + aggs: + counter_value_count: + value_count: + field: counter + + - match: { hits.total.value: 4 } + - length: { hits.hits: 4 } + - match: { aggregations.counter_value_count.value: 4 } + +--- +"weighted avg": + - do: + search: + index: test_counter + body: + aggs: + counter_weighted_avg: + weighted_avg: + value: + field: counter + weight: + field: weight + + - match: { hits.total.value: 4 } + - length: { hits.hits: 4 } + - close_to: { aggregations.counter_weighted_avg.value: { value: 18.66, error: 0.01 } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AvgAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AvgAggregatorFactory.java index 1ad0e02c003f1..8861467160591 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AvgAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AvgAggregatorFactory.java @@ -14,6 +14,7 @@ import org.elasticsearch.search.aggregations.CardinalityUpperBound; import org.elasticsearch.search.aggregations.support.AggregationContext; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.TimeSeriesValuesSourceType; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; @@ -41,7 +42,12 @@ class AvgAggregatorFactory extends ValuesSourceAggregatorFactory { static void registerAggregators(ValuesSourceRegistry.Builder builder) { builder.register( AvgAggregationBuilder.REGISTRY_KEY, - List.of(CoreValuesSourceType.NUMERIC, CoreValuesSourceType.DATE, CoreValuesSourceType.BOOLEAN), + List.of( + CoreValuesSourceType.NUMERIC, + CoreValuesSourceType.DATE, + CoreValuesSourceType.BOOLEAN, + TimeSeriesValuesSourceType.COUNTER + ), AvgAggregator::new, true ); diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/CardinalityAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/CardinalityAggregatorFactory.java index b2c5254bf2b5e..f27efafaf64cf 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/CardinalityAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/CardinalityAggregatorFactory.java @@ -15,6 +15,7 @@ import org.elasticsearch.search.aggregations.CardinalityUpperBound; import org.elasticsearch.search.aggregations.support.AggregationContext; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.TimeSeriesValuesSourceType; import org.elasticsearch.search.aggregations.support.ValuesSource; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; @@ -24,6 +25,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.stream.Stream; public class CardinalityAggregatorFactory extends ValuesSourceAggregatorFactory { @@ -151,7 +153,7 @@ public abstract boolean useGlobalOrdinals(AggregationContext context, ValuesSour public static void registerAggregators(ValuesSourceRegistry.Builder builder) { builder.register( CardinalityAggregationBuilder.REGISTRY_KEY, - CoreValuesSourceType.ALL_CORE, + Stream.concat(CoreValuesSourceType.ALL_CORE.stream(), Stream.of(TimeSeriesValuesSourceType.COUNTER)).toList(), (name, valuesSourceConfig, precision, executionMode, context, parent, metadata) -> { // check global ords if (valuesSourceConfig.hasValues()) { diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsAggregatorFactory.java index e200dabbf325e..13bdeb6e7e72d 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsAggregatorFactory.java @@ -14,6 +14,7 @@ import org.elasticsearch.search.aggregations.CardinalityUpperBound; import org.elasticsearch.search.aggregations.support.AggregationContext; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.TimeSeriesValuesSourceType; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; @@ -46,7 +47,12 @@ class ExtendedStatsAggregatorFactory extends ValuesSourceAggregatorFactory { static void registerAggregators(ValuesSourceRegistry.Builder builder) { builder.register( ExtendedStatsAggregationBuilder.REGISTRY_KEY, - List.of(CoreValuesSourceType.NUMERIC, CoreValuesSourceType.DATE, CoreValuesSourceType.BOOLEAN), + List.of( + CoreValuesSourceType.NUMERIC, + CoreValuesSourceType.DATE, + CoreValuesSourceType.BOOLEAN, + TimeSeriesValuesSourceType.COUNTER + ), ExtendedStatsAggregator::new, true ); diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MedianAbsoluteDeviationAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MedianAbsoluteDeviationAggregatorFactory.java index 91e96b17ff666..a38cafe253257 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MedianAbsoluteDeviationAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MedianAbsoluteDeviationAggregatorFactory.java @@ -14,11 +14,13 @@ import org.elasticsearch.search.aggregations.CardinalityUpperBound; import org.elasticsearch.search.aggregations.support.AggregationContext; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.TimeSeriesValuesSourceType; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; import java.io.IOException; +import java.util.List; import java.util.Map; public class MedianAbsoluteDeviationAggregatorFactory extends ValuesSourceAggregatorFactory { @@ -47,7 +49,7 @@ public class MedianAbsoluteDeviationAggregatorFactory extends ValuesSourceAggreg static void registerAggregators(ValuesSourceRegistry.Builder builder) { builder.register( MedianAbsoluteDeviationAggregationBuilder.REGISTRY_KEY, - CoreValuesSourceType.NUMERIC, + List.of(CoreValuesSourceType.NUMERIC, TimeSeriesValuesSourceType.COUNTER), MedianAbsoluteDeviationAggregator::new, true ); diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/PercentileRanksAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/PercentileRanksAggregatorFactory.java index 1d1da52716250..6dcf3644459fc 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/PercentileRanksAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/PercentileRanksAggregatorFactory.java @@ -14,6 +14,7 @@ import org.elasticsearch.search.aggregations.CardinalityUpperBound; import org.elasticsearch.search.aggregations.support.AggregationContext; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.TimeSeriesValuesSourceType; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; @@ -33,7 +34,12 @@ class PercentileRanksAggregatorFactory extends ValuesSourceAggregatorFactory { static void registerAggregators(ValuesSourceRegistry.Builder builder) { builder.register( PercentileRanksAggregationBuilder.REGISTRY_KEY, - List.of(CoreValuesSourceType.NUMERIC, CoreValuesSourceType.DATE, CoreValuesSourceType.BOOLEAN), + List.of( + CoreValuesSourceType.NUMERIC, + CoreValuesSourceType.DATE, + CoreValuesSourceType.BOOLEAN, + TimeSeriesValuesSourceType.COUNTER + ), (name, config, context, parent, percents, percentilesConfig, keyed, formatter, metadata) -> percentilesConfig .createPercentileRanksAggregator(name, config, context, parent, percents, keyed, formatter, metadata), true diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/PercentilesAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/PercentilesAggregatorFactory.java index 67c44e0dfb66a..673536ec3a1b3 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/PercentilesAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/PercentilesAggregatorFactory.java @@ -14,6 +14,7 @@ import org.elasticsearch.search.aggregations.CardinalityUpperBound; import org.elasticsearch.search.aggregations.support.AggregationContext; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.TimeSeriesValuesSourceType; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; @@ -37,7 +38,12 @@ class PercentilesAggregatorFactory extends ValuesSourceAggregatorFactory { static void registerAggregators(ValuesSourceRegistry.Builder builder) { builder.register( PercentilesAggregationBuilder.REGISTRY_KEY, - List.of(CoreValuesSourceType.NUMERIC, CoreValuesSourceType.DATE, CoreValuesSourceType.BOOLEAN), + List.of( + CoreValuesSourceType.NUMERIC, + CoreValuesSourceType.DATE, + CoreValuesSourceType.BOOLEAN, + TimeSeriesValuesSourceType.COUNTER + ), (name, config, context, parent, percents, percentilesConfig, keyed, formatter, metadata) -> percentilesConfig .createPercentilesAggregator(name, config, context, parent, percents, keyed, formatter, metadata), true diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/StatsAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/StatsAggregatorFactory.java index ffaada37238ff..797a0f69454fe 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/StatsAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/StatsAggregatorFactory.java @@ -14,6 +14,7 @@ import org.elasticsearch.search.aggregations.CardinalityUpperBound; import org.elasticsearch.search.aggregations.support.AggregationContext; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.TimeSeriesValuesSourceType; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; @@ -44,7 +45,12 @@ class StatsAggregatorFactory extends ValuesSourceAggregatorFactory { static void registerAggregators(ValuesSourceRegistry.Builder builder) { builder.register( StatsAggregationBuilder.REGISTRY_KEY, - List.of(CoreValuesSourceType.NUMERIC, CoreValuesSourceType.DATE, CoreValuesSourceType.BOOLEAN), + List.of( + CoreValuesSourceType.NUMERIC, + CoreValuesSourceType.DATE, + CoreValuesSourceType.BOOLEAN, + TimeSeriesValuesSourceType.COUNTER + ), StatsAggregator::new, true ); diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/SumAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/SumAggregatorFactory.java index 875cdf56fac33..7aef869d580b9 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/SumAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/SumAggregatorFactory.java @@ -14,6 +14,7 @@ import org.elasticsearch.search.aggregations.CardinalityUpperBound; import org.elasticsearch.search.aggregations.support.AggregationContext; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.TimeSeriesValuesSourceType; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; @@ -43,7 +44,12 @@ class SumAggregatorFactory extends ValuesSourceAggregatorFactory { static void registerAggregators(ValuesSourceRegistry.Builder builder) { builder.register( SumAggregationBuilder.REGISTRY_KEY, - List.of(CoreValuesSourceType.NUMERIC, CoreValuesSourceType.DATE, CoreValuesSourceType.BOOLEAN), + List.of( + CoreValuesSourceType.NUMERIC, + CoreValuesSourceType.DATE, + CoreValuesSourceType.BOOLEAN, + TimeSeriesValuesSourceType.COUNTER + ), SumAggregator::new, true ); diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ValueCountAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ValueCountAggregatorFactory.java index 9868913d63c2a..10f8bf198b7fe 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ValueCountAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ValueCountAggregatorFactory.java @@ -14,19 +14,26 @@ import org.elasticsearch.search.aggregations.CardinalityUpperBound; import org.elasticsearch.search.aggregations.support.AggregationContext; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.TimeSeriesValuesSourceType; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; import java.io.IOException; import java.util.Map; +import java.util.stream.Stream; class ValueCountAggregatorFactory extends ValuesSourceAggregatorFactory { private final MetricAggregatorSupplier aggregatorSupplier; public static void registerAggregators(ValuesSourceRegistry.Builder builder) { - builder.register(ValueCountAggregationBuilder.REGISTRY_KEY, CoreValuesSourceType.ALL_CORE, ValueCountAggregator::new, true); + builder.register( + ValueCountAggregationBuilder.REGISTRY_KEY, + Stream.concat(CoreValuesSourceType.ALL_CORE.stream(), Stream.of(TimeSeriesValuesSourceType.COUNTER)).toList(), + ValueCountAggregator::new, + true + ); } ValueCountAggregatorFactory( diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/boxplot/BoxplotAggregatorFactory.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/boxplot/BoxplotAggregatorFactory.java index ae057f7a37a75..cf62b07d6c97c 100644 --- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/boxplot/BoxplotAggregatorFactory.java +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/boxplot/BoxplotAggregatorFactory.java @@ -15,6 +15,7 @@ import org.elasticsearch.search.aggregations.metrics.TDigestExecutionHint; import org.elasticsearch.search.aggregations.support.AggregationContext; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.TimeSeriesValuesSourceType; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; @@ -34,7 +35,7 @@ public class BoxplotAggregatorFactory extends ValuesSourceAggregatorFactory { static void registerAggregators(ValuesSourceRegistry.Builder builder) { builder.register( BoxplotAggregationBuilder.REGISTRY_KEY, - List.of(CoreValuesSourceType.NUMERIC, AnalyticsValuesSourceType.HISTOGRAM), + List.of(CoreValuesSourceType.NUMERIC, AnalyticsValuesSourceType.HISTOGRAM, TimeSeriesValuesSourceType.COUNTER), BoxplotAggregator::new, true ); diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/boxplot.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/boxplot.yml index d98b2a92b8106..b65ab8a771c82 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/boxplot.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/boxplot.yml @@ -192,3 +192,66 @@ setup: - match: { aggregations.plot.q3: 13.75 } - match: { aggregations.plot.lower: 2.0 } - match: { aggregations.plot.upper: 15.0 } + +--- +"Counter field": + - skip: + version: " - 8.9.99" + reason: "counter field support added in 8.10" + features: close_to + + - do: + indices.create: + index: test_counter + body: + settings: + index: + mode: time_series + routing_path: [ key ] + time_series: + start_time: 2021-01-01T00:00:00Z + end_time: 2021-01-31T00:00:00Z + number_of_shards: 1 + mappings: + properties: + "@timestamp": + type: date + key: + type: keyword + time_series_dimension: true + counter: + type: long + time_series_metric: counter + + - do: + bulk: + index: test_counter + refresh: true + body: + - '{ "index": {} }' + - '{ "@timestamp": "2021-01-01T00:01:00Z", "key": "bar", "counter": 10 }' + - '{ "index": {} }' + - '{ "@timestamp": "2021-01-01T00:02:00Z", "key": "bar", "counter": 20 }' + - '{ "index": {} }' + - '{ "@timestamp": "2021-01-01T00:03:00Z", "key": "bar", "counter": 22 }' + - '{ "index": {} }' + - '{ "@timestamp": "2021-01-01T00:04:00Z", "key": "bar", "counter": 28 }' + + - do: + search: + index: test_counter + body: + aggs: + counter_boxplot: + boxplot: + field: counter + + - match: { hits.total.value: 4 } + - length: { hits.hits: 4 } + - close_to: { aggregations.counter_boxplot.min: { value: 10.00, error: 0.01 } } + - close_to: { aggregations.counter_boxplot.max: { value: 28.00, error: 0.01 } } + - close_to: { aggregations.counter_boxplot.q1: { value: 17.50, error: 0.01 } } + - close_to: { aggregations.counter_boxplot.q2: { value: 21.00, error: 0.01 } } + - close_to: { aggregations.counter_boxplot.q3: { value: 23.50, error: 0.01 } } + - close_to: { aggregations.counter_boxplot.lower: { value: 10.00, error: 0.01 } } + - close_to: { aggregations.counter_boxplot.upper: { value: 28.00, error: 0.01 } } diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/matrix_stats.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/matrix_stats.yml index 5dfff391e5b4c..4544a9beed307 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/matrix_stats.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/matrix_stats.yml @@ -206,3 +206,78 @@ setup: - close_to: { aggregations.matrix.buckets.0.stats.fields.1.covariance.value2: { value: 0.0, error: 0.0001 } } - close_to: { aggregations.matrix.buckets.0.stats.fields.1.correlation.value1: { value: 1.0, error: 0.0001 } } - close_to: { aggregations.matrix.buckets.0.stats.fields.1.correlation.value2: { value: 0.0, error: 0.0001 } } + +--- +"Counter field": + - skip: + version: " - 8.9.99" + reason: "counter field support added in 8.10" + features: close_to + + - do: + indices.create: + index: test_counter + body: + settings: + index: + mode: time_series + routing_path: [ key ] + time_series: + start_time: 2021-01-01T00:00:00Z + end_time: 2021-01-31T00:00:00Z + number_of_shards: 1 + mappings: + properties: + "@timestamp": + type: date + key: + type: keyword + time_series_dimension: true + counter1: + type: long + time_series_metric: counter + counter2: + type: long + time_series_metric: counter + + - do: + bulk: + index: test_counter + refresh: true + body: + - '{ "index": {} }' + - '{ "@timestamp": "2021-01-01T00:01:00Z", "key": "bar", "counter1": 10, "counter2": 7 }' + - '{ "index": {} }' + - '{ "@timestamp": "2021-01-01T00:02:00Z", "key": "bar", "counter1": 20, "counter2": 10 }' + - '{ "index": {} }' + - '{ "@timestamp": "2021-01-01T00:03:00Z", "key": "bar", "counter1": 22, "counter2": 12 }' + - '{ "index": {} }' + - '{ "@timestamp": "2021-01-01T00:04:00Z", "key": "bar", "counter1": 28, "counter2": 13 }' + + - do: + search: + index: test_counter + body: + aggs: + counter_matrix_stats: + matrix_stats: + fields: [ "counter1", "counter2" ] + + - match: { hits.total.value: 4 } + - length: { hits.hits: 4 } + - match: { aggregations.counter_matrix_stats.fields.0.name: "counter2" } + - match: { aggregations.counter_matrix_stats.fields.0.count: 4 } + - close_to: { aggregations.counter_matrix_stats.fields.0.mean: { value: 10.50, error: 0.01 } } + - close_to: { aggregations.counter_matrix_stats.fields.0.variance: { value: 7.00, error: 0.01 } } + - close_to: { aggregations.counter_matrix_stats.fields.0.covariance.counter2: { value: 7.00, error: 0.01 } } + - close_to: { aggregations.counter_matrix_stats.fields.0.covariance.counter1: { value: 19.33, error: 0.01 } } + - close_to: { aggregations.counter_matrix_stats.fields.0.correlation.counter2: { value: 1.00, error: 0.01 } } + - close_to: { aggregations.counter_matrix_stats.fields.0.correlation.counter1: { value: 0.97, error: 0.01 } } + - match: { aggregations.counter_matrix_stats.fields.1.name: "counter1" } + - match: { aggregations.counter_matrix_stats.fields.1.count: 4 } + - close_to: { aggregations.counter_matrix_stats.fields.1.mean: { value: 20.00, error: 0.01 } } + - close_to: { aggregations.counter_matrix_stats.fields.1.variance: { value: 56.00, error: 0.01 } } + - close_to: { aggregations.counter_matrix_stats.fields.1.covariance.counter2: { value: 19.33, error: 0.01 } } + - close_to: { aggregations.counter_matrix_stats.fields.1.covariance.counter1: { value: 56.00, error: 0.01 } } + - close_to: { aggregations.counter_matrix_stats.fields.1.correlation.counter2: { value: 0.97, error: 0.01 } } + - close_to: { aggregations.counter_matrix_stats.fields.1.correlation.counter1: { value: 1.00, error: 0.01 } }