From e47e3c3f45510ef7ac439d42dfc67cbf9fb14d10 Mon Sep 17 00:00:00 2001 From: Kaituo Li Date: Tue, 3 Sep 2024 13:15:37 -0700 Subject: [PATCH] Correct handling of null max aggregation values in SearchResponse (#1292) Previously, when the max aggregation in SearchResponse returned a null value, it was converted to a negative number (-9223372036854775808), leading to erroneous timestamps. This fix adds a check to ensure that only valid positive aggregation values are considered, preventing null or invalid values from being misinterpreted as negative numbers. The change applies to only timestamps. So a positive number filter is always right. Testing done: 1. Added unit tests to cover scenarios where max aggregation returns null or negative values, ensuring the method returns Optional.empty() when the value is invalid. 2. Added unit tests to verify correct handling of valid positive max aggregation values. 3. Ran e2e manual testing. Previously: ``` POST /_plugins/_anomaly_detection/detectors/_validate/model { "name": "test-detector-13", "description": "Test detector", "time_field": "customer_birth_date", "indices": ["opensearch_dashboards_sample_data_ecommerce"], "feature_attributes": [ { "feature_name": "cpu", "feature_enabled": true, "aggregation_query": { "total_revenue_usd-field": { "max": { "field": "day_of_week_i" } } } } ], "detection_interval": { "period": { "interval":1, "unit": "Minutes" } }, "window_delay": { "period": { "interval": 1, "unit": "Minutes" } }, "category_field": ["currency"] } ``` returns: ``` nested: OpenSearchParseException[failed to parse date field [-9223372036854775808] with format [strict_date_optional_time||epoch_millis]: [failed to parse date field [-9223372036854775808] with format [strict_date_optional_time||epoch_millis]]]; nested: IllegalArgumentException[failed to parse date field [-9223372036854775808] with format [strict_date_optional_time||epoch_millis]]; nested: DateTimeParseException[Failed to parse with all enclosed parsers]; } opensearch-ccs-node1 | [2024-09-03T15:05:45,776][ERROR][o.o.t.r.h.ModelValidationActionHandler] [2a2cd14da04d] Failed to create search request for last data point opensearch-ccs-node1 | org.opensearch.action.search.SearchPhaseExecutionException: all shards failed opensearch-ccs-node1 | at org.opensearch.action.search.AbstractSearchAsyncAction.onPhaseFailure(AbstractSearchAsyncAction.java:770) [opensearch-3.0.0.jar:3.0.0] ``` Now the call returns: ``` { "model": { "time_field": { "message": "There isn't enough historical data found with current timefield selected." } } } ``` Signed-off-by: Kaituo Li --- .../timeseries/util/ParseUtils.java | 8 +- .../timeseries/util/ParseUtilsTests.java | 157 ++++++++++++++++++ 2 files changed, 164 insertions(+), 1 deletion(-) create mode 100644 src/test/java/org/opensearch/timeseries/util/ParseUtilsTests.java diff --git a/src/main/java/org/opensearch/timeseries/util/ParseUtils.java b/src/main/java/org/opensearch/timeseries/util/ParseUtils.java index 3b64be259..7d7ac7889 100644 --- a/src/main/java/org/opensearch/timeseries/util/ParseUtils.java +++ b/src/main/java/org/opensearch/timeseries/util/ParseUtils.java @@ -699,7 +699,13 @@ public static Optional getLatestDataTime(SearchResponse searchResponse) { .map(SearchResponse::getAggregations) .map(aggs -> aggs.asMap()) .map(map -> (Max) map.get(CommonName.AGG_NAME_MAX_TIME)) - .map(agg -> (long) agg.getValue()); + // Explanation: + // - `agg != null`: Ensures the aggregation object is not null. + // - `agg.getValue() > 0`: Checks that the value is positive, as the aggregation might return a default negative value + // like -9223372036854775808 if the underlying data is null or invalid. This check prevents converting such invalid + // values to a timestamp, which would not make sense in the context of time-based data. + .filter(agg -> agg != null && agg.getValue() > 0) // Check if the value is valid and positive. + .map(agg -> (long) agg.getValue()); // Cast to long if valid } /** diff --git a/src/test/java/org/opensearch/timeseries/util/ParseUtilsTests.java b/src/test/java/org/opensearch/timeseries/util/ParseUtilsTests.java new file mode 100644 index 000000000..b40de8152 --- /dev/null +++ b/src/test/java/org/opensearch/timeseries/util/ParseUtilsTests.java @@ -0,0 +1,157 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.timeseries.util; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.util.Collections; +import java.util.Optional; + +import org.apache.lucene.search.TotalHits; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.action.search.SearchResponseSections; +import org.opensearch.action.search.ShardSearchFailure; +import org.opensearch.search.SearchHit; +import org.opensearch.search.SearchHits; +import org.opensearch.search.aggregations.Aggregations; +import org.opensearch.search.aggregations.metrics.Max; +import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.timeseries.constant.CommonName; + +public class ParseUtilsTests extends OpenSearchTestCase { + + public void testGetLatestDataTime_withValidMaxValue() { + Max maxAggregation = mock(Max.class); + when(maxAggregation.getValue()).thenReturn(1623840000000.0); // A valid timestamp value + + Aggregations aggregations = new Aggregations(Collections.singletonList(maxAggregation)); + when(maxAggregation.getName()).thenReturn(CommonName.AGG_NAME_MAX_TIME); + + SearchResponseSections sections = new SearchResponseSections( + new SearchHits(new SearchHit[0], new TotalHits(0, TotalHits.Relation.EQUAL_TO), 0), + aggregations, + null, + false, + null, + null, + 1 + ); + SearchResponse searchResponse = new SearchResponse( + sections, + null, + 0, + 0, + 0, + 0L, + ShardSearchFailure.EMPTY_ARRAY, + SearchResponse.Clusters.EMPTY + ); + + Optional result = ParseUtils.getLatestDataTime(searchResponse); + + assertTrue(result.isPresent()); + assertEquals("got " + result.get(), 1623840000000L, result.get().longValue()); + } + + public void testGetLatestDataTime_withNullValue() { + Max maxAggregation = mock(Max.class); + when(maxAggregation.getValue()).thenReturn(Double.NEGATIVE_INFINITY); // Simulating a missing value scenario + + Aggregations aggregations = new Aggregations(Collections.singletonList(maxAggregation)); + when(maxAggregation.getName()).thenReturn(CommonName.AGG_NAME_MAX_TIME); + + SearchResponseSections sections = new SearchResponseSections( + new SearchHits(new SearchHit[0], new TotalHits(0, TotalHits.Relation.EQUAL_TO), 0), + aggregations, + null, + false, + null, + null, + 1 + ); + SearchResponse searchResponse = new SearchResponse( + sections, + null, + 0, + 0, + 0, + 0L, + ShardSearchFailure.EMPTY_ARRAY, + SearchResponse.Clusters.EMPTY + ); + + // Act + Optional result = ParseUtils.getLatestDataTime(searchResponse); + + // Assert + assertTrue(result.isEmpty()); + } + + public void testGetLatestDataTime_withNoAggregations() { + // Arrange + SearchResponseSections sections = new SearchResponseSections( + new SearchHits(new SearchHit[0], new TotalHits(0, TotalHits.Relation.EQUAL_TO), 0), + null, // No aggregations provided + null, + false, + null, + null, + 1 + ); + SearchResponse searchResponse = new SearchResponse( + sections, + null, + 0, + 0, + 0, + 0L, + ShardSearchFailure.EMPTY_ARRAY, + SearchResponse.Clusters.EMPTY + ); + + // Act + Optional result = ParseUtils.getLatestDataTime(searchResponse); + + // Assert + assertTrue(result.isEmpty()); + } + + public void testGetLatestDataTime_withNegativeMaxValue() { + // Arrange + Max maxAggregation = mock(Max.class); + when(maxAggregation.getValue()).thenReturn(-9223372036854775808.0); // Invalid negative value + + Aggregations aggregations = new Aggregations(Collections.singletonList(maxAggregation)); + when(maxAggregation.getName()).thenReturn(CommonName.AGG_NAME_MAX_TIME); + + SearchResponseSections sections = new SearchResponseSections( + new SearchHits(new SearchHit[0], new TotalHits(0, TotalHits.Relation.EQUAL_TO), 0), + aggregations, + null, + false, + null, + null, + 1 + ); + SearchResponse searchResponse = new SearchResponse( + sections, + null, + 0, + 0, + 0, + 0L, + ShardSearchFailure.EMPTY_ARRAY, + SearchResponse.Clusters.EMPTY + ); + + // Act + Optional result = ParseUtils.getLatestDataTime(searchResponse); + + // Assert + assertTrue(result.isEmpty()); + } +}