Skip to content
This repository has been archived by the owner on Aug 2, 2022. It is now read-only.

Commit

Permalink
Use aggregation's default value if no docs (#167)
Browse files Browse the repository at this point in the history
Currently, for sparse data (e.g., we only have docs half an hour every day), the preview/prediction API cannot find aggregated value and thus doesn't work.

This PR mitigates the sparse data issue by using an aggregation's default value if any.

Testing done:
1. Without this PR, preview/prediction cannot find any features to use given sparse data. With this PR, preview/prediction can.
2. Verified no unexpected exceptions are thrown with this PR during prediction or preview.
  • Loading branch information
kaituo authored Jun 23, 2020
1 parent 50be929 commit f62e570
Show file tree
Hide file tree
Showing 2 changed files with 15 additions and 7 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -173,10 +173,7 @@ public void getFeaturesForPeriod(AnomalyDetector detector, long startTime, long
}

private Optional<double[]> parseResponse(SearchResponse response, List<String> featureIds) {
return parseAggregations(
Optional.ofNullable(response).filter(resp -> response.getHits().getTotalHits().value > 0L).map(resp -> resp.getAggregations()),
featureIds
);
return parseAggregations(Optional.ofNullable(response).map(resp -> resp.getAggregations()), featureIds);
}

private double parseAggregation(Aggregation aggregation) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@
import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
import org.elasticsearch.search.aggregations.metrics.InternalTDigestPercentiles;
import org.elasticsearch.search.aggregations.metrics.Max;
import org.elasticsearch.search.aggregations.metrics.NumericMetricsAggregation;
import org.elasticsearch.search.aggregations.metrics.Percentile;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.junit.Before;
Expand Down Expand Up @@ -97,7 +98,6 @@
@PowerMockRunnerDelegate(JUnitParamsRunner.class)
@PrepareForTest({ ParseUtils.class })
public class SearchFeatureDaoTests {

private SearchFeatureDao searchFeatureDao;

@Mock
Expand Down Expand Up @@ -435,19 +435,30 @@ public void test_getFeaturesForPeriod_returnEmpty_givenNoData() throws Exception
}

@Test
public void getFeaturesForPeriod_returnEmpty_givenNoHits() throws Exception {
public void getFeaturesForPeriod_returnNonEmpty_givenDefaultValue() throws Exception {
long start = 100L;
long end = 200L;

// pre-conditions
when(ParseUtils.generateInternalFeatureQuery(eq(detector), eq(start), eq(end), eq(xContent))).thenReturn(searchSourceBuilder);
when(searchResponse.getHits()).thenReturn(new SearchHits(new SearchHit[0], new TotalHits(0L, TotalHits.Relation.EQUAL_TO), 1f));

List<Aggregation> aggList = new ArrayList<>(1);

NumericMetricsAggregation.SingleValue agg = mock(NumericMetricsAggregation.SingleValue.class);
when(agg.getName()).thenReturn("deny_max");
when(agg.value()).thenReturn(0d);

aggList.add(agg);

Aggregations aggregations = new Aggregations(aggList);
when(searchResponse.getAggregations()).thenReturn(aggregations);

// test
Optional<double[]> result = searchFeatureDao.getFeaturesForPeriod(detector, start, end);

// verify
assertFalse(result.isPresent());
assertTrue(result.isPresent());
}

private Object[] getFeaturesForSampledPeriodsData() {
Expand Down

0 comments on commit f62e570

Please sign in to comment.