multi-category support, rate limiting, and pagination (opensearch-pro…

…ject#121) This PR is a conglomerate of the following PRs. opensearch-project#60 opensearch-project#64 opensearch-project#65 opensearch-project#67 opensearch-project#68 opensearch-project#69 opensearch-project#70 opensearch-project#71 opensearch-project#74 opensearch-project#75 opensearch-project#76 opensearch-project#77 opensearch-project#78 opensearch-project#79 opensearch-project#82 opensearch-project#83 opensearch-project#84 opensearch-project#92 opensearch-project#94 opensearch-project#93 opensearch-project#95 kaituo#1 kaituo#2 kaituo#3 kaituo#4 kaituo#5 kaituo#6 kaituo#7 kaituo#8 kaituo#9 kaituo#10 This spreadsheet contains the mappings from files to PR number (bug fix in my AD fork and tests are not included): https://gist.github.com/kaituo/9e1592c4ac4f2f449356cb93d0591167
ohltyler · Sep 1, 2021 · c9fe262 · c9fe262
1 parent 3619e5b
commit c9fe262
Show file tree

Hide file tree

Showing 153 changed files with 10,651 additions and 3,688 deletions.
diff --git a/build.gradle b/build.gradle
@@ -278,7 +278,7 @@ evaluationDependsOnChildren()
 task release(type: Copy, group: 'build') {
     dependsOn allprojects*.tasks.build
     from(zipTree(project.tasks.bundlePlugin.outputs.files.getSingleFile()))
-    into "build/plugins/opendistro-anomaly-detection"
+    into "build/plugins/opensearch-anomaly-detection"
     includeEmptyDirs = false
     // ES versions < 6.3 have a top-level opensearch directory inside the plugin zip which we need to remove
     eachFile { it.path = it.path - "opensearch/" }
@@ -327,6 +327,20 @@ List<String> jacocoExclusions = [
         'org.opensearch.ad.indices.AnomalyDetectionIndices',
         'org.opensearch.ad.transport.handler.MultiEntityResultHandler',
         'org.opensearch.ad.util.ThrowingSupplierWrapper',
+        'org.opensearch.ad.transport.EntityResultTransportAction',
+        'org.opensearch.ad.transport.EntityResultTransportAction.*',
+        'org.opensearch.ad.transport.AnomalyResultTransportAction.*',
+        'org.opensearch.ad.transport.ProfileNodeResponse',
+        'org.opensearch.ad.transport.ADResultBulkResponse',
+        'org.opensearch.ad.transport.AggregationType',
+        'org.opensearch.ad.EntityProfileRunner',
+        'org.opensearch.ad.NodeStateManager',
+        'org.opensearch.ad.util.BulkUtil',
+        'org.opensearch.ad.util.ExceptionUtil',
+        'org.opensearch.ad.feature.SearchFeatureDao',
+        'org.opensearch.ad.feature.CompositeRetriever.*',
+        'org.opensearch.ad.feature.ScriptMaker',
+        'org.opensearch.ad.ml.EntityModel',
 ]
 
 jacocoTestCoverageVerification {
@@ -431,11 +445,11 @@ afterEvaluate {
         prefix '/usr'
 
         license 'ASL-2.0'
-        maintainer 'OpenDistro for Elasticsearch Team <opendistro@amazon.com>'
-        url 'https://opendistro.github.io/for-elasticsearch/downloads.html'
+        maintainer 'OpenSearch <opensearch@amazon.com>'
+        url 'https://opensearch.org/downloads.html'
         summary '''
-         Anomaly Detection plugin for OpenDistro for Elasticsearch.
-         Reference documentation can be found at https://opendistro.github.io/for-elasticsearch-docs/.
+         Anomaly Detection plugin for OpenSearch.
+         Reference documentation can be found at https://docs-beta.opensearch.org/docs/ad/.
     '''.stripIndent().replace('\n', ' ').trim()
     }
 

diff --git a/src/main/java/org/opensearch/ad/AnomalyDetectorJobRunner.java b/src/main/java/org/opensearch/ad/AnomalyDetectorJobRunner.java
@@ -468,12 +468,14 @@ private void indexAnomalyResult(
         String detectorId = jobParameter.getName();
         detectorEndRunExceptionCount.remove(detectorId);
         try {
+            // reset error if different from previously recorded one
+            detectionStateHandler.saveError(response.getError(), detectorId);
             // skipping writing to the result index if not necessary
             // For a single-entity detector, the result is not useful if error is null
             // and rcf score (thus anomaly grade/confidence) is null.
-            // For a multi-entity detector, we don't need to save on the detector level.
-            // We always return 0 rcf score if there is no error.
-            if (response.getAnomalyScore() <= 0 && response.getError() == null) {
+            // For a HCAD detector, we don't need to save on the detector level.
+            // We return 0 or Double.NaN rcf score if there is no error.
+            if ((response.getAnomalyScore() <= 0 || Double.isNaN(response.getAnomalyScore())) && response.getError() == null) {
                 return;
             }
             IntervalTimeConfiguration windowDelay = (IntervalTimeConfiguration) ((AnomalyDetectorJob) jobParameter).getWindowDelay();
@@ -499,7 +501,6 @@ private void indexAnomalyResult(
                 indexUtil.getSchemaVersion(ADIndex.RESULT)
             );
             anomalyResultHandler.index(anomalyResult, detectorId);
-            detectionStateHandler.saveError(response.getError(), detectorId);
         } catch (Exception e) {
             log.error("Failed to index anomaly result for " + detectorId, e);
         } finally {