feat: Adding support for date_nanos to Anomaly Detection (#1238)

Signed-off-by: Babacar Diasse <[email protected]>

src/main/java/org/opensearch/timeseries/constant/CommonName.java

@@ -28,6 +28,7 @@ public class CommonName {
     public static final String KEYWORD_TYPE = "keyword";
     public static final String IP_TYPE = "ip";
     public static final String DATE_TYPE = "date";
+    public static final String DATE_NANOS_TYPE = "date_nanos";
 
     // ======================================
     // Index name

src/main/java/org/opensearch/timeseries/rest/handler/AbstractTimeSeriesActionHandler.java

@@ -326,7 +326,7 @@ protected void validateTimeField(boolean indexingDryRun, ActionListener<T> liste
                                     foundField = true;
                                     Map<String, Object> metadataMap = (Map<String, Object>) type;
                                     String typeName = (String) metadataMap.get(CommonName.TYPE);
-                                    if (!typeName.equals(CommonName.DATE_TYPE)) {
+                                    if (!typeName.equals(CommonName.DATE_TYPE) && !typeName.equals(CommonName.DATE_NANOS_TYPE)) {
                                         listener
                                             .onFailure(
                                                 new ValidationException(

src/test/java/org/opensearch/ad/ADIntegTestCase.java

@@ -146,19 +146,29 @@ public void createDetectionStateIndex() throws IOException {
         createIndex(ADCommonName.DETECTION_STATE_INDEX, ADIndexManagement.getStateMappings());
     }
 
-    public void createTestDataIndex(String indexName) {
-        String mappings = "{\"properties\":{\""
-            + timeField
-            + "\":{\"type\":\"date\",\"format\":\"strict_date_time||epoch_millis\"},"
-            + "\"value\":{\"type\":\"double\"}, \""
-            + categoryField
-            + "\":{\"type\":\"keyword\"},\""
-            + ipField
-            + "\":{\"type\":\"ip\"},"
-            + "\"is_error\":{\"type\":\"boolean\"}, \"message\":{\"type\":\"text\"}}}";
+    public void createTestDataIndex(String indexName, boolean useDateNanos) {
+        StringBuilder mappingsBuilder = new StringBuilder("{\"properties\":{\"").append(timeField);
+        if (useDateNanos) {
+            mappingsBuilder.append("\":{\"type\":\"date_nanos\",\"format\":\"strict_date_time||epoch_millis\"},");
+        } else {
+            mappingsBuilder.append("\":{\"type\":\"date\",\"format\":\"strict_date_time||epoch_millis\"},");
+        }
+        mappingsBuilder
+            .append("\"value\":{\"type\":\"double\"}, \"")
+            .append(categoryField)
+            .append("\":{\"type\":\"keyword\"},\"")
+            .append(ipField)
+            .append("\":{\"type\":\"ip\"},")
+            .append("\"is_error\":{\"type\":\"boolean\"}, \"message\":{\"type\":\"text\"}}}");
+
+        String mappings = mappingsBuilder.toString();
         createIndex(indexName, mappings);
     }
 
+    public void createTestDataIndex(String indexName) {
+        createTestDataIndex(indexName, false);
+    }
+
     public void createIndex(String indexName, String mappings) {
         CreateIndexResponse createIndexResponse = TestHelpers.createIndex(admin(), indexName, mappings);
         assertEquals(true, createIndexResponse.isAcknowledged());
@@ -283,8 +293,25 @@ public void ingestTestDataValidate(String testIndex, Instant startTime, int dete
         ingestTestDataValidate(testIndex, startTime, detectionIntervalInMinutes, type, DEFAULT_TEST_DATA_DOCS);
     }
 
-    public void ingestTestDataValidate(String testIndex, Instant startTime, int detectionIntervalInMinutes, String type, int totalDocs) {
-        createTestDataIndex(testIndex);
+    public void ingestTestDataValidate(
+        String testIndex,
+        Instant startTime,
+        int detectionIntervalInMinutes,
+        String type,
+        boolean useDateNanos
+    ) {
+        ingestTestDataValidate(testIndex, startTime, detectionIntervalInMinutes, type, DEFAULT_TEST_DATA_DOCS, useDateNanos);
+    }
+
+    public void ingestTestDataValidate(
+        String testIndex,
+        Instant startTime,
+        int detectionIntervalInMinutes,
+        String type,
+        int totalDocs,
+        boolean useDateNanos
+    ) {
+        createTestDataIndex(testIndex, useDateNanos);
         List<Map<String, ?>> docs = new ArrayList<>();
         Instant currentInterval = Instant.from(startTime);
 
@@ -315,6 +342,10 @@ public void ingestTestDataValidate(String testIndex, Instant startTime, int dete
         assertEquals(totalDocs, count);
     }
 
+    public void ingestTestDataValidate(String testIndex, Instant startTime, int detectionIntervalInMinutes, String type, int totalDocs) {
+        ingestTestDataValidate(testIndex, startTime, detectionIntervalInMinutes, type, totalDocs, false);
+    }
+
     public Feature maxValueFeature() throws IOException {
         return maxValueFeature(nameField, valueField, nameField);
     }

src/test/java/org/opensearch/ad/e2e/RuleModelPerfIT.java

@@ -50,6 +50,21 @@ public void testRule() throws Exception {
         }
     }
 
+    public void testRuleWithDateNanos() throws Exception {
+        // TODO: this test case will run for a much longer time and timeout with security enabled
+        if (!isHttps()) {
+            disableResourceNotFoundFaultTolerence();
+            // there are 8 entities in the data set. Each one needs 1500 rows as training data.
+            Map<String, Double> minPrecision = new HashMap<>();
+            minPrecision.put("Phoenix", 0.5);
+            minPrecision.put("Scottsdale", 0.5);
+            Map<String, Double> minRecall = new HashMap<>();
+            minRecall.put("Phoenix", 0.9);
+            minRecall.put("Scottsdale", 0.6);
+            verifyRule("rule", 10, minPrecision.size(), 1500, minPrecision, minRecall, 20, true);
+        }
+    }
+
     private void verifyTestResults(
         Triple<Map<String, double[]>, Integer, Map<String, Set<Integer>>> testResults,
         Map<String, List<Entry<Instant, Instant>>> anomalies,
@@ -115,6 +130,19 @@ public void verifyRule(
         Map<String, Double> minPrecision,
         Map<String, Double> minRecall,
         int maxError
+    ) throws Exception {
+        verifyRule(datasetName, intervalMinutes, numberOfEntities, trainTestSplit, minPrecision, minRecall, maxError, false);
+    }
+
+    public void verifyRule(
+        String datasetName,
+        int intervalMinutes,
+        int numberOfEntities,
+        int trainTestSplit,
+        Map<String, Double> minPrecision,
+        Map<String, Double> minRecall,
+        int maxError,
+        boolean useDateNanos
     ) throws Exception {
         String dataFileName = String.format(Locale.ROOT, "data/%s.data", datasetName);
         String labelFileName = String.format(Locale.ROOT, "data/%s.label", datasetName);
@@ -127,7 +155,9 @@ public void verifyRule(
         String mapping = String
             .format(
                 Locale.ROOT,
-                "{ \"mappings\": { \"properties\": { \"timestamp\": { \"type\": \"date\"},"
+                "{ \"mappings\": { \"properties\": { \"timestamp\": { \"type\":"
+                    + (useDateNanos ? "\"date_nanos\"" : "\"date\"")
+                    + "},"
                     + " \"transform._doc_count\": { \"type\": \"integer\" },"
                     + "\"%s\": { \"type\": \"keyword\"} } } }",
                 categoricalField

src/test/java/org/opensearch/ad/rest/AnomalyDetectorRestApiIT.java

@@ -117,7 +117,12 @@ private AnomalyDetector createIndexAndGetAnomalyDetector(String indexName) throw
     }
 
     private AnomalyDetector createIndexAndGetAnomalyDetector(String indexName, List<Feature> features) throws IOException {
-        TestHelpers.createIndexWithTimeField(client(), indexName, TIME_FIELD);
+        return createIndexAndGetAnomalyDetector(indexName, features, false);
+    }
+
+    private AnomalyDetector createIndexAndGetAnomalyDetector(String indexName, List<Feature> features, boolean useDateNanos)
+        throws IOException {
+        TestHelpers.createIndexWithTimeField(client(), indexName, TIME_FIELD, useDateNanos);
         String testIndexData = "{\"keyword-field\": \"field-1\", \"ip-field\": \"1.2.3.4\", \"timestamp\": 1}";
         TestHelpers.ingestDataToIndex(client(), indexName, TestHelpers.toHttpEntity(testIndexData));
         AnomalyDetector detector = TestHelpers.randomAnomalyDetector(TIME_FIELD, indexName, features);
@@ -200,6 +205,35 @@ public void testCreateAnomalyDetector() throws Exception {
         assertTrue("incorrect version", version > 0);
     }
 
+    public void testCreateAnomalyDetectorWithDateNanos() throws Exception {
+        AnomalyDetector detector = createIndexAndGetAnomalyDetector(INDEX_NAME, ImmutableList.of(TestHelpers.randomFeature(true)), true);
+        updateClusterSettings(ADEnabledSetting.AD_ENABLED, false);
+
+        Exception ex = expectThrows(
+            ResponseException.class,
+            () -> TestHelpers
+                .makeRequest(
+                    client(),
+                    "POST",
+                    TestHelpers.AD_BASE_DETECTORS_URI,
+                    ImmutableMap.of(),
+                    TestHelpers.toHttpEntity(detector),
+                    null
+                )
+        );
+        assertThat(ex.getMessage(), containsString(ADCommonMessages.DISABLED_ERR_MSG));
+
+        updateClusterSettings(ADEnabledSetting.AD_ENABLED, true);
+        Response response = TestHelpers
+            .makeRequest(client(), "POST", TestHelpers.AD_BASE_DETECTORS_URI, ImmutableMap.of(), TestHelpers.toHttpEntity(detector), null);
+        assertEquals("Create anomaly detector failed", RestStatus.CREATED, TestHelpers.restStatus(response));
+        Map<String, Object> responseMap = entityAsMap(response);
+        String id = (String) responseMap.get("_id");
+        int version = (int) responseMap.get("_version");
+        assertNotEquals("response is missing Id", AnomalyDetector.NO_ID, id);
+        assertTrue("incorrect version", version > 0);
+    }
+
     public void testUpdateAnomalyDetectorCategoryField() throws Exception {
         AnomalyDetector detector = createIndexAndGetAnomalyDetector(INDEX_NAME);
         Response response = TestHelpers

src/test/java/org/opensearch/ad/transport/ValidateAnomalyDetectorTransportActionTests.java

@@ -514,4 +514,24 @@ public void testValidateAnomalyDetectorWithNonDateTimeField() throws IOException
             response.getIssue().getMessage()
         );
     }
+
+    @Test
+    public void testValidateAnomalyDetectorWithDateNanosWithoutIssue() throws IOException {
+        AnomalyDetector anomalyDetector = TestHelpers
+            .randomAnomalyDetector(timeField, "test-index", ImmutableList.of(sumValueFeature(nameField, ipField + ".is_error", "test-2")));
+        ingestTestDataValidate(anomalyDetector.getIndices().get(0), Instant.now().minus(1, ChronoUnit.DAYS), 1, "error", true);
+        ValidateConfigRequest request = new ValidateConfigRequest(
+            AnalysisType.AD,
+            anomalyDetector,
+            ValidationAspect.DETECTOR.getName(),
+            5,
+            5,
+            5,
+            new TimeValue(5_000L),
+            10
+        );
+        ValidateConfigResponse response = client().execute(ValidateAnomalyDetectorAction.INSTANCE, request).actionGet(5_000);
+        assertNull(response.getIssue());
+    }
+
 }

src/test/java/org/opensearch/timeseries/TestHelpers.java

@@ -1166,9 +1166,18 @@ public static void createIndex(RestClient client, String indexName, HttpEntity d
     }
 
     public static void createIndexWithTimeField(RestClient client, String indexName, String timeField) throws IOException {
+        createIndexWithTimeField(client, indexName, timeField, false);
+    }
+
+    public static void createIndexWithTimeField(RestClient client, String indexName, String timeField, boolean useDateNanos)
+        throws IOException {
         StringBuilder indexMappings = new StringBuilder();
         indexMappings.append("{\"properties\":{");
-        indexMappings.append("\"" + timeField + "\":{\"type\":\"date\"}");
+        if (useDateNanos) {
+            indexMappings.append("\"" + timeField + "\":{\"type\":\"date_nanos\"}");
+        } else {
+            indexMappings.append("\"" + timeField + "\":{\"type\":\"date\"}");
+        }
         indexMappings.append("}}");
         createIndex(client, indexName.toLowerCase(Locale.ROOT), TestHelpers.toHttpEntity("{\"name\": \"test\"}"));
         createIndexMapping(client, indexName.toLowerCase(Locale.ROOT), TestHelpers.toHttpEntity(indexMappings.toString()));