diff --git a/src/main/java/org/opensearch/securityanalytics/config/monitors/DetectorMonitorConfig.java b/src/main/java/org/opensearch/securityanalytics/config/monitors/DetectorMonitorConfig.java index f20656a50..02f6595ec 100644 --- a/src/main/java/org/opensearch/securityanalytics/config/monitors/DetectorMonitorConfig.java +++ b/src/main/java/org/opensearch/securityanalytics/config/monitors/DetectorMonitorConfig.java @@ -24,7 +24,7 @@ public class DetectorMonitorConfig { public static final String OPENSEARCH_SAP_RULE_INDEX_TEMPLATE = ".opensearch-sap-detectors-queries-index-template"; public static String getRuleIndex(String logType) { - return String.format(Locale.getDefault(), ".opensearch-sap-%s-detectors-queries*", logType); + return String.format(Locale.getDefault(), ".opensearch-sap-%s-detectors-queries", logType); } public static String getRuleIndexOptimized(String logType) { diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java index 7ceae503c..48fe08752 100644 --- a/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java +++ b/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java @@ -1258,10 +1258,10 @@ void createDetector() { request.getDetector().setFindingsIndexPattern(DetectorMonitorConfig.getFindingsIndexPattern(ruleTopic)); if (enableDetectorWithDedicatedQueryIndices) { + // disabling the setting after enabling it will mean delete & re-create the detector request.getDetector().setRuleIndex(DetectorMonitorConfig.getRuleIndexOptimized(ruleTopic)); } else { - String ruleTopicIndex = DetectorMonitorConfig.getRuleIndex(ruleTopic); - request.getDetector().setRuleIndex(ruleTopicIndex.substring(0, ruleTopicIndex.length()-1)); + request.getDetector().setRuleIndex(DetectorMonitorConfig.getRuleIndex(ruleTopic)); } User originalContextUser = this.user; @@ -1383,10 +1383,10 @@ void onGetResponse(Detector currentDetector, User user) { request.getDetector().setRuleIndex(currentDetector.getRuleIndex()); } else { if (enableDetectorWithDedicatedQueryIndices) { + // disabling the setting after enabling it will mean delete & re-create the detector request.getDetector().setRuleIndex(DetectorMonitorConfig.getRuleIndexOptimized(ruleTopic)); } else { - String ruleTopicIndex = DetectorMonitorConfig.getRuleIndex(ruleTopic); - request.getDetector().setRuleIndex(ruleTopicIndex.substring(0, ruleTopicIndex.length() - 1)); + request.getDetector().setRuleIndex(DetectorMonitorConfig.getRuleIndex(ruleTopic)); } } request.getDetector().setUser(user); diff --git a/src/main/java/org/opensearch/securityanalytics/util/RuleTopicIndices.java b/src/main/java/org/opensearch/securityanalytics/util/RuleTopicIndices.java index 7f21e2a41..7aa2def36 100644 --- a/src/main/java/org/opensearch/securityanalytics/util/RuleTopicIndices.java +++ b/src/main/java/org/opensearch/securityanalytics/util/RuleTopicIndices.java @@ -83,7 +83,8 @@ private void getAllRuleIndices(ActionListener> listener) { listener.onResponse( logTypes .stream() - .map(logType -> DetectorMonitorConfig.getRuleIndex(logType)) + // use index pattern here to define rule topic index template for all query indices which match the pattern + .map(logType -> DetectorMonitorConfig.getRuleIndex(logType) + "*") .collect(Collectors.toList()) ); }, listener::onFailure)); diff --git a/src/test/java/org/opensearch/securityanalytics/DetectorThreatIntelIT.java b/src/test/java/org/opensearch/securityanalytics/DetectorThreatIntelIT.java index 9432f3f68..ff80ca329 100644 --- a/src/test/java/org/opensearch/securityanalytics/DetectorThreatIntelIT.java +++ b/src/test/java/org/opensearch/securityanalytics/DetectorThreatIntelIT.java @@ -69,11 +69,22 @@ public void testCreateDetectorWithThreatIntelEnabled_updateDetectorWithThreatInt Detector detector = randomDetectorWithInputsAndThreatIntelAndTriggers(List.of(input), true, List.of(trigger)); Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + String request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); + + + assertEquals(2, response.getHits().getTotalHits().value); + assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); Map responseBody = asMap(createResponse); String detectorId = responseBody.get("_id").toString(); - String request = "{\n" + + request = "{\n" + " \"query\" : {\n" + " \"match\":{\n" + " \"_id\": \"" + detectorId + "\"\n" + @@ -258,11 +269,22 @@ public void testCreateDetectorWithThreatIntelDisabled_updateDetectorWithThreatIn Detector detector = randomDetectorWithInputsAndThreatIntel(List.of(input), false); Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + String request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); + + + assertEquals(1, response.getHits().getTotalHits().value); + assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); Map responseBody = asMap(createResponse); String detectorId = responseBody.get("_id").toString(); - String request = "{\n" + + request = "{\n" + " \"query\" : {\n" + " \"match\":{\n" + " \"_id\": \"" + detectorId + "\"\n" + @@ -344,11 +366,22 @@ public void testCreateDetectorWithThreatIntelEnabledAndNoRules_triggerDetectionT Detector detector = randomDetectorWithInputsAndThreatIntelAndTriggers(List.of(input), true, List.of(trigger)); Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + String request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); + + + assertEquals(1, response.getHits().getTotalHits().value); + assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); Map responseBody = asMap(createResponse); String detectorId = responseBody.get("_id").toString(); - String request = "{\n" + + request = "{\n" + " \"query\" : {\n" + " \"match\":{\n" + " \"_id\": \"" + detectorId + "\"\n" + @@ -427,11 +460,22 @@ public void testCreateDetectorWithThreatIntelEnabled_triggerDetectionTypeOnlyThr Detector detector = randomDetectorWithInputsAndThreatIntelAndTriggers(List.of(input), true, List.of(trigger)); Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + String request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); + + + assertEquals(1, response.getHits().getTotalHits().value); + assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); Map responseBody = asMap(createResponse); String detectorId = responseBody.get("_id").toString(); - String request = "{\n" + + request = "{\n" + " \"query\" : {\n" + " \"match\":{\n" + " \"_id\": \"" + detectorId + "\"\n" + @@ -511,11 +555,22 @@ public void testCreateDetectorWithThreatIntelEnabled_triggerWithBothDetectionTyp Detector detector = randomDetectorWithInputsAndThreatIntelAndTriggers(List.of(input), true, List.of(trigger)); Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + String request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); + + + assertEquals(1, response.getHits().getTotalHits().value); + assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); Map responseBody = asMap(createResponse); String detectorId = responseBody.get("_id").toString(); - String request = "{\n" + + request = "{\n" + " \"query\" : {\n" + " \"match\":{\n" + " \"_id\": \"" + detectorId + "\"\n" + @@ -592,11 +647,22 @@ public void testCreateDetectorWithThreatIntelDisabled_triggerWithThreatIntelDete Detector detector = randomDetectorWithInputsAndThreatIntelAndTriggers(List.of(input), false, List.of(trigger)); Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + String request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); + + + assertEquals(1, response.getHits().getTotalHits().value); + assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); Map responseBody = asMap(createResponse); String detectorId = responseBody.get("_id").toString(); - String request = "{\n" + + request = "{\n" + " \"query\" : {\n" + " \"match\":{\n" + " \"_id\": \"" + detectorId + "\"\n" + @@ -673,11 +739,22 @@ public void testCreateDetectorWithThreatIntelDisabled_triggerWithRulesDetectionT Detector detector = randomDetectorWithInputsAndThreatIntelAndTriggers(List.of(input), false, List.of(trigger)); Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + String request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); + + + assertEquals(1, response.getHits().getTotalHits().value); + assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); Map responseBody = asMap(createResponse); String detectorId = responseBody.get("_id").toString(); - String request = "{\n" + + request = "{\n" + " \"query\" : {\n" + " \"match\":{\n" + " \"_id\": \"" + detectorId + "\"\n" + @@ -725,4 +802,4 @@ public void testCreateDetectorWithThreatIntelDisabled_triggerWithRulesDetectionT /** findings are present but alerts are NOT generated as detection type mentioned in trigger is threat_intel only but finding is from rules*/ Assert.assertEquals(3, getAlertsBody.get("total_alerts")); } -} +} \ No newline at end of file diff --git a/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java b/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java index 333d5dbd2..0574821e1 100644 --- a/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java +++ b/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java @@ -296,7 +296,7 @@ public void testGetAlertsByStartTimeAndEndTimeSuccess() throws IOException, Inte } public void testGetAlerts_noDetector_failure() throws IOException { - // Call GetAlerts API + // Call GetAlerts API Map params = new HashMap<>(); params.put("detector_id", "nonexistent_detector_id"); try { @@ -821,14 +821,25 @@ public void testMultipleAggregationAndDocRules_alertSuccess() throws IOException Collections.emptyList()); Detector detector = randomDetectorWithInputsAndTriggers(List.of(input), List.of(new DetectorTrigger("randomtrigegr", "test-trigger", "1", List.of(randomDetectorType()), List.of(), List.of(), List.of(), List.of(), List.of())) - ); + ); Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); + + assertEquals(1, response.getHits().getTotalHits().value); // 5 for rules, 1 for match_all query in chained findings monitor + assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); Map responseBody = asMap(createResponse); String detectorId = responseBody.get("_id").toString(); - String request = "{\n" + + request = "{\n" + " \"query\" : {\n" + " \"match\":{\n" + " \"_id\": \"" + detectorId + "\"\n" + @@ -859,8 +870,8 @@ public void testMultipleAggregationAndDocRules_alertSuccess() throws IOException } } - assertEquals(1, numberOfMonitorTypes.get(Monitor.MonitorType.BUCKET_LEVEL_MONITOR.getValue()).intValue()); - assertEquals(1, numberOfMonitorTypes.get(Monitor.MonitorType.DOC_LEVEL_MONITOR.getValue()).intValue()); + assertEquals(1, numberOfMonitorTypes.get(Monitor.MonitorType.BUCKET_LEVEL_MONITOR.getValue()).intValue()); + assertEquals(1, numberOfMonitorTypes.get(Monitor.MonitorType.DOC_LEVEL_MONITOR.getValue()).intValue()); Map params = new HashMap<>(); params.put("detector_id", detectorId); @@ -884,13 +895,13 @@ public void testMultipleAggregationAndDocRules_alertSuccess() throws IOException List> queries = (List>) finding.get("queries"); Set findingRuleIds = queries.stream().map(it -> it.get("id").toString()).collect(Collectors.toSet()); - // In the case of bucket level monitors, queries will always contain one value - String aggRuleId = findingRuleIds.iterator().next(); - List findingDocs = (List) finding.get("related_doc_ids"); + // In the case of bucket level monitors, queries will always contain one value + String aggRuleId = findingRuleIds.iterator().next(); + List findingDocs = (List) finding.get("related_doc_ids"); - if (aggRuleId.equals(sumRuleId)) { - assertTrue(List.of("1", "2", "3", "4", "5", "6", "7").containsAll(findingDocs)); - } + if (aggRuleId.equals(sumRuleId)) { + assertTrue(List.of("1", "2", "3", "4", "5", "6", "7").containsAll(findingDocs)); + } } assertTrue(Arrays.asList("1", "2", "3", "4", "5", "6", "7", "8").containsAll(docLevelFinding)); diff --git a/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java b/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java index d3b0c3bc9..aed4cbfab 100644 --- a/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java +++ b/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java @@ -1209,11 +1209,21 @@ public void testCreateDetectorWithNotCondition_verifyFindingsAndNoFindings_succe Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + String request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); + + assertEquals(1, response.getHits().getTotalHits().value); + assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); Map responseBody = asMap(createResponse); String detectorId = responseBody.get("_id").toString(); - String request = "{\n" + + request = "{\n" + " \"query\" : {\n" + " \"match\":{\n" + " \"_id\": \"" + detectorId + "\"\n" + @@ -1251,7 +1261,7 @@ public void testCreateDetectorWithNotCondition_verifyFindingsAndNoFindings_succe " }\n" + " }\n" + "}"; - SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getFindingsIndex(randomDetectorType()), request, true); + response = executeSearchAndGetResponse(DetectorMonitorConfig.getFindingsIndex(randomDetectorType()), request, true); assertEquals(2, response.getHits().getTotalHits().value); diff --git a/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java index 5752aa41e..dacb89bfa 100644 --- a/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java @@ -22,7 +22,6 @@ import org.apache.hc.core5.http.io.entity.StringEntity; import org.apache.hc.core5.http.message.BasicHeader; import org.junit.Assert; -import org.junit.Ignore; import org.opensearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.opensearch.action.search.SearchResponse; import org.opensearch.client.Request; @@ -48,7 +47,6 @@ import static org.opensearch.securityanalytics.SecurityAnalyticsPlugin.MAPPER_BASE_URI; import static org.opensearch.securityanalytics.TestHelpers.randomDetectorWithInputs; -import static org.opensearch.securityanalytics.TestHelpers.randomDoc; public class MapperRestApiIT extends SecurityAnalyticsRestTestCase { @@ -1659,8 +1657,8 @@ public void testTraverseAndCopy() { " \"type\":\"keyword\"," + " \"ignore_above\":256" + " }" + - " }" + - " }" + + " }" + + " }" + " }" + " }" + "}"; @@ -1705,7 +1703,6 @@ public void onError(String error) { } } - @Ignore public void testAzureMappings() throws IOException { String indexName = "azure-test-index"; @@ -1725,16 +1722,12 @@ public void testAzureMappings() throws IOException { DetectorInput input = new DetectorInput("windows detector for security analytics", List.of(indexName), List.of(), getPrePackagedRules("azure").stream().map(DetectorRule::new).collect(Collectors.toList())); Detector detector = randomDetectorWithInputs(List.of(input), "azure"); - String detectorId = createDetector(detector); - Response response = makeRequest(client(), "POST", ".opensearch-sap-detectors-config/_search", Map.of(), - new StringEntity("{\"query\": {\"match\": {\"_id\": \"" + detectorId + "\"}}}"), new BasicHeader("Content-Type", "application/json")); - String ruleTopicIndex = ((Map) ((Map) ((List>) ((Map) responseAsMap(response).get("hits")) - .get("hits")).get(0).get("_source")).get("detector")).get("rule_topic_index").toString() + "-000001"; - List hits = executeSearch(ruleTopicIndex, matchAllSearchBody); + createDetector(detector); + + List hits = executeSearch(".opensearch-sap-azure-detectors-queries-*", matchAllSearchBody); Assert.assertEquals(127, hits.size()); } - @Ignore public void testADLDAPMappings() throws IOException { String indexName = "adldap-test-index"; @@ -1744,37 +1737,20 @@ public void testADLDAPMappings() throws IOException { indexDoc(indexName, "1", sampleDoc); -// createMappingsAPI(indexName, "ad_ldap"); + createMappingsAPI(indexName, "ad_ldap"); //Expect only "timestamp" alias to be applied Map mappings = getIndexMappingsSAFlat(indexName); -// assertTrue(mappings.containsKey("timestamp")); + assertTrue(mappings.containsKey("timestamp")); // Verify that all rules are working DetectorInput input = new DetectorInput("windows detector for security analytics", List.of(indexName), List.of(), getPrePackagedRules("ad_ldap").stream().map(DetectorRule::new).collect(Collectors.toList())); Detector detector = randomDetectorWithInputs(List.of(input), "ad_ldap"); - String detectorId = createDetector(detector); - - String request = "{\n" + - " \"query\" : {\n" + - " \"match\":{\n" + - " \"_id\": \"" + detectorId + "\"\n" + - " }\n" + - " }\n" + - "}"; - List hits = executeSearch(Detector.DETECTORS_INDEX, request); - SearchHit hit = hits.get(0); + createDetector(detector); - String monitorId = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); - - indexDoc(indexName, "2", sampleDoc); - - Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); - Map executeResults = entityAsMap(executeResponse); - - int noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); - Assert.assertEquals(5, noOfSigmaRuleMatches); + List hits = executeSearch(".opensearch-sap-ad_ldap-detectors-queries-*", matchAllSearchBody); + Assert.assertEquals(11, hits.size()); } public void testCloudtrailMappings() throws IOException { @@ -1796,29 +1772,12 @@ public void testCloudtrailMappings() throws IOException { DetectorInput input = new DetectorInput("windows detector for security analytics", List.of(indexName), List.of(), getPrePackagedRules("cloudtrail").stream().map(DetectorRule::new).collect(Collectors.toList())); Detector detector = randomDetectorWithInputs(List.of(input), "cloudtrail"); - String detectorId = createDetector(detector); - String request = "{\n" + - " \"query\" : {\n" + - " \"match\":{\n" + - " \"_id\": \"" + detectorId + "\"\n" + - " }\n" + - " }\n" + - "}"; - List hits = executeSearch(Detector.DETECTORS_INDEX, request); - SearchHit hit = hits.get(0); + createDetector(detector); - String monitorId = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); - - indexDoc(indexName, "2", sampleDoc); - - Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); - Map executeResults = entityAsMap(executeResponse); - - int noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); - Assert.assertEquals(1, noOfSigmaRuleMatches); + List hits = executeSearch(".opensearch-sap-cloudtrail-detectors-queries-*", matchAllSearchBody); + Assert.assertEquals(39, hits.size()); } - @Ignore public void testS3Mappings() throws IOException { String indexName = "s3-test-index"; @@ -1838,16 +1797,12 @@ public void testS3Mappings() throws IOException { DetectorInput input = new DetectorInput("windows detector for security analytics", List.of(indexName), List.of(), getPrePackagedRules("s3").stream().map(DetectorRule::new).collect(Collectors.toList())); Detector detector = randomDetectorWithInputs(List.of(input), "s3"); - String detectorId = createDetector(detector); - Response response = makeRequest(client(), "POST", ".opensearch-sap-detectors-config/_search", Map.of(), - new StringEntity("{\"query\": {\"match\": {\"_id\": \"" + detectorId + "\"}}}"), new BasicHeader("Content-Type", "application/json")); - String ruleTopicIndex = ((Map) ((Map) ((List>) ((Map) responseAsMap(response).get("hits")) - .get("hits")).get(0).get("_source")).get("detector")).get("rule_topic_index").toString() + "-000001"; - List hits = executeSearch(ruleTopicIndex, matchAllSearchBody); + createDetector(detector); + + List hits = executeSearch(".opensearch-sap-s3-detectors-queries-*", matchAllSearchBody); Assert.assertEquals(1, hits.size()); } - @Ignore public void testWAFMappings() throws IOException { String indexName = "waf-test-index"; String sampleDoc = readResource("waf-sample.json"); @@ -1869,26 +1824,10 @@ public void testWAFMappings() throws IOException { DetectorInput input = new DetectorInput("waf detector for security analytics", List.of(indexName), List.of(), getPrePackagedRules("waf").stream().map(DetectorRule::new).collect(Collectors.toList())); Detector detector = randomDetectorWithInputs(List.of(input), "waf"); - String detectorId = createDetector(detector); - String request = "{\n" + - " \"query\" : {\n" + - " \"match\":{\n" + - " \"_id\": \"" + detectorId + "\"\n" + - " }\n" + - " }\n" + - "}"; - List hits = executeSearch(Detector.DETECTORS_INDEX, request); - SearchHit hit = hits.get(0); - - String monitorId = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); - - indexDoc(indexName, "2", sampleDoc); - - Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); - Map executeResults = entityAsMap(executeResponse); + createDetector(detector); - int noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); - Assert.assertEquals(5, noOfSigmaRuleMatches); + List hits = executeSearch(".opensearch-sap-waf-detectors-queries-*", matchAllSearchBody); + Assert.assertEquals(5, hits.size()); } @SuppressWarnings("unchecked") diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java index 35767edbe..930f35940 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java @@ -1036,11 +1036,21 @@ public void testCreateDetector_verifyWorkflowCreation_success_WithoutGroupByRule Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + String request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); + + assertEquals(2, response.getHits().getTotalHits().value); + assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); Map responseBody = asMap(createResponse); String detectorId = responseBody.get("_id").toString(); - String request = "{\n" + + request = "{\n" + " \"query\" : {\n" + " \"match\":{\n" + " \"_id\": \"" + detectorId + "\"\n" + @@ -1094,11 +1104,21 @@ public void testCreateDetector_verifyWorkflowCreation_success_WithGroupByRulesIn Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + String request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); + + assertEquals(2, response.getHits().getTotalHits().value); + assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); Map responseBody = asMap(createResponse); String detectorId = responseBody.get("_id").toString(); - String request = "{\n" + + request = "{\n" + " \"query\" : {\n" + " \"match\":{\n" + " \"_id\": \"" + detectorId + "\"\n" + @@ -1353,11 +1373,21 @@ public void testCreateDetector_workflowWithDuplicateMonitor_failure() throws IOE Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + String request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); + + assertEquals(2, response.getHits().getTotalHits().value); + assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); Map responseBody = asMap(createResponse); String detectorId = responseBody.get("_id").toString(); - String request = "{\n" + + request = "{\n" + " \"query\" : {\n" + " \"match\":{\n" + " \"_id\": \"" + detectorId + "\"\n" + @@ -1413,11 +1443,21 @@ public void testCreateDetector_verifyWorkflowExecutionBucketLevelDocLevelMonitor Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + String request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); + + assertEquals(2, response.getHits().getTotalHits().value); + assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); Map responseBody = asMap(createResponse); String detectorId = responseBody.get("_id").toString(); - String request = "{\n" + + request = "{\n" + " \"query\" : {\n" + " \"match\":{\n" + " \"_id\": \"" + detectorId + "\"\n" + diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java index bf6ce597b..2a6b6b9c8 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java @@ -11,6 +11,7 @@ import org.apache.hc.core5.http.message.BasicHeader; import org.junit.Assert; import org.junit.Ignore; +import org.opensearch.action.search.SearchResponse; import org.opensearch.client.Request; import org.opensearch.client.Response; import org.opensearch.client.ResponseException; @@ -22,6 +23,7 @@ import org.opensearch.search.SearchHit; import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; import org.opensearch.securityanalytics.SecurityAnalyticsRestTestCase; +import org.opensearch.securityanalytics.config.monitors.DetectorMonitorConfig; import org.opensearch.securityanalytics.model.Detector; import org.opensearch.securityanalytics.model.DetectorInput; import org.opensearch.securityanalytics.model.DetectorRule; @@ -154,7 +156,7 @@ private void validateDetectorDeletion(final String detectorId) throws IOExceptio Assert.assertEquals(0, hits.size()); } - @SuppressWarnings("unchecked") + @SuppressWarnings("unchecked") public void testCreatingADetector() throws IOException { String index = createTestIndex(randomIndex(), windowsIndexMapping()); @@ -838,26 +840,18 @@ public void testUpdateADetector() throws IOException { Map responseBody = asMap(createResponse); String detectorId = responseBody.get("_id").toString(); + String request = "{\n" + " \"query\" : {\n" + - " \"match\":{\n" + - " \"_id\": \"" + detectorId + "\"\n" + + " \"match_all\":{\n" + " }\n" + " }\n" + "}"; - List hits = executeSearch(Detector.DETECTORS_INDEX, request); - SearchHit hit = hits.get(0); - - String monitorId = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); - indexDoc(index, "1", randomDoc()); - - Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); - Map executeResults = entityAsMap(executeResponse); - - int noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); - Assert.assertEquals(5, noOfSigmaRuleMatches); + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); + Assert.assertEquals(5, response.getHits().getTotalHits().value); String rule = randomRule(); + createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.RULE_BASE_URI, Collections.singletonMap("category", randomDetectorType()), new StringEntity(rule), new BasicHeader("Content-Type", "application/json")); Assert.assertEquals("Create rule failed", RestStatus.CREATED, restStatus(createResponse)); @@ -876,13 +870,14 @@ public void testUpdateADetector() throws IOException { String detectorTypeInResponse = (String) ((Map) (asMap(updateResponse).get("detector"))).get("detector_type"); Assert.assertEquals("Detector type incorrect", randomDetectorType().toLowerCase(Locale.ROOT), detectorTypeInResponse); - indexDoc(index, "2", randomDoc()); - - executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); - executeResults = entityAsMap(executeResponse); - - noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); - Assert.assertEquals(6, noOfSigmaRuleMatches); + request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); + Assert.assertEquals(6, response.getHits().getTotalHits().value); } public void testUpdateANonExistingDetector() throws IOException { @@ -1068,10 +1063,10 @@ public void testDeletingADetector_single_Monitor() throws IOException { Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); // both req params and req body are supported createMappingRequest.setJsonEntity( - "{ \"index_name\":\"" + index + "\"," + - " \"rule_topic\":\"" + randomDetectorType() + "\", " + - " \"partial\":true" + - "}" + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"" + randomDetectorType() + "\", " + + " \"partial\":true" + + "}" ); Response response = client().performRequest(createMappingRequest); @@ -1081,12 +1076,12 @@ public void testDeletingADetector_single_Monitor() throws IOException { String detectorId1 = createDetector(detector1); String request = "{\n" + - " \"query\" : {\n" + - " \"match\":{\n" + - " \"_id\": \"" + detectorId1 + "\"\n" + - " }\n" + - " }\n" + - "}"; + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId1 + "\"\n" + + " }\n" + + " }\n" + + "}"; List hits = executeSearch(Detector.DETECTORS_INDEX, request); SearchHit hit = hits.get(0); @@ -1108,12 +1103,12 @@ public void testDeletingADetector_single_Monitor() throws IOException { String detectorId2 = createDetector(detector2); request = "{\n" + - " \"query\" : {\n" + - " \"match\":{\n" + - " \"_id\": \"" + detectorId2 + "\"\n" + - " }\n" + - " }\n" + - "}"; + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId2 + "\"\n" + + " }\n" + + " }\n" + + "}"; hits = executeSearch(Detector.DETECTORS_INDEX, request); hit = hits.get(0); @@ -1149,22 +1144,22 @@ public void testDeletingADetector_single_Monitor() throws IOException { Assert.assertFalse(doesIndexExist(String.format(Locale.ROOT, ".opensearch-sap-%s-detectors-queries-000001", "test_windows"))); request = "{\n" + - " \"query\" : {\n" + - " \"match\":{\n" + - " \"_id\": \"" + detectorId1 + "\"\n" + - " }\n" + - " }\n" + - "}"; + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId1 + "\"\n" + + " }\n" + + " }\n" + + "}"; hits = executeSearch(Detector.DETECTORS_INDEX, request); Assert.assertEquals(0, hits.size()); request = "{\n" + - " \"query\" : {\n" + - " \"match\":{\n" + - " \"_id\": \"" + detectorId2 + "\"\n" + - " }\n" + - " }\n" + - "}"; + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId2 + "\"\n" + + " }\n" + + " }\n" + + "}"; hits = executeSearch(Detector.DETECTORS_INDEX, request); Assert.assertEquals(0, hits.size()); } @@ -1178,10 +1173,10 @@ public void testDeletingADetector_single_Monitor_workflow_enabled() throws IOExc Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); // both req params and req body are supported createMappingRequest.setJsonEntity( - "{ \"index_name\":\"" + index + "\"," + - " \"rule_topic\":\"" + randomDetectorType() + "\", " + - " \"partial\":true" + - "}" + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"" + randomDetectorType() + "\", " + + " \"partial\":true" + + "}" ); Response response = client().performRequest(createMappingRequest); @@ -1191,12 +1186,12 @@ public void testDeletingADetector_single_Monitor_workflow_enabled() throws IOExc String detectorId1 = createDetector(detector1); String request = "{\n" + - " \"query\" : {\n" + - " \"match\":{\n" + - " \"_id\": \"" + detectorId1 + "\"\n" + - " }\n" + - " }\n" + - "}"; + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId1 + "\"\n" + + " }\n" + + " }\n" + + "}"; List hits = executeSearch(Detector.DETECTORS_INDEX, request); SearchHit hit = hits.get(0); @@ -1218,12 +1213,12 @@ public void testDeletingADetector_single_Monitor_workflow_enabled() throws IOExc String detectorId2 = createDetector(detector2); request = "{\n" + - " \"query\" : {\n" + - " \"match\":{\n" + - " \"_id\": \"" + detectorId2 + "\"\n" + - " }\n" + - " }\n" + - "}"; + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId2 + "\"\n" + + " }\n" + + " }\n" + + "}"; hits = executeSearch(Detector.DETECTORS_INDEX, request); hit = hits.get(0); @@ -1259,22 +1254,22 @@ public void testDeletingADetector_single_Monitor_workflow_enabled() throws IOExc Assert.assertFalse(doesIndexExist(String.format(Locale.ROOT, ".opensearch-sap-%s-detectors-queries-000001", "test_windows"))); request = "{\n" + - " \"query\" : {\n" + - " \"match\":{\n" + - " \"_id\": \"" + detectorId1 + "\"\n" + - " }\n" + - " }\n" + - "}"; + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId1 + "\"\n" + + " }\n" + + " }\n" + + "}"; hits = executeSearch(Detector.DETECTORS_INDEX, request); Assert.assertEquals(0, hits.size()); request = "{\n" + - " \"query\" : {\n" + - " \"match\":{\n" + - " \"_id\": \"" + detectorId2 + "\"\n" + - " }\n" + - " }\n" + - "}"; + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId2 + "\"\n" + + " }\n" + + " }\n" + + "}"; hits = executeSearch(Detector.DETECTORS_INDEX, request); Assert.assertEquals(0, hits.size()); } @@ -1468,9 +1463,9 @@ public void testCreatingADetectorWithTimestampFieldAliasMapping_verifyTimeRangeI Request updateRequest = new Request("PUT", SecurityAnalyticsPlugin.MAPPER_BASE_URI); updateRequest.setJsonEntity(MediaTypeRegistry.JSON.contentBuilder().map(Map.of( - "index_name", index, - "field", "time", - "alias", "timestamp")) + "index_name", index, + "field", "time", + "alias", "timestamp")) .toString()); Response apiResponse = client().performRequest(updateRequest); assertEquals(HttpStatus.SC_OK, apiResponse.getStatusLine().getStatusCode()); @@ -1729,4 +1724,157 @@ public void testDetector_withAlias_endToEnd_success() throws IOException { List findings = (List) getFindingsBody.get("findings"); Assert.assertEquals(findings.size(), 1); } + + @SuppressWarnings("unchecked") + public void testCreatingDetectorWithDynamicQueryIndexDisabledAndThenEnabledToUpdate() throws IOException { + updateClusterSetting("plugins.security_analytics.enable_detectors_with_dedicated_query_indices", "false"); + String index = createTestIndex(randomIndex(), windowsIndexMapping()); + + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"" + randomDetectorType() + "\", " + + " \"partial\":true" + + "}" + ); + + Response response = client().performRequest(createMappingRequest); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + + Detector detector = randomDetectorWithTriggers(getRandomPrePackagedRules(), List.of(new DetectorTrigger(null, "test-trigger", "1", List.of(randomDetectorType()), List.of(), List.of(), List.of(), List.of(), List.of()))); + + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + Map responseBody = asMap(createResponse); + + String detectorId1 = responseBody.get("_id").toString(); + + detector = randomDetectorWithTriggers(getRandomPrePackagedRules(), List.of(new DetectorTrigger(null, "test-trigger", "1", List.of(randomDetectorType()), List.of(), List.of(), List.of(), List.of(), List.of()))); + + createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + responseBody = asMap(createResponse); + + String detectorId2 = responseBody.get("_id").toString(); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId1 + "\"\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + + String monitorId1 = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); + + request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId2 + "\"\n" + + " }\n" + + " }\n" + + "}"; + hits = executeSearch(Detector.DETECTORS_INDEX, request); + hit = hits.get(0); + + String monitorId2 = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); + + indexDoc(index, "1", randomDoc()); + + Response executeResponse = executeAlertingMonitor(monitorId1, Collections.emptyMap()); + Map executeResults = entityAsMap(executeResponse); + + int noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + Assert.assertEquals(5, noOfSigmaRuleMatches); + + executeResponse = executeAlertingMonitor(monitorId2, Collections.emptyMap()); + executeResults = entityAsMap(executeResponse); + + noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + Assert.assertEquals(5, noOfSigmaRuleMatches); + + Assert.assertTrue(doesIndexExist(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "-000001")); + + updateClusterSetting("plugins.security_analytics.enable_detectors_with_dedicated_query_indices", "true"); + + Response updateResponse = makeRequest(client(), "PUT", SecurityAnalyticsPlugin.DETECTOR_BASE_URI + "/" + detectorId1, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Update detector failed", RestStatus.OK, restStatus(updateResponse)); + + indexDoc(index, "2", randomDoc()); + + executeResponse = executeAlertingMonitor(monitorId1, Collections.emptyMap()); + executeResults = entityAsMap(executeResponse); + + noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + Assert.assertEquals(5, noOfSigmaRuleMatches); + + executeResponse = executeAlertingMonitor(monitorId2, Collections.emptyMap()); + executeResults = entityAsMap(executeResponse); + + noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + Assert.assertEquals(5, noOfSigmaRuleMatches); + + response = makeRequest(client(), "POST", ".opensearch-sap-detectors-config/_search", Map.of(), + new StringEntity("{\"query\": {\"match\": {\"_id\": \"" + detectorId1 + "\"}}}"), new BasicHeader("Content-Type", "application/json")); + String ruleTopicIndex1 = ((Map) ((Map) ((List>) ((Map) responseAsMap(response).get("hits")) + .get("hits")).get(0).get("_source")).get("detector")).get("rule_topic_index").toString() + "-000001"; + Assert.assertTrue(doesIndexExist(ruleTopicIndex1)); + } + + @SuppressWarnings("unchecked") + public void testCreatingDetectorWithDynamicQueryIndexEnabledAndThenDisabled() throws IOException { + String index = createTestIndex(randomIndex(), windowsIndexMapping()); + + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"" + randomDetectorType() + "\", " + + " \"partial\":true" + + "}" + ); + + Response response = client().performRequest(createMappingRequest); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + + Detector detector = randomDetectorWithTriggers(getRandomPrePackagedRules(), List.of(new DetectorTrigger(null, "test-trigger", "1", List.of(randomDetectorType()), List.of(), List.of(), List.of(), List.of(), List.of()))); + + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + Map responseBody = asMap(createResponse); + + String detectorId1 = responseBody.get("_id").toString(); + + response = makeRequest(client(), "POST", ".opensearch-sap-detectors-config/_search", Map.of(), + new StringEntity("{\"query\": {\"match\": {\"_id\": \"" + detectorId1 + "\"}}}"), new BasicHeader("Content-Type", "application/json")); + String ruleTopicIndex1 = ((Map) ((Map) ((List>) ((Map) responseAsMap(response).get("hits")) + .get("hits")).get(0).get("_source")).get("detector")).get("rule_topic_index").toString() + "-000001"; + Assert.assertTrue(doesIndexExist(ruleTopicIndex1)); + Assert.assertFalse(doesIndexExist(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "-000001")); + + updateClusterSetting("plugins.security_analytics.enable_detectors_with_dedicated_query_indices", "false"); + + Response updateResponse = makeRequest(client(), "PUT", SecurityAnalyticsPlugin.DETECTOR_BASE_URI + "/" + detectorId1, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Update detector failed", RestStatus.OK, restStatus(updateResponse)); + + response = makeRequest(client(), "POST", ".opensearch-sap-detectors-config/_search", Map.of(), + new StringEntity("{\"query\": {\"match\": {\"_id\": \"" + detectorId1 + "\"}}}"), new BasicHeader("Content-Type", "application/json")); + ruleTopicIndex1 = ((Map) ((Map) ((List>) ((Map) responseAsMap(response).get("hits")) + .get("hits")).get(0).get("_source")).get("detector")).get("rule_topic_index").toString() + "-000001"; + Assert.assertTrue(doesIndexExist(ruleTopicIndex1)); + + detector = randomDetectorWithTriggers(getRandomPrePackagedRules(), List.of(new DetectorTrigger(null, "test-trigger", "1", List.of(randomDetectorType()), List.of(), List.of(), List.of(), List.of(), List.of()))); + + createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + Assert.assertTrue(doesIndexExist(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "-000001")); + } } \ No newline at end of file diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/RuleRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/RuleRestApiIT.java index 2421bb2c7..3fb397167 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/RuleRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/RuleRestApiIT.java @@ -9,7 +9,6 @@ import org.apache.hc.core5.http.io.entity.StringEntity; import org.apache.hc.core5.http.message.BasicHeader; import org.junit.Assert; -import org.opensearch.action.search.SearchResponse; import org.opensearch.client.Request; import org.opensearch.client.Response; import org.opensearch.client.ResponseException; @@ -19,7 +18,6 @@ import org.opensearch.core.xcontent.XContentParser; import org.opensearch.core.rest.RestStatus; import org.opensearch.search.SearchHit; -import org.opensearch.search.SearchHits; import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; import org.opensearch.securityanalytics.SecurityAnalyticsRestTestCase; import org.opensearch.securityanalytics.config.monitors.DetectorMonitorConfig; @@ -124,14 +122,14 @@ public void testCreatingARule_custom_category() throws IOException { } catch (ResponseException e) { assertEquals(HttpStatus.SC_BAD_REQUEST, e.getResponse().getStatusLine().getStatusCode()); Assert.assertTrue( - e.getMessage().contains("Invalid rule category") + e.getMessage().contains("Invalid rule category") ); } } public void testCreatingAggregationRule() throws SigmaError, IOException { Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.RULE_BASE_URI, Collections.singletonMap("category", "windows"), - new StringEntity(countAggregationTestRule()), new BasicHeader("Content-Type", "application/json")); + new StringEntity(countAggregationTestRule()), new BasicHeader("Content-Type", "application/json")); Assert.assertEquals("Create rule failed", RestStatus.CREATED, restStatus(createResponse)); Map responseBody = asMap(createResponse); @@ -144,24 +142,24 @@ public void testCreatingAggregationRule() throws SigmaError, IOException { String index = Rule.CUSTOM_RULES_INDEX; String request = "{\n" + - " \"query\": {\n" + - " \"nested\": {\n" + - " \"path\": \"rule\",\n" + - " \"query\": {\n" + - " \"bool\": {\n" + - " \"must\": [\n" + - " { \"match\": {\"rule.category\": \"windows\"}}\n" + - " ]\n" + - " }\n" + - " }\n" + - " }\n" + - " }\n" + - "}"; + " \"query\": {\n" + + " \"nested\": {\n" + + " \"path\": \"rule\",\n" + + " \"query\": {\n" + + " \"bool\": {\n" + + " \"must\": [\n" + + " { \"match\": {\"rule.category\": \"windows\"}}\n" + + " ]\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + "}"; List hits = executeSearch(index, request); XContentParser xcp = XContentType.JSON.xContent() - .createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, hits.get(0).getSourceAsString()); + .createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, hits.get(0).getSourceAsString()); Rule result = Rule.docParse(xcp, null, null); Assert.assertEquals(1, result.getAggregationQueries().size()); @@ -734,11 +732,31 @@ public void testDeletingUsedRule() throws IOException { .contains(String.format(Locale.getDefault(), "Rule with id %s is actively used by detectors. Deletion can be forced by setting forced flag to true", createdId))); } + String request = "{\n" + + " \"query\": {\n" + + " \"script\": {\n" + + " \"script\": \"doc['_id'][0].indexOf('" + createdId + "') > -1\"\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request); + Assert.assertEquals(2, hits.size()); + Response deleteResponse = makeRequest(client(), "DELETE", SecurityAnalyticsPlugin.RULE_BASE_URI + "/" + createdId, Collections.singletonMap("forced", "true"), null); Assert.assertEquals("Delete rule failed", RestStatus.OK, restStatus(deleteResponse)); + request = "{\n" + + " \"query\": {\n" + + " \"script\": {\n" + + " \"script\": \"doc['_id'][0].indexOf('" + createdId + "') > -1\"\n" + + " }\n" + + " }\n" + + "}"; + hits = executeSearch(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request); + Assert.assertEquals(0, hits.size()); + index = Rule.CUSTOM_RULES_INDEX; - String request = "{\n" + + request = "{\n" + " \"query\": {\n" + " \"nested\": {\n" + " \"path\": \"rule\",\n" + @@ -752,7 +770,7 @@ public void testDeletingUsedRule() throws IOException { " }\n" + " }\n" + "}"; - List hits = executeSearch(index, request); + hits = executeSearch(index, request); Assert.assertEquals(0, hits.size()); }