From 3a90e2162233d5edb67c3672399dc0da432e757c Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Fri, 18 Oct 2024 12:40:39 -0700 Subject: [PATCH] [Backport 2.x] Fix notifications listener leak in threat intel monitor (#1361) * Fix notifications listener leak in threat intel monitor (#1356) * notifications listener leak Signed-off-by: Surya Sashank Nistala * change error handling to succeed monitor execution when alerts or notifications fail Signed-off-by: Surya Sashank Nistala --------- Signed-off-by: Surya Sashank Nistala (cherry picked from commit 98edd700183b169a439d246142dd7044fa57edb1) Signed-off-by: github-actions[bot] * fix stringentity constructor issue Signed-off-by: Subhobrata Dey --------- Signed-off-by: Surya Sashank Nistala Signed-off-by: github-actions[bot] Signed-off-by: Subhobrata Dey Co-authored-by: github-actions[bot] Co-authored-by: Subhobrata Dey --- .../notifications/NotificationService.java | 1 + .../iocscan/dao/BaseEntityCrudService.java | 9 +- .../iocscan/service/IoCScanService.java | 8 +- .../iocscan/service/SaIoCScanService.java | 62 ++-- .../securityanalytics/TestHelpers.java | 10 + .../ThreatIntelMonitorRestApiIT.java | 294 +++++++++++++++++- 6 files changed, 345 insertions(+), 39 deletions(-) diff --git a/src/main/java/org/opensearch/securityanalytics/correlation/alert/notifications/NotificationService.java b/src/main/java/org/opensearch/securityanalytics/correlation/alert/notifications/NotificationService.java index 7ed1bb0ae..d081fd00f 100644 --- a/src/main/java/org/opensearch/securityanalytics/correlation/alert/notifications/NotificationService.java +++ b/src/main/java/org/opensearch/securityanalytics/correlation/alert/notifications/NotificationService.java @@ -81,6 +81,7 @@ public void sendNotification(String configId, String severity, String subject, S sendNotificationResponse -> { if (sendNotificationResponse.getStatus() == RestStatus.OK) { logger.info("Successfully sent a notification, Notification Event: " + sendNotificationResponse.getNotificationEvent()); + listener.onResponse(null); } else { listener.onFailure(new Exception("Error while sending a notification, Notification Event: " + sendNotificationResponse.getNotificationEvent())); } diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/BaseEntityCrudService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/BaseEntityCrudService.java index dc5daf891..f435db5ce 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/BaseEntityCrudService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/BaseEntityCrudService.java @@ -115,14 +115,19 @@ public void bulkIndexEntities(List newEntityList, List updatedEn } } actionListener.onResponse(null); - }, actionListener::onFailure), bulkRequestList.size()); + }, e1 -> { + log.error("Failed to bulk index " + getEntityName(), e1); + actionListener.onFailure(e1); + }), bulkRequestList.size()); + for (BulkRequest req : bulkRequestList) { try { - client.bulk(req, groupedListener); //todo why stash context here? + client.bulk(req, groupedListener); } catch (Exception e) { log.error( () -> new ParameterizedMessage("Failed to bulk save {} {}.", req.batchSize(), getEntityName()), e); + groupedListener.onFailure(e); } } }, e -> { diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/IoCScanService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/IoCScanService.java index ad591e42f..f60af7afd 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/IoCScanService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/IoCScanService.java @@ -56,10 +56,10 @@ public void scanIoCs(IocScanContext iocScanContext, (iocFindings, e1) -> { if (e1 != null) { log.error( - () -> new ParameterizedMessage("Threat intel monitor {}: Failed to create ioc findings/ ", + () -> new ParameterizedMessage("Threat intel monitor {}: Failed to create ioc findings", iocScanContext.getMonitor().getId(), data.size()), e1); - scanCallback.accept(null, e1); + scanCallback.accept(data, e1); } else { BiConsumer, Exception> triggerResultConsumer = (alerts, e2) -> { if (e2 != null) { @@ -67,8 +67,8 @@ public void scanIoCs(IocScanContext iocScanContext, () -> new ParameterizedMessage("Threat intel monitor {}: Failed to execute threat intel triggers/ ", iocScanContext.getMonitor().getId(), data.size()), e2); - scanCallback.accept(null, e2); - return; + // if findings are generated successfully but alerts/notifications fail we mark execution as succeeded, so that duplicate findings are not created + scanCallback.accept(data, null); } else { scanCallback.accept(data, null); } diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/SaIoCScanService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/SaIoCScanService.java index 109fc0bcb..8a3c4a206 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/SaIoCScanService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/SaIoCScanService.java @@ -113,7 +113,7 @@ private void executeTrigger(List iocFindings, } else { fetchExistingAlertsForTrigger(monitor, triggerMatchedFindings, trigger, ActionListener.wrap( existingAlerts -> { - executeActionsAndSaveAlerts(iocFindings, trigger, monitor, existingAlerts, triggerMatchedFindings, threatIntelTrigger, listener); + saveAlertsAndExecuteActions(iocFindings, trigger, monitor, existingAlerts, triggerMatchedFindings, threatIntelTrigger, listener); }, e -> { log.error(() -> new ParameterizedMessage( @@ -132,7 +132,7 @@ private void executeTrigger(List iocFindings, } } - private void executeActionsAndSaveAlerts(List iocFindings, + private void saveAlertsAndExecuteActions(List iocFindings, Trigger trigger, Monitor monitor, List existingAlerts, @@ -147,36 +147,38 @@ private void executeActionsAndSaveAlerts(List iocFindings, newAlerts, existingAlerts); if (false == trigger.getActions().isEmpty()) { - GroupedActionListener notifsListener = new GroupedActionListener<>(ActionListener.wrap( - r -> { - saveAlerts(new ArrayList<>(iocToUpdatedAlertsMap.values()), - newAlerts, - monitor, - (threatIntelAlerts, e) -> { - if (e != null) { - log.error(String.format("Threat intel monitor %s: Failed to save alerts for trigger {}", monitor.getId(), trigger.getId()), e); - listener.onFailure(e); - } else { + saveAlerts(new ArrayList<>(iocToUpdatedAlertsMap.values()), + newAlerts, + monitor, + (threatIntelAlerts, e) -> { + if (e != null) { + log.error(String.format("Threat intel monitor %s: Failed to save alerts for trigger %s", monitor.getId(), trigger.getId()), e); + listener.onFailure(e); + } else { + GroupedActionListener notifsListener = new GroupedActionListener<>(ActionListener.wrap( + r -> { listener.onResponse(threatIntelAlerts); + }, ex -> { + log.error(String.format("Threat intel monitor {}: Failed to send notification for trigger {}", monitor.getId(), trigger.getId()), ex); + listener.onFailure(new SecurityAnalyticsException("Failed to send notification", RestStatus.INTERNAL_SERVER_ERROR, ex)); } - }); - }, e -> { - log.error(String.format("Threat intel monitor %s: Failed to send notification for trigger {}", monitor.getId(), trigger.getId()), e); - listener.onFailure(new SecurityAnalyticsException("Failed to send notification", RestStatus.INTERNAL_SERVER_ERROR, e)); - } - ), trigger.getActions().size()); - for (Action action : trigger.getActions()) { - try { - String transformedSubject = NotificationService.compileTemplate(ctx, action.getSubjectTemplate()); - String transformedMessage = NotificationService.compileTemplate(ctx, action.getMessageTemplate()); - String configId = action.getDestinationId(); - notificationService.sendNotification(configId, trigger.getSeverity(), transformedSubject, transformedMessage, notifsListener); - } catch (Exception e) { - log.error(String.format("Threat intel monitor %s: Failed to send notification to %s for trigger %s", monitor.getId(), action.getDestinationId(), trigger.getId()), e); - notifsListener.onFailure(new SecurityAnalyticsException("Failed to send notification", RestStatus.INTERNAL_SERVER_ERROR, e)); - } + ), trigger.getActions().size()); + + for (Action action : trigger.getActions()) { + try { + String transformedSubject = NotificationService.compileTemplate(ctx, action.getSubjectTemplate()); + String transformedMessage = NotificationService.compileTemplate(ctx, action.getMessageTemplate()); + String configId = action.getDestinationId(); + notificationService.sendNotification(configId, trigger.getSeverity(), transformedSubject, transformedMessage, notifsListener); + } catch (Exception ex) { + log.error(String.format("Threat intel monitor %s: Failed to send notification to %s for trigger %s", monitor.getId(), action.getDestinationId(), trigger.getId()), ex); + notifsListener.onFailure(new SecurityAnalyticsException("Failed to send notification", RestStatus.INTERNAL_SERVER_ERROR, ex)); + } + + } + } + }); - } } else { saveAlerts(new ArrayList<>(iocToUpdatedAlertsMap.values()), newAlerts, @@ -235,7 +237,7 @@ private GroupedActionListener> getGroupedListenerForAllTr r -> { List list = new ArrayList<>(); r.forEach(list::addAll); - triggerResultConsumer.accept(list, null); //todo change emptylist to actual response + triggerResultConsumer.accept(list, null); }, e -> { log.error(() -> new ParameterizedMessage( "Threat intel monitor {} Failed to execute triggers {}", monitor.getId()), diff --git a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java index 78044e6d0..4a7fffd06 100644 --- a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java +++ b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java @@ -900,6 +900,16 @@ public static Action randomAction(String destinationId) { return new Action(name, destinationId, template, template, throttleEnabled, throttle, OpenSearchRestTestCase.randomAlphaOfLength(10), null); } + public static Action randomThreatInteMonitorAction(String destinationId) { + String name = OpenSearchRestTestCase.randomUnicodeOfLength(10); + Script template = randomTemplateScript("Threat intel Monitor {{ctx.monitor.name}} just entered alert status. Please investigate the issue.\n" + + " - Trigger: {{ctx.trigger.name}}\n" + + " - Severity: {{ctx.trigger.severity}}", null); + Boolean throttleEnabled = false; + Throttle throttle = randomThrottle(null, null); + return new Action(name, destinationId, template, template, throttleEnabled, throttle, OpenSearchRestTestCase.randomAlphaOfLength(10), null); + } + public static Script randomTemplateScript(String source, Map params) { if (params == null) { params = new HashMap<>(); diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/ThreatIntelMonitorRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/ThreatIntelMonitorRestApiIT.java index fe15784c2..77fafd157 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/ThreatIntelMonitorRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/ThreatIntelMonitorRestApiIT.java @@ -12,6 +12,7 @@ import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.commons.alerting.model.IntervalSchedule; import org.opensearch.commons.alerting.model.Monitor; +import org.opensearch.commons.alerting.model.action.Action; import org.opensearch.core.rest.RestStatus; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.search.SearchHit; @@ -35,6 +36,7 @@ import org.opensearch.securityanalytics.threatIntel.sacommons.monitor.ThreatIntelTriggerDto; import java.io.IOException; +import java.nio.charset.Charset; import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.ArrayList; @@ -44,9 +46,11 @@ import java.util.Map; import static java.util.Collections.emptyList; +import static org.opensearch.securityanalytics.TestHelpers.randomAction; import static org.opensearch.securityanalytics.TestHelpers.randomDetectorType; import static org.opensearch.securityanalytics.TestHelpers.randomDetectorWithTriggers; import static org.opensearch.securityanalytics.TestHelpers.randomIndex; +import static org.opensearch.securityanalytics.TestHelpers.randomThreatInteMonitorAction; import static org.opensearch.securityanalytics.TestHelpers.windowsIndexMapping; import static org.opensearch.securityanalytics.threatIntel.resthandler.monitor.RestSearchThreatIntelMonitorAction.SEARCH_THREAT_INTEL_MONITOR_PATH; @@ -315,8 +319,6 @@ public void testCreateThreatIntelMonitor_monitorAliases() throws IOException { assertEquals(totalHitsVal.intValue(), 0); } - - public void testCreateThreatIntelMonitor_configureMultipleIndicatorTypesInMonitor() throws IOException { updateClusterSetting(SecurityAnalyticsSettings.IOC_SCAN_MAX_TERMS_COUNT.getKey(), "1"); Response iocFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search", @@ -536,7 +538,280 @@ public void testCreateThreatIntelMonitor() throws IOException { assertEquals(totalHitsVal.intValue(), 0); } - public void testCreateThreatIntelMonitorWithExistingDetector() throws IOException { + // verify scenario where findings are generated but alert creation fails - monitor execution should be marked successful + public void testCreateThreatIntelMonitor_testExecution_findingSucceedsButAlertFails_ExecutionSucceeds() throws IOException { + updateClusterSetting(SecurityAnalyticsSettings.IOC_SCAN_MAX_TERMS_COUNT.getKey(), "1"); + Response iocFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search", + Map.of(), null); + Map responseAsMap = responseAsMap(iocFindingsResponse); + Assert.assertEquals(0, ((List>) responseAsMap.get("ioc_findings")).size()); + List vals = List.of("ip1", "ip2"); + String createdId = indexSourceConfigsAndIocs(vals); + String index = createTestIndex(randomIndex(), windowsIndexMapping()); + String monitorName = "test_monitor_name"; + + + /**create monitor */ + ThreatIntelMonitorDto iocScanMonitor = randomIocScanMonitorDto(index); + Response response = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI, Collections.emptyMap(), toHttpEntity(iocScanMonitor)); + Assert.assertEquals(201, response.getStatusLine().getStatusCode()); + Map responseBody = asMap(response); + + try { + makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI, Collections.emptyMap(), toHttpEntity(iocScanMonitor)); + fail(); + } catch (Exception e) { + /** creating a second threat intel monitor should fail*/ + assertTrue(e.getMessage().contains("already exists")); + } + + final String monitorId = responseBody.get("id").toString(); + Assert.assertNotEquals("response is missing Id", Monitor.NO_ID, monitorId); + + Response alertingMonitorResponse = getAlertingMonitor(client(), monitorId); + Assert.assertEquals(200, alertingMonitorResponse.getStatusLine().getStatusCode()); + int i = 1; + for (String val : vals) { + String doc = String.format("{\"ip\":\"%s\", \"ip1\":\"%s\"}", val, val); + try { + indexDoc(index, "" + i++, doc); + indexDoc(index, "" + i++, String.format("{\"ip\":\"1.2.3.4\", \"ip1\":\"1.2.3.4\"}", val, val)); + indexDoc(index, "" + i++, String.format("{\"random\":\"%s\", \"random1\":\"%s\"}", val, val)); + } catch (IOException e) { + fail(); + } + } + + Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); + Map executeResults = entityAsMap(executeResponse); + + String matchAllRequest = getMatchAllRequest(); + Response searchMonitorResponse = makeRequest(client(), "POST", SEARCH_THREAT_INTEL_MONITOR_PATH, Collections.emptyMap(), new StringEntity(matchAllRequest, + ContentType.APPLICATION_JSON.getMimeType(), ContentType.APPLICATION_JSON.getCharset().name())); + Assert.assertEquals(200, alertingMonitorResponse.getStatusLine().getStatusCode()); + HashMap hits = (HashMap) asMap(searchMonitorResponse).get("hits"); + HashMap totalHits = (HashMap) hits.get("total"); + Integer totalHitsVal = (Integer) totalHits.get("value"); + assertEquals(totalHitsVal.intValue(), 1); + makeRequest(client(), "POST", SEARCH_THREAT_INTEL_MONITOR_PATH, Collections.emptyMap(), new StringEntity(matchAllRequest, + ContentType.APPLICATION_JSON.getMimeType(), ContentType.APPLICATION_JSON.getCharset().name())); + + + iocFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search", + Map.of(), null); + responseAsMap = responseAsMap(iocFindingsResponse); + Assert.assertEquals(2, ((List>) responseAsMap.get("ioc_findings")).size()); + + //alerts + List searchHits = executeSearch(ThreatIntelAlertService.THREAT_INTEL_ALERT_ALIAS_NAME, matchAllRequest); + Assert.assertEquals(4, searchHits.size()); + + for (String val : vals) { + String doc = String.format("{\"ip\":\"%s\", \"ip1\":\"%s\"}", val, val); + try { + indexDoc(index, "" + i++, doc); + } catch (IOException e) { + fail(); + } + } + closeIndex(ThreatIntelAlertService.THREAT_INTEL_ALERT_ALIAS_NAME); + executeAlertingMonitor(monitorId, Collections.emptyMap()); + iocFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search", + Map.of(), null); + responseAsMap = responseAsMap(iocFindingsResponse); + Assert.assertEquals(4, ((List>) responseAsMap.get("ioc_findings")).size()); + openIndex(ThreatIntelAlertService.THREAT_INTEL_ALERT_ALIAS_NAME); + // Use ListIOCs API to confirm expected number of findings are returned + String listIocsUri = String.format("?%s=%s", ListIOCsActionRequest.FEED_IDS_FIELD, createdId); + Response listIocsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.LIST_IOCS_URI + listIocsUri, Collections.emptyMap(), null); + Map listIocsResponseMap = responseAsMap(listIocsResponse); + List> iocsMap = (List>) listIocsResponseMap.get("iocs"); + assertEquals(2, iocsMap.size()); + iocsMap.forEach((iocDetails) -> { + String iocId = (String) iocDetails.get("id"); + int numFindings = (Integer) iocDetails.get("num_findings"); + assertTrue(testIocDtos.stream().anyMatch(ioc -> iocId.equals(ioc.getId()))); + assertEquals(2, numFindings); + }); + + //alerts via system index search + searchHits = executeSearch(ThreatIntelAlertService.THREAT_INTEL_ALERT_ALIAS_NAME, matchAllRequest); + Assert.assertEquals(4, searchHits.size()); + + // alerts via API + Map params = new HashMap<>(); + Response getAlertsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_ALERTS_URI, params, null); + Map getAlertsBody = asMap(getAlertsResponse); + Assert.assertEquals(4, getAlertsBody.get("total_alerts")); + + + ThreatIntelMonitorDto updateMonitorDto = new ThreatIntelMonitorDto( + monitorId, + iocScanMonitor.getName() + "update", + iocScanMonitor.getPerIocTypeScanInputList(), + new IntervalSchedule(5, ChronoUnit.MINUTES, Instant.now()), + false, + null, + List.of(iocScanMonitor.getTriggers().get(0), iocScanMonitor.getTriggers().get(1)) + ); + //update monitor + response = makeRequest(client(), "PUT", SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI + "/" + monitorId, Collections.emptyMap(), toHttpEntity(updateMonitorDto)); + Assert.assertEquals(200, response.getStatusLine().getStatusCode()); + responseBody = asMap(response); + assertEquals(responseBody.get("id").toString(), monitorId); + assertEquals(((HashMap) responseBody.get("monitor")).get("name").toString(), iocScanMonitor.getName() + "update"); + + //delete + Response delete = makeRequest(client(), "DELETE", SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI + "/" + monitorId, Collections.emptyMap(), null); + Assert.assertEquals(200, delete.getStatusLine().getStatusCode()); + + searchMonitorResponse = makeRequest(client(), "POST", SEARCH_THREAT_INTEL_MONITOR_PATH, Collections.emptyMap(), new StringEntity(matchAllRequest, + ContentType.APPLICATION_JSON.getMimeType(), ContentType.APPLICATION_JSON.getCharset().name())); + Assert.assertEquals(200, alertingMonitorResponse.getStatusLine().getStatusCode()); + hits = (HashMap) asMap(searchMonitorResponse).get("hits"); + totalHits = (HashMap) hits.get("total"); + totalHitsVal = (Integer) totalHits.get("value"); + assertEquals(totalHitsVal.intValue(), 0); + } + + // verify scenario where alerts and findings are generated but notification fails - monitor execution should be marked successful + public void testCreateThreatIntelMonitor_testNotifications_invaliConfigExecutionSucceeds() throws IOException { + + updateClusterSetting(SecurityAnalyticsSettings.IOC_SCAN_MAX_TERMS_COUNT.getKey(), "1"); + Response iocFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search", + Map.of(), null); + Map responseAsMap = responseAsMap(iocFindingsResponse); + Assert.assertEquals(0, ((List>) responseAsMap.get("ioc_findings")).size()); + List vals = List.of("ip1", "ip2"); + String createdId = indexSourceConfigsAndIocs(vals); + String index = createTestIndex(randomIndex(), windowsIndexMapping()); + String monitorName = "test_monitor_name"; + + Action triggerAction1 = randomThreatInteMonitorAction(randomAlphaOfLength(10)); + Action triggerAction2 = randomThreatInteMonitorAction(randomAlphaOfLength(10)); + /**create monitor with trigger*/ + ThreatIntelMonitorDto iocScanMonitor = randomIocScanMonitorDtoWithTriggers(index, List.of(triggerAction1, triggerAction2)); + Response response = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI, Collections.emptyMap(), toHttpEntity(iocScanMonitor)); + Assert.assertEquals(201, response.getStatusLine().getStatusCode()); + Map responseBody = asMap(response); + + try { + makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI, Collections.emptyMap(), toHttpEntity(iocScanMonitor)); + fail(); + } catch (Exception e) { + /** creating a second threat intel monitor should fail*/ + assertTrue(e.getMessage().contains("already exists")); + } + + final String monitorId = responseBody.get("id").toString(); + Assert.assertNotEquals("response is missing Id", Monitor.NO_ID, monitorId); + + Response alertingMonitorResponse = getAlertingMonitor(client(), monitorId); + Assert.assertEquals(200, alertingMonitorResponse.getStatusLine().getStatusCode()); + int i = 1; + for (String val : vals) { + String doc = String.format("{\"ip\":\"%s\", \"ip1\":\"%s\"}", val, val); + try { + indexDoc(index, "" + i++, doc); + indexDoc(index, "" + i++, String.format("{\"ip\":\"1.2.3.4\", \"ip1\":\"1.2.3.4\"}", val, val)); + indexDoc(index, "" + i++, String.format("{\"random\":\"%s\", \"random1\":\"%s\"}", val, val)); + } catch (IOException e) { + fail(); + } + } + + Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); + Map executeResults = entityAsMap(executeResponse); + + String matchAllRequest = getMatchAllRequest(); + Response searchMonitorResponse = makeRequest(client(), "POST", SEARCH_THREAT_INTEL_MONITOR_PATH, Collections.emptyMap(), new StringEntity(matchAllRequest, + ContentType.APPLICATION_JSON.getMimeType(), ContentType.APPLICATION_JSON.getCharset().name())); + Assert.assertEquals(200, alertingMonitorResponse.getStatusLine().getStatusCode()); + HashMap hits = (HashMap) asMap(searchMonitorResponse).get("hits"); + HashMap totalHits = (HashMap) hits.get("total"); + Integer totalHitsVal = (Integer) totalHits.get("value"); + assertEquals(totalHitsVal.intValue(), 1); + makeRequest(client(), "POST", SEARCH_THREAT_INTEL_MONITOR_PATH, Collections.emptyMap(), new StringEntity(matchAllRequest, + ContentType.APPLICATION_JSON.getMimeType(), ContentType.APPLICATION_JSON.getCharset().name())); + + + iocFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search", + Map.of(), null); + responseAsMap = responseAsMap(iocFindingsResponse); + Assert.assertEquals(2, ((List>) responseAsMap.get("ioc_findings")).size()); + + //alerts + List searchHits = executeSearch(ThreatIntelAlertService.THREAT_INTEL_ALERT_ALIAS_NAME, matchAllRequest); + Assert.assertEquals(2, searchHits.size()); + + for (String val : vals) { + String doc = String.format("{\"ip\":\"%s\", \"ip1\":\"%s\"}", val, val); + try { + indexDoc(index, "" + i++, doc); + } catch (IOException e) { + fail(); + } + } + executeAlertingMonitor(monitorId, Collections.emptyMap()); + iocFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search", + Map.of(), null); + responseAsMap = responseAsMap(iocFindingsResponse); + Assert.assertEquals(4, ((List>) responseAsMap.get("ioc_findings")).size()); + + // Use ListIOCs API to confirm expected number of findings are returned + String listIocsUri = String.format("?%s=%s", ListIOCsActionRequest.FEED_IDS_FIELD, createdId); + Response listIocsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.LIST_IOCS_URI + listIocsUri, Collections.emptyMap(), null); + Map listIocsResponseMap = responseAsMap(listIocsResponse); + List> iocsMap = (List>) listIocsResponseMap.get("iocs"); + assertEquals(2, iocsMap.size()); + iocsMap.forEach((iocDetails) -> { + String iocId = (String) iocDetails.get("id"); + int numFindings = (Integer) iocDetails.get("num_findings"); + assertTrue(testIocDtos.stream().anyMatch(ioc -> iocId.equals(ioc.getId()))); + assertEquals(2, numFindings); + }); + + //alerts via system index search + searchHits = executeSearch(ThreatIntelAlertService.THREAT_INTEL_ALERT_ALIAS_NAME, matchAllRequest); + Assert.assertEquals(2, searchHits.size()); + + // alerts via API + Map params = new HashMap<>(); + Response getAlertsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_ALERTS_URI, params, null); + Map getAlertsBody = asMap(getAlertsResponse); + Assert.assertEquals(2, getAlertsBody.get("total_alerts")); + + + ThreatIntelMonitorDto updateMonitorDto = new ThreatIntelMonitorDto( + monitorId, + iocScanMonitor.getName() + "update", + iocScanMonitor.getPerIocTypeScanInputList(), + new IntervalSchedule(5, ChronoUnit.MINUTES, Instant.now()), + false, + null, + List.of(iocScanMonitor.getTriggers().get(0)) + ); + //update monitor + response = makeRequest(client(), "PUT", SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI + "/" + monitorId, Collections.emptyMap(), toHttpEntity(updateMonitorDto)); + Assert.assertEquals(200, response.getStatusLine().getStatusCode()); + responseBody = asMap(response); + assertEquals(responseBody.get("id").toString(), monitorId); + assertEquals(((HashMap) responseBody.get("monitor")).get("name").toString(), iocScanMonitor.getName() + "update"); + + //delete + Response delete = makeRequest(client(), "DELETE", SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI + "/" + monitorId, Collections.emptyMap(), null); + Assert.assertEquals(200, delete.getStatusLine().getStatusCode()); + + searchMonitorResponse = makeRequest(client(), "POST", SEARCH_THREAT_INTEL_MONITOR_PATH, Collections.emptyMap(), new StringEntity(matchAllRequest, + ContentType.APPLICATION_JSON.getMimeType(), ContentType.APPLICATION_JSON.getCharset().name())); + Assert.assertEquals(200, alertingMonitorResponse.getStatusLine().getStatusCode()); + hits = (HashMap) asMap(searchMonitorResponse).get("hits"); + totalHits = (HashMap) hits.get("total"); + totalHitsVal = (Integer) totalHits.get("value"); + assertEquals(totalHitsVal.intValue(), 0); + } + + public void testCreateThreatIntelMonitorWithExistingDetector() throws IOException { String index = createTestIndex(randomIndex(), windowsIndexMapping()); // Execute CreateMappingsAction to add alias mapping for index @@ -726,6 +1001,19 @@ public static ThreatIntelMonitorDto randomIocScanMonitorDto(String index) { List.of(t1, t2, t3, t4)); } + public static ThreatIntelMonitorDto randomIocScanMonitorDtoWithTriggers(String index, List actions) { + ThreatIntelTriggerDto t1 = new ThreatIntelTriggerDto(List.of(), List.of(IOCType.IPV4_TYPE, IOCType.DOMAIN_NAME_TYPE), actions, "match", null, "severity"); + + return new ThreatIntelMonitorDto( + Monitor.NO_ID, + randomAlphaOfLength(10), + List.of(new PerIocTypeScanInputDto(IOCType.IPV4_TYPE, Map.of(index, List.of("ip")))), + new IntervalSchedule(1, ChronoUnit.MINUTES, Instant.now()), + false, + null, + List.of(t1)); + } + public static ThreatIntelMonitorDto randomIocScanMonitorDtoWithMultipleIndicatorTypesToScan(String ipIndex, String hashIndex, String domainIndex) { ThreatIntelTriggerDto t1 = new ThreatIntelTriggerDto(List.of(ipIndex, "randomIndex"), List.of(IOCType.IPV4_TYPE, IOCType.DOMAIN_NAME_TYPE), emptyList(), "match", null, "severity"); ThreatIntelTriggerDto t2 = new ThreatIntelTriggerDto(List.of("randomIndex"), List.of(IOCType.DOMAIN_NAME_TYPE), emptyList(), "nomatch", null, "severity");