diff --git a/build.gradle b/build.gradle index 70b9e0bd3..c81cc9dc0 100644 --- a/build.gradle +++ b/build.gradle @@ -155,7 +155,7 @@ dependencies { implementation group: 'org.apache.commons', name: 'commons-lang3', version: "${versions.commonslang}" implementation "org.antlr:antlr4-runtime:4.10.1" implementation "com.cronutils:cron-utils:9.1.6" - api "org.opensearch:common-utils:${common_utils_version}@jar" + api files("/Users/snistala/Documents/opensearch/common-utils/build/libs/common-utils-3.0.0.0-SNAPSHOT.jar") api "org.opensearch.client:opensearch-rest-client:${opensearch_version}" implementation "org.jetbrains.kotlin:kotlin-stdlib:${kotlin_version}" implementation "org.opensearch:opensearch-job-scheduler-spi:${opensearch_build}" diff --git a/src/main/java/org/opensearch/securityanalytics/logtype/LogTypeService.java b/src/main/java/org/opensearch/securityanalytics/logtype/LogTypeService.java index fe1402e59..bec6ef8ae 100644 --- a/src/main/java/org/opensearch/securityanalytics/logtype/LogTypeService.java +++ b/src/main/java/org/opensearch/securityanalytics/logtype/LogTypeService.java @@ -10,6 +10,7 @@ import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -660,6 +661,13 @@ public void getRuleFieldMappings(String logType, ActionListener getIocFieldsList(String logType) { + LogType logTypeByName = builtinLogTypeLoader.getLogTypeByName(logType); + if(logTypeByName == null) + return Collections.emptyList(); + return logTypeByName.getIocFieldsList(); + } + public void getRuleFieldMappingsAllSchemas(String logType, ActionListener> listener) { if (builtinLogTypeLoader.logTypeExists(logType)) { diff --git a/src/main/java/org/opensearch/securityanalytics/model/LogType.java b/src/main/java/org/opensearch/securityanalytics/model/LogType.java index a983b592f..8cee7ab23 100644 --- a/src/main/java/org/opensearch/securityanalytics/model/LogType.java +++ b/src/main/java/org/opensearch/securityanalytics/model/LogType.java @@ -6,6 +6,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -66,13 +67,16 @@ public LogType(Map logTypeAsMap) { new Mapping(e.get(RAW_FIELD), e.get(ECS), e.get(OCSF)) ).collect(Collectors.toList()); } - - List> iocFieldsList = (List>)logTypeAsMap.get(IOC_FIELDS); - if (iocFieldsList.size() > 0) { - this.iocFieldsList = new ArrayList<>(mappings.size()); - this.iocFieldsList = iocFieldsList.stream().map(e -> - new IocFields(e.get(IOC).toString(), (List)e.get(FIELDS)) - ).collect(Collectors.toList()); + if(logTypeAsMap.containsKey(IOC_FIELDS)) { + List> iocFieldsList = (List>) logTypeAsMap.get(IOC_FIELDS); + if (iocFieldsList.size() > 0) { + this.iocFieldsList = new ArrayList<>(mappings.size()); + this.iocFieldsList = iocFieldsList.stream().map(e -> + new IocFields(e.get(IOC).toString(), (List) e.get(FIELDS)) + ).collect(Collectors.toList()); + } + } else { + iocFieldsList = Collections.emptyList(); } } diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/DetectorThreatIntelService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/DetectorThreatIntelService.java index b0891f413..2ba8b634b 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/DetectorThreatIntelService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/DetectorThreatIntelService.java @@ -1,15 +1,19 @@ package org.opensearch.securityanalytics.threatIntel; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.client.Client; +import org.opensearch.common.settings.Settings; import org.opensearch.commons.alerting.model.DocLevelQuery; import org.opensearch.core.action.ActionListener; -import org.opensearch.core.rest.RestStatus; -import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; import org.opensearch.securityanalytics.model.Detector; +import org.opensearch.securityanalytics.model.LogType; import org.opensearch.securityanalytics.model.ThreatIntelFeedData; -import org.opensearch.securityanalytics.util.SecurityAnalyticsException; +import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.UUID; import java.util.concurrent.CountDownLatch; @@ -19,33 +23,56 @@ public class DetectorThreatIntelService { + private static final Logger log = LogManager.getLogger(DetectorThreatIntelService.class); + private final ThreatIntelFeedDataService threatIntelFeedDataService; public DetectorThreatIntelService(ThreatIntelFeedDataService threatIntelFeedDataService) { this.threatIntelFeedDataService = threatIntelFeedDataService; } + /** * Convert the feed data IOCs into query string query format to create doc level queries. */ - public DocLevelQuery createDocLevelQueryFromThreatIntelList( - List tifdList, String docLevelQueryId + public List createDocLevelQueriesFromThreatIntelList( + List iocFieldList, List tifdList, Detector detector ) { + List queries = new ArrayList<>(); Set iocs = tifdList.stream().map(ThreatIntelFeedData::getIocValue).collect(Collectors.toSet()); - String query = buildQueryStringQueryWithIocList(iocs); - return new DocLevelQuery( - docLevelQueryId, tifdList.get(0).getFeedId(), - Collections.singletonList("*"), - query, - Collections.singletonList("threat_intel") - ); + //ioc types supported by log type + List logTypeIocs = iocFieldList.stream().map(LogType.IocFields::getIoc).collect(Collectors.toList()); + // filter out ioc types not supported for given log types + Map> iocTypeToValues = tifdList.stream().filter(t -> logTypeIocs.contains(t.getIocType())) + .collect(Collectors.groupingBy( + ThreatIntelFeedData::getIocType, + Collectors.mapping(ThreatIntelFeedData::getIocValue, Collectors.toSet()) + )); + + for (Map.Entry> entry : iocTypeToValues.entrySet()) { + String query = buildQueryStringQueryWithIocList(iocs); + List fields = iocFieldList.stream().filter(t -> entry.getKey().matches(t.getIoc())).findFirst().get().getFields(); + + // create doc + for (String field : fields) { //todo increase max clause count from 1024 + queries.add(new DocLevelQuery( + constructId(detector, entry.getKey()), tifdList.get(0).getFeedId(), + Collections.emptyList(), + "windows-hostname:(120.85.114.146 OR 103.104.106.223 OR 185.191.246.45 OR 120.86.237.94)", + List.of("threat_intel", entry.getKey() /*ioc_type*/) + )); + } + } + return queries; } private String buildQueryStringQueryWithIocList(Set iocs) { StringBuilder sb = new StringBuilder(); + sb.append("%s"); + sb.append(":"); sb.append("("); for (String ioc : iocs) { - if (sb.length() > 2) { + if (sb.length() > 4) { sb.append(" OR "); } sb.append(ioc); @@ -55,30 +82,30 @@ private String buildQueryStringQueryWithIocList(Set iocs) { return sb.toString(); } - public void createDocLevelQueryFromThreatIntel(Detector detector, ActionListener listener) { + public void createDocLevelQueryFromThreatIntel(List iocFieldList, Detector detector, ActionListener> listener) { try { - if (detector.getThreatIntelEnabled() == false) { - listener.onResponse(null); + if (false == detector.getThreatIntelEnabled() || iocFieldList.isEmpty()) { + listener.onResponse(Collections.emptyList()); return; - } + CountDownLatch latch = new CountDownLatch(1); threatIntelFeedDataService.getThreatIntelFeedData(new ActionListener<>() { @Override public void onResponse(List threatIntelFeedData) { if (threatIntelFeedData.isEmpty()) { - listener.onResponse(null); + listener.onResponse(Collections.emptyList()); } else { - listener.onResponse(createDocLevelQueryFromThreatIntelList( - threatIntelFeedData, - detector.getName() + "_threat_intel" + UUID.randomUUID() - )); + listener.onResponse( + createDocLevelQueriesFromThreatIntelList(iocFieldList, threatIntelFeedData, detector) + ); } latch.countDown(); } @Override public void onFailure(Exception e) { + log.error("Failed to get threat intel feeds for doc level query creation", e); listener.onFailure(e); latch.countDown(); } @@ -86,11 +113,16 @@ public void onFailure(Exception e) { latch.await(30, TimeUnit.SECONDS); } catch (InterruptedException e) { + log.error("Failed to create doc level queries from threat intel feeds", e); listener.onFailure(e); } } + private static String constructId(Detector detector, String iocType) { + return detector.getName() + "_threat_intel_" + iocType + "_" + UUID.randomUUID(); + } + public void updateDetectorsWithLatestThreatIntelRules() { } diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java index 4805179df..3eb0a5112 100644 --- a/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java +++ b/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java @@ -88,6 +88,7 @@ import org.opensearch.securityanalytics.model.DetectorInput; import org.opensearch.securityanalytics.model.DetectorRule; import org.opensearch.securityanalytics.model.DetectorTrigger; +import org.opensearch.securityanalytics.model.LogType; import org.opensearch.securityanalytics.model.Rule; import org.opensearch.securityanalytics.model.Value; import org.opensearch.securityanalytics.rules.aggregation.AggregationItem; @@ -323,7 +324,9 @@ private void createMonitorFromQueries(List> rulesById, Detect monitorResponses.add(addedFirstMonitorResponse); saveWorkflow(rulesById, detector, monitorResponses, refreshPolicy, listener); }, - listener::onFailure + e -> { + listener.onFailure(e); + } ); } } @@ -653,30 +656,7 @@ private IndexMonitorRequest createDocLevelMonitorRequest(List DocLevelQuery docLevelQuery = new DocLevelQuery(id, name, Collections.emptyList(), actualQuery, tags); docLevelQueries.add(docLevelQuery); } - try { - if (detector.getThreatIntelEnabled()) { - CountDownLatch countDownLatch = new CountDownLatch(1); - detectorThreatIntelService.createDocLevelQueryFromThreatIntel(detector, new ActionListener<>() { - @Override - public void onResponse(DocLevelQuery dlq) { - if (dlq != null) - docLevelQueries.add(dlq); - countDownLatch.countDown(); - } - - @Override - public void onFailure(Exception e) { - // not failing detector creation if any fatal exception occurs during doc level query creation from threat intel feed data - log.error("Failed to convert threat intel feed to. Proceeding with detector creation", e); - countDownLatch.countDown(); - } - }); - countDownLatch.await(); - } - } catch (Exception e) { - // not failing detector creation if any fatal exception occurs during doc level query creation from threat intel feed data - log.error("Failed to convert threat intel feed to. Proceeding with detector creation", e); - } + addThreatIntelBasedDocLevelQueries(detector, docLevelQueries); DocLevelMonitorInput docLevelMonitorInput = new DocLevelMonitorInput(detector.getName(), detector.getInputs().get(0).getIndices(), docLevelQueries); docLevelMonitorInputs.add(docLevelMonitorInput); @@ -707,6 +687,39 @@ public void onFailure(Exception e) { return new IndexMonitorRequest(monitorId, SequenceNumbers.UNASSIGNED_SEQ_NO, SequenceNumbers.UNASSIGNED_PRIMARY_TERM, refreshPolicy, restMethod, monitor, null); } + private void addThreatIntelBasedDocLevelQueries(Detector detector, List docLevelQueries) { + try { + + if (detector.getThreatIntelEnabled()) { + List iocFieldsList = logTypeService.getIocFieldsList(detector.getDetectorType()); + if (iocFieldsList == null || iocFieldsList.isEmpty()) { + + } else { + CountDownLatch countDownLatch = new CountDownLatch(1); + detectorThreatIntelService.createDocLevelQueryFromThreatIntel(iocFieldsList, detector, new ActionListener<>() { + @Override + public void onResponse(List dlqs) { + if (dlqs != null) + docLevelQueries.addAll(dlqs); + countDownLatch.countDown(); + } + + @Override + public void onFailure(Exception e) { + // not failing detector creation if any fatal exception occurs during doc level query creation from threat intel feed data + log.error("Failed to convert threat intel feed to. Proceeding with detector creation", e); + countDownLatch.countDown(); + } + }); + countDownLatch.await(); + } + } + } catch (Exception e) { + // not failing detector creation if any fatal exception occurs during doc level query creation from threat intel feed data + log.error("Failed to convert threat intel feed to doc level query. Proceeding with detector creation", e); + } + } + /** * Creates doc level monitor which generates per document alerts for the findings of the bucket level delegate monitors in a workflow. * This monitor has match all query applied to generate the alerts per each finding doc. diff --git a/src/main/resources/OSMapping/test_windows_logtype.json b/src/main/resources/OSMapping/test_windows_logtype.json index 59e9cecad..cc619c5a1 100644 --- a/src/main/resources/OSMapping/test_windows_logtype.json +++ b/src/main/resources/OSMapping/test_windows_logtype.json @@ -2,8 +2,13 @@ "name": "test_windows", "description": "Test Log Type used by tests. It is created as a lightweight log type for integration tests", "is_builtin": true, - "ioc_fields" : [], - "mappings":[ + "ioc_fields": [ + { + "ioc": "ip", + "fields": ["HostName"] + } + ], + "mappings": [ { "raw_field":"EventID", "ecs":"event_uid" diff --git a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java index abc9caad8..5114d1504 100644 --- a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java +++ b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java @@ -1373,6 +1373,46 @@ public static String randomDoc(int severity, int version, String opCode) { } + //Add IPs in HostName field. + public static String randomDocWithIpIoc(int severity, int version, String ioc) { + String doc = "{\n" + + "\"EventTime\":\"2020-02-04T14:59:39.343541+00:00\",\n" + + "\"HostName\":\"%s\",\n" + + "\"Keywords\":\"9223372036854775808\",\n" + + "\"SeverityValue\":%s,\n" + + "\"Severity\":\"INFO\",\n" + + "\"EventID\":22,\n" + + "\"SourceName\":\"Microsoft-Windows-Sysmon\",\n" + + "\"ProviderGuid\":\"{5770385F-C22A-43E0-BF4C-06F5698FFBD9}\",\n" + + "\"Version\":%s,\n" + + "\"TaskValue\":22,\n" + + "\"OpcodeValue\":0,\n" + + "\"RecordNumber\":9532,\n" + + "\"ExecutionProcessID\":1996,\n" + + "\"ExecutionThreadID\":2616,\n" + + "\"Channel\":\"Microsoft-Windows-Sysmon/Operational\",\n" + + "\"Domain\":\"NT AUTHORITY\",\n" + + "\"AccountName\":\"SYSTEM\",\n" + + "\"UserID\":\"S-1-5-18\",\n" + + "\"AccountType\":\"User\",\n" + + "\"Message\":\"Dns query:\\r\\nRuleName: \\r\\nUtcTime: 2020-02-04 14:59:38.349\\r\\nProcessGuid: {b3c285a4-3cda-5dc0-0000-001077270b00}\\r\\nProcessId: 1904\\r\\nQueryName: EC2AMAZ-EPO7HKA\\r\\nQueryStatus: 0\\r\\nQueryResults: 172.31.46.38;\\r\\nImage: C:\\\\Program Files\\\\nxlog\\\\nxlog.exe\",\n" + + "\"Category\":\"Dns query (rule: DnsQuery)\",\n" + + "\"Opcode\":\"%blahblah\",\n" + + "\"UtcTime\":\"2020-02-04 14:59:38.349\",\n" + + "\"ProcessGuid\":\"{b3c285a4-3cda-5dc0-0000-001077270b00}\",\n" + + "\"ProcessId\":\"1904\",\"QueryName\":\"EC2AMAZ-EPO7HKA\",\"QueryStatus\":\"0\",\n" + + "\"QueryResults\":\"172.31.46.38;\",\n" + + "\"Image\":\"C:\\\\Program Files\\\\nxlog\\\\regsvr32.exe\",\n" + + "\"EventReceivedTime\":\"2020-02-04T14:59:40.780905+00:00\",\n" + + "\"SourceModuleName\":\"in\",\n" + + "\"SourceModuleType\":\"im_msvistalog\",\n" + + "\"CommandLine\": \"eachtest\",\n" + + "\"Initiated\": \"true\"\n" + + "}"; + return String.format(Locale.ROOT, ioc, doc, severity, version); + + } + public static String randomDoc() { return "{\n" + "\"@timestamp\":\"2020-02-04T14:59:39.343541+00:00\",\n" + diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java index 15e9f9bad..f6f34ea6b 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java @@ -1088,6 +1088,7 @@ public void testCreateDetector_threatIntelEnabled_updateDetectorWithNewThreatInt "}"; SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); + assertEquals(2, response.getHits().getTotalHits().value); assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); @@ -1118,7 +1119,7 @@ public void testCreateDetector_threatIntelEnabled_updateDetectorWithNewThreatInt List iocs = getThreatIntelFeedIocs(3); int i=1; for (String ioc : iocs) { - indexDoc(index, i+"", randomDoc(5, 3, ioc)); + indexDoc(index, i+"", randomDoc(5, 3, i==1? "120.85.114.146" : "120.86.237.94")); i++; } String workflowId = ((List) detectorMap.get("workflow_ids")).get(0);