diff --git a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java index bdf67ee50..e7fe43106 100644 --- a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java +++ b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java @@ -295,6 +295,7 @@ public List> getSettings() { SecurityAnalyticsSettings.CORRELATION_HISTORY_RETENTION_PERIOD, SecurityAnalyticsSettings.IS_CORRELATION_INDEX_SETTING, SecurityAnalyticsSettings.CORRELATION_TIME_WINDOW, + SecurityAnalyticsSettings.ENABLE_AUTO_CORRELATIONS, SecurityAnalyticsSettings.DEFAULT_MAPPING_SCHEMA, SecurityAnalyticsSettings.ENABLE_WORKFLOW_USAGE, SecurityAnalyticsSettings.TIF_UPDATE_INTERVAL, diff --git a/src/main/java/org/opensearch/securityanalytics/correlation/JoinEngine.java b/src/main/java/org/opensearch/securityanalytics/correlation/JoinEngine.java index 8c986fd3a..8a907cb71 100644 --- a/src/main/java/org/opensearch/securityanalytics/correlation/JoinEngine.java +++ b/src/main/java/org/opensearch/securityanalytics/correlation/JoinEngine.java @@ -4,7 +4,8 @@ */ package org.opensearch.securityanalytics.correlation; -import kotlin.Pair; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.commons.lang3.tuple.Triple; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.lucene.search.join.ScoreMode; @@ -49,7 +50,6 @@ import java.util.Map; import java.util.Optional; import java.util.Set; -import java.util.function.Predicate; import java.util.stream.Collectors; @@ -63,6 +63,8 @@ public class JoinEngine { private volatile long corrTimeWindow; + private volatile boolean enableAutoCorrelations; + private final TransportCorrelateFindingAction.AsyncCorrelateFindingAction correlateFindingAction; private final LogTypeService logTypeService; @@ -71,18 +73,23 @@ public class JoinEngine { public JoinEngine(Client client, PublishFindingsRequest request, NamedXContentRegistry xContentRegistry, long corrTimeWindow, TransportCorrelateFindingAction.AsyncCorrelateFindingAction correlateFindingAction, - LogTypeService logTypeService) { + LogTypeService logTypeService, boolean enableAutoCorrelations) { this.client = client; this.request = request; this.xContentRegistry = xContentRegistry; this.corrTimeWindow = corrTimeWindow; this.correlateFindingAction = correlateFindingAction; this.logTypeService = logTypeService; + this.enableAutoCorrelations = enableAutoCorrelations; } public void onSearchDetectorResponse(Detector detector, Finding finding) { try { - generateAutoCorrelations(detector, finding); + if (enableAutoCorrelations) { + generateAutoCorrelations(detector, finding); + } else { + onAutoCorrelations(detector, finding, Map.of()); + } } catch (IOException ex) { correlateFindingAction.onFailures(ex); } @@ -265,6 +272,7 @@ public void onFailure(Exception e) { private void getValidDocuments(String detectorType, List indices, List correlationRules, List relatedDocIds, Map> autoCorrelations) { MultiSearchRequest mSearchRequest = new MultiSearchRequest(); List validCorrelationRules = new ArrayList<>(); + List validFields = new ArrayList<>(); for (CorrelationRule rule: correlationRules) { Optional query = rule.getCorrelationQueries().stream() @@ -272,12 +280,19 @@ private void getValidDocuments(String detectorType, List indices, List indices, List indices, List filteredCorrelationRules = new ArrayList<>(); + List> filteredCorrelationRules = new ArrayList<>(); int idx = 0; for (MultiSearchResponse.Item response : responses) { @@ -304,14 +320,17 @@ public void onResponse(MultiSearchResponse items) { } if (response.getResponse().getHits().getTotalHits().value > 0L) { - filteredCorrelationRules.add(validCorrelationRules.get(idx)); + filteredCorrelationRules.add(Triple.of(validCorrelationRules.get(idx), + response.getResponse().getHits().getHits(), validFields.get(idx))); } ++idx; } Map> categoryToQueriesMap = new HashMap<>(); - for (CorrelationRule rule: filteredCorrelationRules) { - List queries = rule.getCorrelationQueries(); + Map categoryToTimeWindowMap = new HashMap<>(); + for (Triple rule: filteredCorrelationRules) { + List queries = rule.getLeft().getCorrelationQueries(); + Long timeWindow = rule.getLeft().getCorrTimeWindow(); for (CorrelationQuery query: queries) { List correlationQueries; @@ -320,12 +339,36 @@ public void onResponse(MultiSearchResponse items) { } else { correlationQueries = new ArrayList<>(); } - correlationQueries.add(query); + if (categoryToTimeWindowMap.containsKey(query.getCategory())) { + categoryToTimeWindowMap.put(query.getCategory(), Math.max(timeWindow, categoryToTimeWindowMap.get(query.getCategory()))); + } else { + categoryToTimeWindowMap.put(query.getCategory(), timeWindow); + } + + if (query.getField() == null) { + correlationQueries.add(query); + } else { + SearchHit[] hits = rule.getMiddle(); + StringBuilder qb = new StringBuilder(query.getField()).append(":("); + for (int i = 0; i < hits.length; ++i) { + String value = hits[i].field(rule.getRight()).getValue(); + qb.append(value); + if (i < hits.length-1) { + qb.append(" OR "); + } else { + qb.append(")"); + } + } + if (query.getQuery() != null) { + qb.append(" AND ").append(query.getQuery()); + } + correlationQueries.add(new CorrelationQuery(query.getIndex(), qb.toString(), query.getCategory(), null)); + } categoryToQueriesMap.put(query.getCategory(), correlationQueries); } } - searchFindingsByTimestamp(detectorType, categoryToQueriesMap, - filteredCorrelationRules.stream().map(CorrelationRule::getId).collect(Collectors.toList()), + searchFindingsByTimestamp(detectorType, categoryToQueriesMap, categoryToTimeWindowMap, + filteredCorrelationRules.stream().map(Triple::getLeft).map(CorrelationRule::getId).collect(Collectors.toList()), autoCorrelations ); } @@ -348,15 +391,15 @@ public void onFailure(Exception e) { * this method searches for parent findings given the log category & correlation time window & collects all related docs * for them. */ - private void searchFindingsByTimestamp(String detectorType, Map> categoryToQueriesMap, List correlationRules, Map> autoCorrelations) { + private void searchFindingsByTimestamp(String detectorType, Map> categoryToQueriesMap, Map categoryToTimeWindowMap, List correlationRules, Map> autoCorrelations) { long findingTimestamp = request.getFinding().getTimestamp().toEpochMilli(); MultiSearchRequest mSearchRequest = new MultiSearchRequest(); List>> categoryToQueriesPairs = new ArrayList<>(); for (Map.Entry> categoryToQueries: categoryToQueriesMap.entrySet()) { RangeQueryBuilder queryBuilder = QueryBuilders.rangeQuery("timestamp") - .gte(findingTimestamp - corrTimeWindow) - .lte(findingTimestamp + corrTimeWindow); + .gte(findingTimestamp - categoryToTimeWindowMap.get(categoryToQueries.getKey())) + .lte(findingTimestamp + categoryToTimeWindowMap.get(categoryToQueries.getKey())); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); searchSourceBuilder.query(queryBuilder); @@ -368,7 +411,7 @@ private void searchFindingsByTimestamp(String detectorType, Map(categoryToQueries.getKey(), categoryToQueries.getValue())); + categoryToQueriesPairs.add(Pair.of(categoryToQueries.getKey(), categoryToQueries.getValue())); } if (!mSearchRequest.requests().isEmpty()) { @@ -392,17 +435,17 @@ public void onResponse(MultiSearchResponse items) { .map(Object::toString).collect(Collectors.toList())); } - List correlationQueries = categoryToQueriesPairs.get(idx).getSecond(); + List correlationQueries = categoryToQueriesPairs.get(idx).getValue(); List indices = correlationQueries.stream().map(CorrelationQuery::getIndex).collect(Collectors.toList()); List queries = correlationQueries.stream().map(CorrelationQuery::getQuery).collect(Collectors.toList()); - relatedDocsMap.put(categoryToQueriesPairs.get(idx).getFirst(), + relatedDocsMap.put(categoryToQueriesPairs.get(idx).getKey(), new DocSearchCriteria( indices, queries, relatedDocIds)); ++idx; } - searchDocsWithFilterKeys(detectorType, relatedDocsMap, correlationRules, autoCorrelations); + searchDocsWithFilterKeys(detectorType, relatedDocsMap, categoryToTimeWindowMap, correlationRules, autoCorrelations); } @Override @@ -422,7 +465,7 @@ public void onFailure(Exception e) { /** * Given the related docs from parent findings, this method filters only those related docs which match parent join criteria. */ - private void searchDocsWithFilterKeys(String detectorType, Map relatedDocsMap, List correlationRules, Map> autoCorrelations) { + private void searchDocsWithFilterKeys(String detectorType, Map relatedDocsMap, Map categoryToTimeWindowMap, List correlationRules, Map> autoCorrelations) { MultiSearchRequest mSearchRequest = new MultiSearchRequest(); List categories = new ArrayList<>(); @@ -433,6 +476,7 @@ private void searchDocsWithFilterKeys(String detectorType, Map> filteredRelatedDocIds, List correlationRules, Map> autoCorrelations) { + private void getCorrelatedFindings(String detectorType, Map> filteredRelatedDocIds, Map categoryToTimeWindowMap, List correlationRules, Map> autoCorrelations) { long findingTimestamp = request.getFinding().getTimestamp().toEpochMilli(); MultiSearchRequest mSearchRequest = new MultiSearchRequest(); List categories = new ArrayList<>(); @@ -499,8 +543,8 @@ private void getCorrelatedFindings(String detectorType, Map for (Map.Entry> relatedDocIds: filteredRelatedDocIds.entrySet()) { BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery() .filter(QueryBuilders.rangeQuery("timestamp") - .gte(findingTimestamp - corrTimeWindow) - .lte(findingTimestamp + corrTimeWindow)) + .gte(findingTimestamp - categoryToTimeWindowMap.get(relatedDocIds.getKey())) + .lte(findingTimestamp + categoryToTimeWindowMap.get(relatedDocIds.getKey()))) .must(QueryBuilders.termsQuery("correlated_doc_ids", relatedDocIds.getValue())); if (relatedDocIds.getKey().equals(detectorType)) { diff --git a/src/main/java/org/opensearch/securityanalytics/model/CorrelationQuery.java b/src/main/java/org/opensearch/securityanalytics/model/CorrelationQuery.java index 480e8185d..0c8f70094 100644 --- a/src/main/java/org/opensearch/securityanalytics/model/CorrelationQuery.java +++ b/src/main/java/org/opensearch/securityanalytics/model/CorrelationQuery.java @@ -22,33 +22,45 @@ public class CorrelationQuery implements Writeable, ToXContentObject { private static final String QUERY = "query"; private static final String CATEGORY = "category"; + private static final String FIELD = "field"; + private String index; private String query; private String category; - public CorrelationQuery(String index, String query, String category) { + private String field; + + public CorrelationQuery(String index, String query, String category, String field) { this.index = index; this.query = query; this.category = category; + this.field = field; } public CorrelationQuery(StreamInput sin) throws IOException { - this(sin.readString(), sin.readString(), sin.readString()); + this(sin.readString(), sin.readOptionalString(), sin.readString(), sin.readOptionalString()); } @Override public void writeTo(StreamOutput out) throws IOException { out.writeString(index); - out.writeString(query); + out.writeOptionalString(query); out.writeString(category); + out.writeOptionalString(field); } @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - builder.field(INDEX, index).field(QUERY, query).field(CATEGORY, category); + builder.field(INDEX, index).field(CATEGORY, category); + if (query != null) { + builder.field(QUERY, query); + } + if (field != null) { + builder.field(FIELD, field); + } return builder.endObject(); } @@ -56,6 +68,7 @@ public static CorrelationQuery parse(XContentParser xcp) throws IOException { String index = null; String query = null; String category = null; + String field = null; XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp); while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { @@ -72,11 +85,14 @@ public static CorrelationQuery parse(XContentParser xcp) throws IOException { case CATEGORY: category = xcp.text(); break; + case FIELD: + field = xcp.text(); + break; default: xcp.skipChildren(); } } - return new CorrelationQuery(index, query, category); + return new CorrelationQuery(index, query, category, field); } public static CorrelationQuery readFrom(StreamInput sin) throws IOException { @@ -94,4 +110,8 @@ public String getQuery() { public String getCategory() { return category; } + + public String getField() { + return field; + } } \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/model/CorrelationRule.java b/src/main/java/org/opensearch/securityanalytics/model/CorrelationRule.java index 685b66c9b..b7f5a4f70 100644 --- a/src/main/java/org/opensearch/securityanalytics/model/CorrelationRule.java +++ b/src/main/java/org/opensearch/securityanalytics/model/CorrelationRule.java @@ -28,6 +28,7 @@ public class CorrelationRule implements Writeable, ToXContentObject { public static final String NO_ID = ""; public static final Long NO_VERSION = 1L; private static final String CORRELATION_QUERIES = "correlate"; + private static final String CORRELATION_TIME_WINDOW = "time_window"; private String id; @@ -37,15 +38,18 @@ public class CorrelationRule implements Writeable, ToXContentObject { private List correlationQueries; - public CorrelationRule(String id, Long version, String name, List correlationQueries) { + private Long corrTimeWindow; + + public CorrelationRule(String id, Long version, String name, List correlationQueries, Long corrTimeWindow) { this.id = id != null ? id : NO_ID; this.version = version != null ? version : NO_VERSION; this.name = name; this.correlationQueries = correlationQueries; + this.corrTimeWindow = corrTimeWindow != null? corrTimeWindow: 300000L; } public CorrelationRule(StreamInput sin) throws IOException { - this(sin.readString(), sin.readLong(), sin.readString(), sin.readList(CorrelationQuery::readFrom)); + this(sin.readString(), sin.readLong(), sin.readString(), sin.readList(CorrelationQuery::readFrom), sin.readLong()); } @Override @@ -57,6 +61,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws CorrelationQuery[] correlationQueries = new CorrelationQuery[] {}; correlationQueries = this.correlationQueries.toArray(correlationQueries); builder.field(CORRELATION_QUERIES, correlationQueries); + builder.field(CORRELATION_TIME_WINDOW, corrTimeWindow); return builder.endObject(); } @@ -69,6 +74,7 @@ public void writeTo(StreamOutput out) throws IOException { for (CorrelationQuery query : correlationQueries) { query.writeTo(out); } + out.writeLong(corrTimeWindow); } public static CorrelationRule parse(XContentParser xcp, String id, Long version) throws IOException { @@ -81,6 +87,7 @@ public static CorrelationRule parse(XContentParser xcp, String id, Long version) String name = null; List correlationQueries = new ArrayList<>(); + Long corrTimeWindow = null; XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.nextToken(), xcp); while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { @@ -98,11 +105,14 @@ public static CorrelationRule parse(XContentParser xcp, String id, Long version) correlationQueries.add(query); } break; + case CORRELATION_TIME_WINDOW: + corrTimeWindow = xcp.longValue(); + break; default: xcp.skipChildren(); } } - return new CorrelationRule(id, version, name, correlationQueries); + return new CorrelationRule(id, version, name, correlationQueries, corrTimeWindow); } public static CorrelationRule readFrom(StreamInput sin) throws IOException { @@ -137,6 +147,10 @@ public List getCorrelationQueries() { return correlationQueries; } + public Long getCorrTimeWindow() { + return corrTimeWindow; + } + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/src/main/java/org/opensearch/securityanalytics/settings/SecurityAnalyticsSettings.java b/src/main/java/org/opensearch/securityanalytics/settings/SecurityAnalyticsSettings.java index b55548069..9abc49760 100644 --- a/src/main/java/org/opensearch/securityanalytics/settings/SecurityAnalyticsSettings.java +++ b/src/main/java/org/opensearch/securityanalytics/settings/SecurityAnalyticsSettings.java @@ -138,6 +138,12 @@ public class SecurityAnalyticsSettings { Setting.Property.NodeScope, Setting.Property.Dynamic ); + public static final Setting ENABLE_AUTO_CORRELATIONS = Setting.boolSetting( + "plugins.security_analytics.enable_auto_correlations", + false, + Setting.Property.NodeScope, Setting.Property.Dynamic + ); + public static final Setting DEFAULT_MAPPING_SCHEMA = Setting.simpleString( "plugins.security_analytics.mappings.default_schema", "ecs", diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportCorrelateFindingAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportCorrelateFindingAction.java index db2b0287d..b7a906159 100644 --- a/src/main/java/org/opensearch/securityanalytics/transport/TransportCorrelateFindingAction.java +++ b/src/main/java/org/opensearch/securityanalytics/transport/TransportCorrelateFindingAction.java @@ -96,6 +96,8 @@ public class TransportCorrelateFindingAction extends HandledTransportAction indexTimeout = it); this.clusterService.getClusterSettings().addSettingsUpdateConsumer(SecurityAnalyticsSettings.CORRELATION_TIME_WINDOW, it -> corrTimeWindow = it.getMillis()); + this.clusterService.getClusterSettings().addSettingsUpdateConsumer(SecurityAnalyticsSettings.ENABLE_AUTO_CORRELATIONS, it -> enableAutoCorrelation = it); this.setupTimestamp = System.currentTimeMillis(); } @@ -220,7 +224,7 @@ public class AsyncCorrelateFindingAction { this.response =new AtomicReference<>(); - this.joinEngine = new JoinEngine(client, request, xContentRegistry, corrTimeWindow, this, logTypeService); + this.joinEngine = new JoinEngine(client, request, xContentRegistry, corrTimeWindow, this, logTypeService, enableAutoCorrelation); this.vectorEmbeddingsEngine = new VectorEmbeddingsEngine(client, indexTimeout, corrTimeWindow, this); } diff --git a/src/main/resources/mappings/finding_mapping.json b/src/main/resources/mappings/finding_mapping.json index 18faa22f6..32a2378fb 100644 --- a/src/main/resources/mappings/finding_mapping.json +++ b/src/main/resources/mappings/finding_mapping.json @@ -46,6 +46,9 @@ "type" : "keyword" } } + }, + "fields": { + "type": "text" } } }, diff --git a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java index 774143aba..97c192104 100644 --- a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java +++ b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java @@ -225,9 +225,9 @@ public static CorrelationRule randomCorrelationRule(String name) { name = name.isEmpty()? ">": name; return new CorrelationRule(CorrelationRule.NO_ID, CorrelationRule.NO_VERSION, name, List.of( - new CorrelationQuery("vpc_flow1", "dstaddr:192.168.1.*", "network"), - new CorrelationQuery("ad_logs1", "azure.platformlogs.result_type:50126", "ad_ldap") - )); + new CorrelationQuery("vpc_flow1", "dstaddr:192.168.1.*", "network", null), + new CorrelationQuery("ad_logs1", "azure.platformlogs.result_type:50126", "ad_ldap", null) + ), 300000L); } public static String randomRule() { @@ -259,6 +259,29 @@ public static String randomRule() { "level: high"; } + public static String randomRuleForCorrelations(String value) { + return "id: 5f92fff9-82e2-48ab-8fc1-8b133556a551\n" + + "logsource:\n" + + " product: cloudtrail\n" + + "title: AWS User Created\n" + + "description: AWS User Created\n" + + "tags:\n" + + " - attack.test1\n" + + "falsepositives:\n" + + " - Legit User Account Administration\n" + + "level: high\n" + + "date: 2022/01/01\n" + + "status: experimental\n" + + "references:\n" + + " - 'https://github.com/RhinoSecurityLabs/AWS-IAM-Privilege-Escalation'\n" + + "author: toffeebr33k\n" + + "detection:\n" + + " condition: selection_source\n" + + " selection_source:\n" + + " EventName:\n" + + " - " + value; + } + public static String randomRuleForMappingView(String field) { return "title: Remote Encrypting File System Abuse\n" + "id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" + @@ -837,6 +860,29 @@ public static String randomAggregationRule(String aggFunction, String signAndVa return String.format(Locale.ROOT, rule, opCode, aggFunction, signAndValue); } + public static String randomCloudtrailAggrRule() { + return "id: c64c5175-5189-431b-a55e-6d9882158250\n" + + "logsource:\n" + + " product: cloudtrail\n" + + "title: Accounts created and deleted within 24h\n" + + "description: Flag suspicious activity of accounts created and deleted within 24h\n" + + "date: 2021/09/23\n" + + "tags:\n" + + " - attack.exfiltration\n" + + "falsepositives: [ ]\n" + + "level: high\n" + + "status: test\n" + + "references: [ ]\n" + + "author: Sashank\n" + + "detection:\n" + + " selection:\n" + + " EventName:\n" + + " - CREATED\n" + + " - DELETED\n" + + " timeframe: 24h\n" + + " condition: selection | count(*) by AccountName >= 2"; + } + public static String windowsIndexMapping() { return "\"properties\": {\n" + " \"@timestamp\": {\"type\":\"date\"},\n" + @@ -850,7 +896,10 @@ public static String windowsIndexMapping() { " \"type\": \"text\"\n" + " },\n" + " \"AccountName\": {\n" + - " \"type\": \"text\"\n" + + " \"type\": \"keyword\"\n" + + " },\n" + + " \"EventName\": {\n" + + " \"type\": \"keyword\"\n" + " },\n" + " \"AccountType\": {\n" + " \"type\": \"text\",\n" + @@ -991,7 +1040,7 @@ public static String windowsIndexMapping() { " \"type\": \"date\"\n" + " },\n" + " \"EventType\": {\n" + - " \"type\": \"integer\"\n" + + " \"type\": \"keyword\"\n" + " },\n" + " \"ExecutionProcessID\": {\n" + " \"type\": \"long\"\n" + @@ -1614,6 +1663,7 @@ public static String randomDoc() { "\"Severity\":\"INFO\",\n" + "\"EventID\":22,\n" + "\"SourceName\":\"Microsoft-Windows-Sysmon\",\n" + + "\"SourceIp\":\"1.2.3.4\",\n" + "\"ProviderGuid\":\"{5770385F-C22A-43E0-BF4C-06F5698FFBD9}\",\n" + "\"Version\":5,\n" + "\"TaskValue\":22,\n" + @@ -1642,6 +1692,13 @@ public static String randomDoc() { "}"; } + public static String randomCloudtrailAggrDoc(String eventType, String accountId) { + return "{\n" + + " \"AccountName\": \"" + accountId + "\",\n" + + " \"EventType\": \"" + eventType + "\"\n" + + "}"; + } + public static String randomVpcFlowDoc() { return "{\n" + " \"version\": 1,\n" + @@ -1664,6 +1721,60 @@ public static String randomAdLdapDoc() { "}"; } + public static String randomCloudtrailDoc(String user, String event) { + return "{\n" + + " \"eventVersion\": \"1.08\",\n" + + " \"userIdentity\": {\n" + + " \"type\": \"IAMUser\",\n" + + " \"principalId\": \"AIDA6ON6E4XEGITEXAMPLE\",\n" + + " \"arn\": \"arn:aws:iam::888888888888:user/Mary\",\n" + + " \"accountId\": \"888888888888\",\n" + + " \"accessKeyId\": \"AKIAIOSFODNN7EXAMPLE\",\n" + + " \"userName\": \"Mary\",\n" + + " \"sessionContext\": {\n" + + " \"sessionIssuer\": {},\n" + + " \"webIdFederationData\": {},\n" + + " \"attributes\": {\n" + + " \"creationDate\": \"2023-07-19T21:11:57Z\",\n" + + " \"mfaAuthenticated\": \"false\"\n" + + " }\n" + + " }\n" + + " },\n" + + " \"eventTime\": \"2023-07-19T21:25:09Z\",\n" + + " \"eventSource\": \"iam.amazonaws.com\",\n" + + " \"EventName\": \"" + event + "\",\n" + + " \"awsRegion\": \"us-east-1\",\n" + + " \"sourceIPAddress\": \"192.0.2.0\",\n" + + " \"AccountName\": \"" + user + "\",\n" + + " \"userAgent\": \"aws-cli/2.13.5 Python/3.11.4 Linux/4.14.255-314-253.539.amzn2.x86_64 exec-env/CloudShell exe/x86_64.amzn.2 prompt/off command/iam.create-user\",\n" + + " \"requestParameters\": {\n" + + " \"userName\": \"" + user + "\"\n" + + " },\n" + + " \"responseElements\": {\n" + + " \"user\": {\n" + + " \"path\": \"/\",\n" + + " \"arn\": \"arn:aws:iam::888888888888:user/Richard\",\n" + + " \"userId\": \"AIDA6ON6E4XEP7EXAMPLE\",\n" + + " \"createDate\": \"Jul 19, 2023 9:25:09 PM\",\n" + + " \"userName\": \"Richard\"\n" + + " }\n" + + " },\n" + + " \"requestID\": \"2d528c76-329e-410b-9516-EXAMPLE565dc\",\n" + + " \"eventID\": \"ba0801a1-87ec-4d26-be87-EXAMPLE75bbb\",\n" + + " \"readOnly\": false,\n" + + " \"eventType\": \"AwsApiCall\",\n" + + " \"managementEvent\": true,\n" + + " \"recipientAccountId\": \"888888888888\",\n" + + " \"eventCategory\": \"Management\",\n" + + " \"tlsDetails\": {\n" + + " \"tlsVersion\": \"TLSv1.2\",\n" + + " \"cipherSuite\": \"ECDHE-RSA-AES128-GCM-SHA256\",\n" + + " \"clientProvidedHostHeader\": \"iam.amazonaws.com\"\n" + + " },\n" + + " \"sessionCredentialFromConsole\": \"true\"\n" + + "}"; + } + public static String randomAppLogDoc() { return "{\n" + " \"endpoint\": \"/customer_records.txt\",\n" + @@ -1694,6 +1805,312 @@ public static String adLdapLogMappings() { " }"; } + public static String cloudtrailMappings() { + return "\"properties\": {\n" + + " \"Records\": {\n" + + " \"properties\": {\n" + + " \"awsRegion\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"eventCategory\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"eventID\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"eventName\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"eventSource\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"eventTime\": {\n" + + " \"type\": \"date\"\n" + + " },\n" + + " \"eventType\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"eventVersion\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"managementEvent\": {\n" + + " \"type\": \"boolean\"\n" + + " },\n" + + " \"readOnly\": {\n" + + " \"type\": \"boolean\"\n" + + " },\n" + + " \"recipientAccountId\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"requestID\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"requestParameters\": {\n" + + " \"properties\": {\n" + + " \"userName\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " },\n" + + " \"responseElements\": {\n" + + " \"properties\": {\n" + + " \"user\": {\n" + + " \"properties\": {\n" + + " \"arn\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"createDate\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"path\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"userId\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"userName\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " },\n" + + " \"sessionCredentialFromConsole\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"sourceIPAddress\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"tlsDetails\": {\n" + + " \"properties\": {\n" + + " \"cipherSuite\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"clientProvidedHostHeader\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"tlsVersion\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " },\n" + + " \"userAgent\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"userIdentity\": {\n" + + " \"properties\": {\n" + + " \"accessKeyId\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"accountId\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"arn\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"principalId\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"sessionContext\": {\n" + + " \"properties\": {\n" + + " \"attributes\": {\n" + + " \"properties\": {\n" + + " \"creationDate\": {\n" + + " \"type\": \"date\"\n" + + " },\n" + + " \"mfaAuthenticated\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " },\n" + + " \"sessionIssuer\": {\n" + + " \"type\": \"object\"\n" + + " },\n" + + " \"webIdFederationData\": {\n" + + " \"type\": \"object\"\n" + + " }\n" + + " }\n" + + " },\n" + + " \"type\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"userName\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " }}"; + } + public static String s3AccessLogMappings() { return " \"properties\": {" + " \"aws.cloudtrail.eventSource\": {" + diff --git a/src/test/java/org/opensearch/securityanalytics/correlation/CorrelationEngineRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/correlation/CorrelationEngineRestApiIT.java index b511bb9c1..a1ae87fba 100644 --- a/src/test/java/org/opensearch/securityanalytics/correlation/CorrelationEngineRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/correlation/CorrelationEngineRestApiIT.java @@ -4,6 +4,8 @@ */ package org.opensearch.securityanalytics.correlation; +import org.apache.hc.core5.http.io.entity.StringEntity; +import org.apache.hc.core5.http.message.BasicHeader; import org.junit.Assert; import org.opensearch.client.Request; import org.opensearch.client.Response; @@ -18,7 +20,6 @@ import org.opensearch.securityanalytics.model.DetectorRule; import org.opensearch.securityanalytics.model.DetectorTrigger; import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; -import org.opensearch.securityanalytics.util.CorrelationIndices; import java.io.IOException; import java.util.Collections; @@ -33,6 +34,8 @@ public class CorrelationEngineRestApiIT extends SecurityAnalyticsRestTestCase { @SuppressWarnings("unchecked") public void testBasicCorrelationEngineWorkflow() throws IOException, InterruptedException { + updateClusterSetting(SecurityAnalyticsSettings.ENABLE_AUTO_CORRELATIONS.getKey(), "true"); + LogIndices indices = createIndices(); String vpcFlowMonitorId = createVpcFlowDetector(indices.vpcFlowsIndex); @@ -73,6 +76,7 @@ public void testBasicCorrelationEngineWorkflow() throws IOException, Interrupted executeResults = entityAsMap(executeResponse); noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); Assert.assertEquals(1, noOfSigmaRuleMatches); + Thread.sleep(5000); // Call GetFindings API Map params = new HashMap<>(); @@ -163,6 +167,7 @@ public void testListCorrelationsWorkflow() throws IOException, InterruptedExcept @SuppressWarnings("unchecked") public void testBasicCorrelationEngineWorkflowWithoutRules() throws IOException, InterruptedException { + updateClusterSetting(SecurityAnalyticsSettings.ENABLE_AUTO_CORRELATIONS.getKey(), "true"); LogIndices indices = createIndices(); String vpcFlowMonitorId = createVpcFlowDetector(indices.vpcFlowsIndex); @@ -230,6 +235,7 @@ public void testBasicCorrelationEngineWorkflowWithoutRules() throws IOException, @SuppressWarnings("unchecked") public void testBasicCorrelationEngineWorkflowWithRolloverByMaxAge() throws IOException, InterruptedException { + updateClusterSetting(SecurityAnalyticsSettings.ENABLE_AUTO_CORRELATIONS.getKey(), "true"); updateClusterSetting(SecurityAnalyticsSettings.CORRELATION_HISTORY_ROLLOVER_PERIOD.getKey(), "1s"); updateClusterSetting(SecurityAnalyticsSettings.CORRELATION_HISTORY_INDEX_MAX_AGE.getKey(), "1s"); @@ -324,6 +330,7 @@ public void testBasicCorrelationEngineWorkflowWithRolloverByMaxAge() throws IOEx } public void testBasicCorrelationEngineWorkflowWithRolloverByMaxDoc() throws IOException, InterruptedException { + updateClusterSetting(SecurityAnalyticsSettings.ENABLE_AUTO_CORRELATIONS.getKey(), "true"); updateClusterSetting(SecurityAnalyticsSettings.CORRELATION_HISTORY_ROLLOVER_PERIOD.getKey(), "1s"); updateClusterSetting(SecurityAnalyticsSettings.CORRELATION_HISTORY_MAX_DOCS.getKey(), "1"); @@ -418,6 +425,7 @@ public void testBasicCorrelationEngineWorkflowWithRolloverByMaxDoc() throws IOEx } public void testBasicCorrelationEngineWorkflowWithRolloverByMaxDocAndShortRetention() throws IOException, InterruptedException { + updateClusterSetting(SecurityAnalyticsSettings.ENABLE_AUTO_CORRELATIONS.getKey(), "true"); updateClusterSetting(SecurityAnalyticsSettings.CORRELATION_HISTORY_ROLLOVER_PERIOD.getKey(), "1s"); updateClusterSetting(SecurityAnalyticsSettings.CORRELATION_HISTORY_MAX_DOCS.getKey(), "1"); @@ -520,6 +528,283 @@ public void testBasicCorrelationEngineWorkflowWithRolloverByMaxDocAndShortRetent } } + public void testBasicCorrelationEngineWorkflowWithFieldBasedRules() throws IOException, InterruptedException { + Long startTime = System.currentTimeMillis(); + String index = createTestIndex("cloudtrail", cloudtrailMappings()); + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{\n" + + " \"index_name\": \"" + index + "\",\n" + + " \"rule_topic\": \"cloudtrail\",\n" + + " \"partial\": true,\n" + + " \"alias_mappings\": {\n" + + " \"properties\": {\n" + + " \"aws.cloudtrail.event_name\": {\n" + + " \"path\": \"Records.eventName\",\n" + + " \"type\": \"alias\"\n" + + " }\n" + + " }\n" + + " }\n" + + "}" + ); + + Response response = client().performRequest(createMappingRequest); + assertEquals(RestStatus.OK.getStatus(), response.getStatusLine().getStatusCode()); + + String rule1 = randomRuleForCorrelations("CreateUser"); + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.RULE_BASE_URI, Collections.singletonMap("category", "cloudtrail"), + new StringEntity(rule1), new BasicHeader("Content-Type", "application/json")); + Assert.assertEquals("Create rule failed", RestStatus.CREATED, restStatus(createResponse)); + Map responseBody = asMap(createResponse); + String createdId1 = responseBody.get("_id").toString(); + + String rule2 = randomRuleForCorrelations("DeleteUser"); + createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.RULE_BASE_URI, Collections.singletonMap("category", "cloudtrail"), + new StringEntity(rule2), new BasicHeader("Content-Type", "application/json")); + Assert.assertEquals("Create rule failed", RestStatus.CREATED, restStatus(createResponse)); + responseBody = asMap(createResponse); + String createdId2 = responseBody.get("_id").toString(); + + createCloudtrailFieldBasedRule(index, "requestParameters.userName", null); + + Detector cloudtrailDetector = randomDetectorWithInputsAndTriggersAndType(List.of(new DetectorInput("cloudtrail detector for security analytics", List.of(index), + List.of(new DetectorRule(createdId1), new DetectorRule(createdId2)), + List.of())), + List.of(new DetectorTrigger(null, "test-trigger", "1", List.of("cloudtrail"), List.of(), List.of(), List.of(), List.of(), List.of())), "cloudtrail"); + + createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(cloudtrailDetector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + responseBody = asMap(createResponse); + + String createdId = responseBody.get("_id").toString(); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + createdId + "\"\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + + String monitorId = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); + + indexDoc(index, "1", randomCloudtrailDoc("Richard", "CreateUser")); + executeAlertingMonitor(monitorId, Collections.emptyMap()); + Thread.sleep(1000); + indexDoc(index, "4", randomCloudtrailDoc("deysubho", "CreateUser")); + executeAlertingMonitor(monitorId, Collections.emptyMap()); + Thread.sleep(1000); + + indexDoc(index, "2", randomCloudtrailDoc("Richard", "DeleteUser")); + executeAlertingMonitor(monitorId, Collections.emptyMap()); + + // Call GetFindings API + Map params = new HashMap<>(); + params.put("detectorType", "cloudtrail"); + Response getFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.FINDINGS_BASE_URI + "/_search", params, null); + Map getFindingsBody = entityAsMap(getFindingsResponse); + + Thread.sleep(5000); + + int count = 0; + while (true) { + try { + Long endTime = System.currentTimeMillis(); + Request restRequest = new Request("GET", "/_plugins/_security_analytics/correlations?start_timestamp=" + startTime + "&end_timestamp=" + endTime); + response = client().performRequest(restRequest); + + Map responseMap = entityAsMap(response); + List results = (List) responseMap.get("findings"); + if (results.size() == 1) { + Assert.assertTrue(true); + break; + } + } catch (Exception ex) { + // suppress ex + } + ++count; + Thread.sleep(5000); + if (count >= 12) { + Assert.assertTrue(false); + break; + } + } + } + + public void testBasicCorrelationEngineWorkflowWithFieldBasedRulesOnMultipleLogTypes() throws IOException, InterruptedException { + updateClusterSetting(SecurityAnalyticsSettings.ENABLE_AUTO_CORRELATIONS.getKey(), "false"); + + LogIndices indices = new LogIndices(); + indices.windowsIndex = createTestIndex(randomIndex(), windowsIndexMapping()); + indices.vpcFlowsIndex = createTestIndex("vpc_flow", vpcFlowMappings()); + + String vpcFlowMonitorId = createVpcFlowDetector(indices.vpcFlowsIndex); + String testWindowsMonitorId = createTestWindowsDetector(indices.windowsIndex); + + String ruleId = createNetworkToWindowsFieldBasedRule(indices); + + indexDoc(indices.windowsIndex, "2", randomDoc()); + Response executeResponse = executeAlertingMonitor(testWindowsMonitorId, Collections.emptyMap()); + Map executeResults = entityAsMap(executeResponse); + int noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + Assert.assertEquals(5, noOfSigmaRuleMatches); + + indexDoc(indices.vpcFlowsIndex, "1", randomVpcFlowDoc()); + executeResponse = executeAlertingMonitor(vpcFlowMonitorId, Collections.emptyMap()); + executeResults = entityAsMap(executeResponse); + noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + Assert.assertEquals(1, noOfSigmaRuleMatches); + Thread.sleep(5000); + + // Call GetFindings API + Map params = new HashMap<>(); + params.put("detectorType", "test_windows"); + Response getFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.FINDINGS_BASE_URI + "/_search", params, null); + Map getFindingsBody = entityAsMap(getFindingsResponse); + String finding = ((List>) getFindingsBody.get("findings")).get(0).get("id").toString(); + + int count = 0; + while (true) { + try { + List> correlatedFindings = searchCorrelatedFindings(finding, "test_windows", 300000L, 10); + if (correlatedFindings.size() == 1) { + Assert.assertTrue(true); + + Assert.assertTrue(correlatedFindings.get(0).get("rules") instanceof List); + + for (var correlatedFinding: correlatedFindings) { + if (correlatedFinding.get("detector_type").equals("network")) { + Assert.assertEquals(1, ((List) correlatedFinding.get("rules")).size()); + Assert.assertTrue(((List) correlatedFinding.get("rules")).contains(ruleId)); + } + } + break; + } + } catch (Exception ex) { + // suppress ex + } + ++count; + Thread.sleep(5000); + if (count >= 12) { + Assert.assertTrue(false); + break; + } + } + } + + public void testBasicCorrelationEngineWorkflowWithFieldBasedRulesAndDynamicTimeWindow() throws IOException, InterruptedException { + Long startTime = System.currentTimeMillis(); + String index = createTestIndex("cloudtrail", cloudtrailMappings()); + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{\n" + + " \"index_name\": \"" + index + "\",\n" + + " \"rule_topic\": \"cloudtrail\",\n" + + " \"partial\": true,\n" + + " \"alias_mappings\": {\n" + + " \"properties\": {\n" + + " \"aws.cloudtrail.event_name\": {\n" + + " \"path\": \"Records.eventName\",\n" + + " \"type\": \"alias\"\n" + + " }\n" + + " }\n" + + " }\n" + + "}" + ); + + Response response = client().performRequest(createMappingRequest); + assertEquals(RestStatus.OK.getStatus(), response.getStatusLine().getStatusCode()); + + String rule1 = randomRuleForCorrelations("CreateUser"); + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.RULE_BASE_URI, Collections.singletonMap("category", "cloudtrail"), + new StringEntity(rule1), new BasicHeader("Content-Type", "application/json")); + Assert.assertEquals("Create rule failed", RestStatus.CREATED, restStatus(createResponse)); + Map responseBody = asMap(createResponse); + String createdId1 = responseBody.get("_id").toString(); + + String rule2 = randomRuleForCorrelations("DeleteUser"); + createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.RULE_BASE_URI, Collections.singletonMap("category", "cloudtrail"), + new StringEntity(rule2), new BasicHeader("Content-Type", "application/json")); + Assert.assertEquals("Create rule failed", RestStatus.CREATED, restStatus(createResponse)); + responseBody = asMap(createResponse); + String createdId2 = responseBody.get("_id").toString(); + + createCloudtrailFieldBasedRule(index, "requestParameters.userName", 5000L); + + Detector cloudtrailDetector = randomDetectorWithInputsAndTriggersAndType(List.of(new DetectorInput("cloudtrail detector for security analytics", List.of(index), + List.of(new DetectorRule(createdId1), new DetectorRule(createdId2)), + List.of())), + List.of(new DetectorTrigger(null, "test-trigger", "1", List.of("cloudtrail"), List.of(), List.of(), List.of(), List.of(), List.of())), "cloudtrail"); + + createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(cloudtrailDetector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + responseBody = asMap(createResponse); + + String createdId = responseBody.get("_id").toString(); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + createdId + "\"\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + + String monitorId = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); + + indexDoc(index, "1", randomCloudtrailDoc("Richard", "CreateUser")); + executeAlertingMonitor(monitorId, Collections.emptyMap()); + Thread.sleep(30000); + indexDoc(index, "4", randomCloudtrailDoc("deysubho", "CreateUser")); + executeAlertingMonitor(monitorId, Collections.emptyMap()); + Thread.sleep(1000); + + indexDoc(index, "2", randomCloudtrailDoc("Richard", "DeleteUser")); + executeAlertingMonitor(monitorId, Collections.emptyMap()); + + // Call GetFindings API + Map params = new HashMap<>(); + params.put("detectorType", "cloudtrail"); + Response getFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.FINDINGS_BASE_URI + "/_search", params, null); + Map getFindingsBody = entityAsMap(getFindingsResponse); + + Thread.sleep(5000); + + int count = 0; + while (true) { + try { + Long endTime = System.currentTimeMillis(); + Request restRequest = new Request("GET", "/_plugins/_security_analytics/correlations?start_timestamp=" + startTime + "&end_timestamp=" + endTime); + response = client().performRequest(restRequest); + + Map responseMap = entityAsMap(response); + List results = (List) responseMap.get("findings"); + if (results.size() == 1) { + Assert.assertTrue(true); + break; + } + } catch (Exception ex) { + // suppress ex + } + ++count; + Thread.sleep(5000); + if (count >= 2) { + break; + } + } + Assert.assertEquals(2, count); + } + private LogIndices createIndices() throws IOException { LogIndices indices = new LogIndices(); indices.adLdapLogsIndex = createTestIndex("ad_logs", adLdapLogMappings()); @@ -530,12 +815,25 @@ private LogIndices createIndices() throws IOException { return indices; } + private String createNetworkToWindowsFieldBasedRule(LogIndices indices) throws IOException { + CorrelationQuery query1 = new CorrelationQuery(indices.vpcFlowsIndex, null, "network", "srcaddr"); + CorrelationQuery query4 = new CorrelationQuery(indices.windowsIndex, null, "test_windows", "SourceIp"); + + CorrelationRule rule = new CorrelationRule(CorrelationRule.NO_ID, CorrelationRule.NO_VERSION, "network to windows", List.of(query1, query4), 300000L); + Request request = new Request("POST", "/_plugins/_security_analytics/correlation/rules"); + request.setJsonEntity(toJsonString(rule)); + Response response = client().performRequest(request); + + Assert.assertEquals(201, response.getStatusLine().getStatusCode()); + return entityAsMap(response).get("_id").toString(); + } + private String createNetworkToAdLdapToWindowsRule(LogIndices indices) throws IOException { - CorrelationQuery query1 = new CorrelationQuery(indices.vpcFlowsIndex, "dstaddr:4.5.6.7", "network"); - CorrelationQuery query2 = new CorrelationQuery(indices.adLdapLogsIndex, "ResultType:50126", "ad_ldap"); - CorrelationQuery query4 = new CorrelationQuery(indices.windowsIndex, "Domain:NTAUTHORI*", "test_windows"); + CorrelationQuery query1 = new CorrelationQuery(indices.vpcFlowsIndex, "dstaddr:4.5.6.7", "network", null); + CorrelationQuery query2 = new CorrelationQuery(indices.adLdapLogsIndex, "ResultType:50126", "ad_ldap", null); + CorrelationQuery query4 = new CorrelationQuery(indices.windowsIndex, "Domain:NTAUTHORI*", "test_windows", null); - CorrelationRule rule = new CorrelationRule(CorrelationRule.NO_ID, CorrelationRule.NO_VERSION, "network to ad_ldap to windows", List.of(query1, query2, query4)); + CorrelationRule rule = new CorrelationRule(CorrelationRule.NO_ID, CorrelationRule.NO_VERSION, "network to ad_ldap to windows", List.of(query1, query2, query4), 300000L); Request request = new Request("POST", "/_plugins/_security_analytics/correlation/rules"); request.setJsonEntity(toJsonString(rule)); Response response = client().performRequest(request); @@ -545,11 +843,24 @@ private String createNetworkToAdLdapToWindowsRule(LogIndices indices) throws IOE } private String createWindowsToAppLogsToS3LogsRule(LogIndices indices) throws IOException { - CorrelationQuery query1 = new CorrelationQuery(indices.windowsIndex, "HostName:EC2AMAZ*", "test_windows"); - CorrelationQuery query2 = new CorrelationQuery(indices.appLogsIndex, "endpoint:\\/customer_records.txt", "others_application"); - CorrelationQuery query4 = new CorrelationQuery(indices.s3AccessLogsIndex, "aws.cloudtrail.eventName:ReplicateObject", "s3"); + CorrelationQuery query1 = new CorrelationQuery(indices.windowsIndex, "HostName:EC2AMAZ*", "test_windows", null); + CorrelationQuery query2 = new CorrelationQuery(indices.appLogsIndex, "endpoint:\\/customer_records.txt", "others_application", null); + CorrelationQuery query4 = new CorrelationQuery(indices.s3AccessLogsIndex, "aws.cloudtrail.eventName:ReplicateObject", "s3", null); + + CorrelationRule rule = new CorrelationRule(CorrelationRule.NO_ID, CorrelationRule.NO_VERSION, "windows to app_logs to s3 logs", List.of(query1, query2, query4), 300000L); + Request request = new Request("POST", "/_plugins/_security_analytics/correlation/rules"); + request.setJsonEntity(toJsonString(rule)); + Response response = client().performRequest(request); + + Assert.assertEquals(201, response.getStatusLine().getStatusCode()); + return entityAsMap(response).get("_id").toString(); + } + + private String createCloudtrailFieldBasedRule(String index, String field, Long timeWindow) throws IOException { + CorrelationQuery query1 = new CorrelationQuery(index, "EventName:CreateUser", "cloudtrail", field); + CorrelationQuery query2 = new CorrelationQuery(index, "EventName:DeleteUser", "cloudtrail", field); - CorrelationRule rule = new CorrelationRule(CorrelationRule.NO_ID, CorrelationRule.NO_VERSION, "windows to app_logs to s3 logs", List.of(query1, query2, query4)); + CorrelationRule rule = new CorrelationRule(CorrelationRule.NO_ID, CorrelationRule.NO_VERSION, "cloudtrail field based", List.of(query1, query2), timeWindow); Request request = new Request("POST", "/_plugins/_security_analytics/correlation/rules"); request.setJsonEntity(toJsonString(rule)); Response response = client().performRequest(request); diff --git a/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java b/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java index a65485670..3b7ca3c0a 100644 --- a/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java +++ b/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java @@ -264,7 +264,7 @@ public void testGetFindings_byDetectorType_success() throws IOException { getFindingsBody = entityAsMap(getFindingsResponse); Assert.assertEquals(1, getFindingsBody.get("total_findings")); } - @Ignore + public void testGetFindings_rolloverByMaxAge_success() throws IOException, InterruptedException { updateClusterSetting(FINDING_HISTORY_ROLLOVER_PERIOD.getKey(), "1s"); diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java index 95a9c3707..e60a9cf03 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java @@ -5,6 +5,8 @@ package org.opensearch.securityanalytics.resthandler; import org.apache.hc.core5.http.HttpStatus; +import org.apache.hc.core5.http.io.entity.StringEntity; +import org.apache.hc.core5.http.message.BasicHeader; import org.junit.Assert; import org.opensearch.action.search.SearchResponse; import org.opensearch.client.Request; @@ -28,12 +30,15 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; import static java.util.Collections.emptyList; import static org.opensearch.securityanalytics.TestHelpers.randomAggregationRule; +import static org.opensearch.securityanalytics.TestHelpers.randomCloudtrailAggrRule; +import static org.opensearch.securityanalytics.TestHelpers.randomCloudtrailDoc; import static org.opensearch.securityanalytics.TestHelpers.randomDetector; import static org.opensearch.securityanalytics.TestHelpers.randomDetectorType; import static org.opensearch.securityanalytics.TestHelpers.randomDetectorWithInputs; @@ -1902,6 +1907,81 @@ public void testCreateDetectorWithKeywordsRule_ensureNoFindingsWithoutDateMappin assertEquals(0, noOfSigmaRuleMatches); } + @SuppressWarnings("unchecked") + public void testCreateDetectorWithCloudtrailAggrRule() throws IOException { + String index = createTestIndex(randomIndex(), windowsIndexMapping()); + indexDoc(index, "0", randomCloudtrailDoc("A12346", "CREATED")); + + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"" + randomDetectorType() + "\", " + + " \"partial\":true" + + "}" + ); + + Response response = client().performRequest(createMappingRequest); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + + String rule = randomCloudtrailAggrRule(); + + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.RULE_BASE_URI, Collections.singletonMap("category", randomDetectorType()), + new StringEntity(rule), new BasicHeader("Content-Type", "application/json")); + Assert.assertEquals("Create rule failed", RestStatus.CREATED, restStatus(createResponse)); + Map responseBody = asMap(createResponse); + String createdId = responseBody.get("_id").toString(); + + DetectorInput input = new DetectorInput("windows detector for security analytics", List.of(index), List.of(new DetectorRule(createdId)), + List.of()); + Detector detector = randomDetectorWithInputsAndTriggers(List.of(input), + List.of(new DetectorTrigger(null, "test-trigger", "1", List.of(), List.of(createdId), List.of(), List.of(), List.of(), List.of()))); + + createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + responseBody = asMap(createResponse); + + createdId = responseBody.get("_id").toString(); + int createdVersion = Integer.parseInt(responseBody.get("_version").toString()); + Assert.assertNotEquals("response is missing Id", Detector.NO_ID, createdId); + Assert.assertTrue("incorrect version", createdVersion > 0); + Assert.assertEquals("Incorrect Location header", String.format(Locale.getDefault(), "%s/%s", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, createdId), createResponse.getHeader("Location")); + Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("rule_topic_index")); + Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("findings_index")); + Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("alert_index")); + + String detectorTypeInResponse = (String) ((Map)responseBody.get("detector")).get("detector_type"); + Assert.assertEquals("Detector type incorrect", randomDetectorType().toLowerCase(Locale.ROOT), detectorTypeInResponse); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + createdId + "\"\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + + String workflowId = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("workflow_ids")).get(0); + + indexDoc(index, "1", randomCloudtrailDoc("A12345", "CREATED")); + executeAlertingWorkflow(workflowId, Collections.emptyMap()); + indexDoc(index, "2", randomCloudtrailDoc("A12345", "DELETED")); + executeAlertingWorkflow(workflowId, Collections.emptyMap()); + + Map params = new HashMap<>(); + params.put("detector_id", createdId); + Response getFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.FINDINGS_BASE_URI + "/_search", params, null); + Map getFindingsBody = entityAsMap(getFindingsResponse); + + // Assert findings + assertNotNull(getFindingsBody); + assertEquals(1, getFindingsBody.get("total_findings")); + } + private static void assertRuleMonitorFinding(Map executeResults, String ruleId, int expectedDocCount, List expectedTriggerResult) { List> buckets = ((List>) (((Map) ((Map) ((Map) ((List) ((Map) executeResults.get("input_results")).get("results")).get(0)).get("aggregations")).get("result_agg")).get("buckets"))); Integer docCount = buckets.stream().mapToInt(it -> (Integer) it.get("doc_count")).sum(); diff --git a/src/test/java/org/opensearch/securityanalytics/rules/aggregation/AggregationBackendTests.java b/src/test/java/org/opensearch/securityanalytics/rules/aggregation/AggregationBackendTests.java index 395f15a79..9103d03ab 100644 --- a/src/test/java/org/opensearch/securityanalytics/rules/aggregation/AggregationBackendTests.java +++ b/src/test/java/org/opensearch/securityanalytics/rules/aggregation/AggregationBackendTests.java @@ -216,4 +216,39 @@ public void testAvgAggregationWithGroupBy() throws IOException, SigmaError { Assert.assertEquals("{\"result_agg\":{\"terms\":{\"field\":\"fieldB\"},\"aggs\":{\"fieldA\":{\"avg\":{\"field\":\"fieldA\"}}}}}", aggQuery); Assert.assertEquals("{\"buckets_path\":{\"fieldA\":\"fieldA\"},\"parent_bucket_path\":\"result_agg\",\"script\":{\"source\":\"params.fieldA > 110.0\",\"lang\":\"painless\"}}", bucketTriggerQuery); } + + public void testCloudtrailAggregationRule() throws IOException, SigmaError { + OSQueryBackend queryBackend = new OSQueryBackend(Map.of(), true, true); + List queries = queryBackend.convertRule(SigmaRule.fromYaml( + "id: c64c5175-5189-431b-a55e-6d9882158250\n" + + "logsource:\n" + + " product: cloudtrail\n" + + "title: Accounts created and deleted within 24h\n" + + "description: Flag suspicious activity of accounts created and deleted within 24h\n" + + "date: 2021/09/23\n" + + "tags:\n" + + " - attack.exfiltration\n" + + "falsepositives: [ ]\n" + + "level: high\n" + + "status: test\n" + + "references: [ ]\n" + + "author: Sashank\n" + + "detection:\n" + + " selection:\n" + + " event:\n" + + " - CREATED\n" + + " - DELETED\n" + + " timeframe: 24h\n" + + " condition: selection | count(*) by accountid > 2", true)); + + String query = queries.get(0).toString(); + Assert.assertEquals("(event: \"CREATED\") OR (event: \"DELETED\")", query); + + OSQueryBackend.AggregationQueries aggQueries = (OSQueryBackend.AggregationQueries) queries.get(1); + String aggQuery = aggQueries.getAggQuery(); + String bucketTriggerQuery = aggQueries.getBucketTriggerQuery(); + + Assert.assertEquals("{\"result_agg\":{\"terms\":{\"field\":\"accountid\"}}}", aggQuery); + Assert.assertEquals("{\"buckets_path\":{\"_cnt\":\"_count\"},\"parent_bucket_path\":\"result_agg\",\"script\":{\"source\":\"params._cnt > 2.0\",\"lang\":\"painless\"}}", bucketTriggerQuery); + } } \ No newline at end of file