Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Backport 2.15] added correlationAlert integ tests (#1099) #1141

Merged
merged 1 commit into from
Jul 7, 2024
Merged

[Backport 2.15] added correlationAlert integ tests (#1099) #1141

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
360 changes: 350 additions & 10 deletions src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java
View file
Open in desktop

Large diffs are not rendered by default.

12 changes: 12 additions & 0 deletions src/test/java/org/opensearch/securityanalytics/TestHelpers.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import org.opensearch.script.ScriptType;
import org.opensearch.securityanalytics.model.CorrelationQuery;
import org.opensearch.securityanalytics.model.CorrelationRule;
import org.opensearch.securityanalytics.model.CorrelationRuleTrigger;
import org.opensearch.securityanalytics.model.CustomLogType;
import org.opensearch.securityanalytics.model.Detector;
import org.opensearch.securityanalytics.model.DetectorInput;
Expand Down Expand Up @@ -230,6 +231,17 @@ public static CorrelationRule randomCorrelationRule(String name) {
), 300000L, null);
}

public static CorrelationRule randomCorrelationRuleWithTrigger(String name) {
name = name.isEmpty()? "><script>prompt(document.domain)</script>": name;
List<Action> actions = new ArrayList<Action>();
CorrelationRuleTrigger trigger = new CorrelationRuleTrigger("trigger-123", "Trigger 1", "high", actions);
return new CorrelationRule(CorrelationRule.NO_ID, CorrelationRule.NO_VERSION, name,
List.of(
new CorrelationQuery("vpc_flow1", "dstaddr:192.168.1.*", "network", null),
new CorrelationQuery("ad_logs1", "azure.platformlogs.result_type:50126", "ad_ldap", null)
), 300000L, trigger);
}

public static String randomRule() {
return "title: Remote Encrypting File System Abuse\n" +
"id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" +
Expand Down
303 changes: 0 additions & 303 deletions src/test/java/org/opensearch/securityanalytics/correlation/CorrelationEngineRestApiIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,6 @@
import org.opensearch.securityanalytics.SecurityAnalyticsPlugin;
import org.opensearch.securityanalytics.SecurityAnalyticsRestTestCase;
import org.opensearch.securityanalytics.TestHelpers;
import org.opensearch.securityanalytics.model.CorrelationQuery;
import org.opensearch.securityanalytics.model.CorrelationRule;
import org.opensearch.securityanalytics.model.CustomLogType;
import org.opensearch.securityanalytics.model.Detector;
import org.opensearch.securityanalytics.model.DetectorInput;
Expand All @@ -32,7 +30,6 @@
import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.function.BooleanSupplier;
import java.util.stream.Collectors;

import static org.opensearch.securityanalytics.TestHelpers.*;

Expand Down Expand Up @@ -954,304 +951,4 @@ public void testBasicCorrelationEngineWorkflowWithCustomLogTypes() throws IOExce
);
}

private LogIndices createIndices() throws IOException {
LogIndices indices = new LogIndices();
indices.adLdapLogsIndex = createTestIndex("ad_logs", adLdapLogMappings());
indices.s3AccessLogsIndex = createTestIndex("s3_access_logs", s3AccessLogMappings());
indices.appLogsIndex = createTestIndex("app_logs", appLogMappings());
indices.windowsIndex = createTestIndex(randomIndex(), windowsIndexMapping());
indices.vpcFlowsIndex = createTestIndex("vpc_flow", vpcFlowMappings());
return indices;
}

private String createNetworkToWindowsFieldBasedRule(LogIndices indices) throws IOException {
CorrelationQuery query1 = new CorrelationQuery(indices.vpcFlowsIndex, null, "network", "srcaddr");
CorrelationQuery query4 = new CorrelationQuery(indices.windowsIndex, null, "test_windows", "SourceIp");

CorrelationRule rule = new CorrelationRule(CorrelationRule.NO_ID, CorrelationRule.NO_VERSION, "network to windows", List.of(query1, query4), 300000L, null);
Request request = new Request("POST", "/_plugins/_security_analytics/correlation/rules");
request.setJsonEntity(toJsonString(rule));
Response response = client().performRequest(request);

Assert.assertEquals(201, response.getStatusLine().getStatusCode());
return entityAsMap(response).get("_id").toString();
}

private String createNetworkToWindowsFilterQueryBasedRule(LogIndices indices) throws IOException {
CorrelationQuery query1 = new CorrelationQuery(indices.vpcFlowsIndex, "srcaddr:1.2.3.4", "network", null);
CorrelationQuery query4 = new CorrelationQuery(indices.windowsIndex, "SourceIp:1.2.3.4", "test_windows", null);

CorrelationRule rule = new CorrelationRule(CorrelationRule.NO_ID, CorrelationRule.NO_VERSION, "network to windows", List.of(query1, query4), 300000L, null);
Request request = new Request("POST", "/_plugins/_security_analytics/correlation/rules");
request.setJsonEntity(toJsonString(rule));
Response response = client().performRequest(request);

Assert.assertEquals(201, response.getStatusLine().getStatusCode());
return entityAsMap(response).get("_id").toString();
}

private String createNetworkToCustomLogTypeFieldBasedRule(LogIndices indices, String customLogTypeName, String customLogTypeIndex) throws IOException {
CorrelationQuery query1 = new CorrelationQuery(indices.vpcFlowsIndex, null, "network", "srcaddr");
CorrelationQuery query4 = new CorrelationQuery(customLogTypeIndex, null, customLogTypeName, "SourceIp");

CorrelationRule rule = new CorrelationRule(CorrelationRule.NO_ID, CorrelationRule.NO_VERSION, "network to custom log type", List.of(query1, query4), 300000L, null);
Request request = new Request("POST", "/_plugins/_security_analytics/correlation/rules");
request.setJsonEntity(toJsonString(rule));
Response response = client().performRequest(request);

Assert.assertEquals(201, response.getStatusLine().getStatusCode());
return entityAsMap(response).get("_id").toString();
}

private String createNetworkToAdLdapToWindowsRule(LogIndices indices) throws IOException {
CorrelationQuery query1 = new CorrelationQuery(indices.vpcFlowsIndex, "dstaddr:4.5.6.7", "network", null);
CorrelationQuery query2 = new CorrelationQuery(indices.adLdapLogsIndex, "ResultType:50126", "ad_ldap", null);
CorrelationQuery query4 = new CorrelationQuery(indices.windowsIndex, "Domain:NTAUTHORI*", "test_windows", null);

CorrelationRule rule = new CorrelationRule(CorrelationRule.NO_ID, CorrelationRule.NO_VERSION, "network to ad_ldap to windows", List.of(query1, query2, query4), 300000L, null);
Request request = new Request("POST", "/_plugins/_security_analytics/correlation/rules");
request.setJsonEntity(toJsonString(rule));
Response response = client().performRequest(request);

Assert.assertEquals(201, response.getStatusLine().getStatusCode());
return entityAsMap(response).get("_id").toString();
}

private String createWindowsToAppLogsToS3LogsRule(LogIndices indices) throws IOException {
CorrelationQuery query1 = new CorrelationQuery(indices.windowsIndex, "HostName:EC2AMAZ*", "test_windows", null);
CorrelationQuery query2 = new CorrelationQuery(indices.appLogsIndex, "endpoint:\\/customer_records.txt", "others_application", null);
CorrelationQuery query4 = new CorrelationQuery(indices.s3AccessLogsIndex, "aws.cloudtrail.eventName:ReplicateObject", "s3", null);

CorrelationRule rule = new CorrelationRule(CorrelationRule.NO_ID, CorrelationRule.NO_VERSION, "windows to app_logs to s3 logs", List.of(query1, query2, query4), 300000L, null);
Request request = new Request("POST", "/_plugins/_security_analytics/correlation/rules");
request.setJsonEntity(toJsonString(rule));
Response response = client().performRequest(request);

Assert.assertEquals(201, response.getStatusLine().getStatusCode());
return entityAsMap(response).get("_id").toString();
}

private String createCloudtrailFieldBasedRule(String index, String field, Long timeWindow) throws IOException {
CorrelationQuery query1 = new CorrelationQuery(index, "EventName:CreateUser", "cloudtrail", field);
CorrelationQuery query2 = new CorrelationQuery(index, "EventName:DeleteUser", "cloudtrail", field);

CorrelationRule rule = new CorrelationRule(CorrelationRule.NO_ID, CorrelationRule.NO_VERSION, "cloudtrail field based", List.of(query1, query2), timeWindow, null);
Request request = new Request("POST", "/_plugins/_security_analytics/correlation/rules");
request.setJsonEntity(toJsonString(rule));
Response response = client().performRequest(request);

Assert.assertEquals(201, response.getStatusLine().getStatusCode());
return entityAsMap(response).get("_id").toString();
}

@SuppressWarnings("unchecked")
private String createVpcFlowDetector(String indexName) throws IOException {
Detector vpcFlowDetector = randomDetectorWithInputsAndTriggersAndType(List.of(new DetectorInput("vpc flow detector for security analytics", List.of(indexName), List.of(),
getPrePackagedRules("network").stream().map(DetectorRule::new).collect(Collectors.toList()))),
List.of(new DetectorTrigger(null, "test-trigger", "1", List.of("network"), List.of(), List.of(), List.of(), List.of(), List.of())), "network");

Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(vpcFlowDetector));
Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse));

Map<String, Object> responseBody = asMap(createResponse);

String createdId = responseBody.get("_id").toString();

String request = "{\n" +
" \"query\" : {\n" +
" \"match\":{\n" +
" \"_id\": \"" + createdId + "\"\n" +
" }\n" +
" }\n" +
"}";
List<SearchHit> hits = executeSearch(Detector.DETECTORS_INDEX, request);
SearchHit hit = hits.get(0);

return ((List<String>) ((Map<String, Object>) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0);
}

@SuppressWarnings("unchecked")
private String createAdLdapDetector(String indexName) throws IOException {
// Execute CreateMappingsAction to add alias mapping for index
Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI);
// both req params and req body are supported
createMappingRequest.setJsonEntity(
"{\n" +
" \"index_name\": \"" + indexName + "\",\n" +
" \"rule_topic\": \"ad_ldap\",\n" +
" \"partial\": true,\n" +
" \"alias_mappings\": {\n" +
" \"properties\": {\n" +
" \"azure.signinlogs.properties.user_id\": {\n" +
" \"path\": \"azure.signinlogs.props.user_id\",\n" +
" \"type\": \"alias\"\n" +
" },\n" +
" \"azure-platformlogs-result_type\": {\n" +
" \"path\": \"azure.platformlogs.result_type\",\n" +
" \"type\": \"alias\"\n" +
" },\n" +
" \"azure-signinlogs-result_description\": {\n" +
" \"path\": \"azure.signinlogs.result_description\",\n" +
" \"type\": \"alias\"\n" +
" },\n" +
" \"timestamp\": {\n" +
" \"path\": \"creationTime\",\n" +
" \"type\": \"alias\"\n" +
" }\n" +
" }\n" +
" }\n" +
"}"
);

Response response = client().performRequest(createMappingRequest);
assertEquals(RestStatus.OK.getStatus(), response.getStatusLine().getStatusCode());

Detector adLdapDetector = randomDetectorWithInputsAndTriggersAndType(List.of(new DetectorInput("ad_ldap logs detector for security analytics", List.of(indexName), List.of(),
getPrePackagedRules("ad_ldap").stream().map(DetectorRule::new).collect(Collectors.toList()))),
List.of(new DetectorTrigger(null, "test-trigger", "1", List.of("ad_ldap"), List.of(), List.of(), List.of(), List.of(), List.of())), "ad_ldap");

Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(adLdapDetector));
Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse));

Map<String, Object> responseBody = asMap(createResponse);

String createdId = responseBody.get("_id").toString();

String request = "{\n" +
" \"query\" : {\n" +
" \"match\":{\n" +
" \"_id\": \"" + createdId + "\"\n" +
" }\n" +
" }\n" +
"}";
List<SearchHit> hits = executeSearch(Detector.DETECTORS_INDEX, request);
SearchHit hit = hits.get(0);

return ((List<String>) ((Map<String, Object>) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0);
}

@SuppressWarnings("unchecked")
private String createTestWindowsDetector(String indexName) throws IOException {
// Execute CreateMappingsAction to add alias mapping for index
Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI);
// both req params and req body are supported
createMappingRequest.setJsonEntity(
"{ \"index_name\":\"" + indexName + "\"," +
" \"rule_topic\":\"" + randomDetectorType() + "\", " +
" \"partial\":true" +
"}"
);

Response response = client().performRequest(createMappingRequest);
assertEquals(RestStatus.OK.getStatus(), response.getStatusLine().getStatusCode());

Detector windowsDetector = randomDetectorWithInputsAndTriggers(List.of(new DetectorInput("windows detector for security analytics", List.of(indexName), List.of(),
getRandomPrePackagedRules().stream().map(DetectorRule::new).collect(Collectors.toList()))),
List.of(new DetectorTrigger(null, "test-trigger", "1", List.of(randomDetectorType()), List.of(), List.of(), List.of(), List.of(), List.of())));

Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(windowsDetector));
Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse));

Map<String, Object> responseBody = asMap(createResponse);

String createdId = responseBody.get("_id").toString();

String request = "{\n" +
" \"query\" : {\n" +
" \"match\":{\n" +
" \"_id\": \"" + createdId + "\"\n" +
" }\n" +
" }\n" +
"}";
List<SearchHit> hits = executeSearch(Detector.DETECTORS_INDEX, request);
SearchHit hit = hits.get(0);

return ((List<String>) ((Map<String, Object>) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0);
}

@SuppressWarnings("unchecked")
private String createAppLogsDetector(String indexName) throws IOException {
Detector appLogsDetector = randomDetectorWithInputsAndTriggersAndType(List.of(new DetectorInput("app logs detector for security analytics", List.of(indexName), List.of(),
getPrePackagedRules("others_application").stream().map(DetectorRule::new).collect(Collectors.toList()))),
List.of(new DetectorTrigger(null, "test-trigger", "1", List.of("others_application"), List.of(), List.of(), List.of(), List.of(), List.of())), "others_application");

Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(appLogsDetector));
Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse));

Map<String, Object> responseBody = asMap(createResponse);

String createdId = responseBody.get("_id").toString();

String request = "{\n" +
" \"query\" : {\n" +
" \"match\":{\n" +
" \"_id\": \"" + createdId + "\"\n" +
" }\n" +
" }\n" +
"}";
List<SearchHit> hits = executeSearch(Detector.DETECTORS_INDEX, request);
SearchHit hit = hits.get(0);

return ((List<String>) ((Map<String, Object>) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0);
}

@SuppressWarnings("unchecked")
private String createS3Detector(String indexName) throws IOException {
// Execute CreateMappingsAction to add alias mapping for index
Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI);
// both req params and req body are supported
createMappingRequest.setJsonEntity(
"{\n" +
" \"index_name\": \"s3_access_logs\",\n" +
" \"rule_topic\": \"s3\",\n" +
" \"partial\": true,\n" +
" \"alias_mappings\": {\n" +
" \"properties\": {\n" +
" \"aws-cloudtrail-event_source\": {\n" +
" \"type\": \"alias\",\n" +
" \"path\": \"aws.cloudtrail.event_source\"\n" +
" },\n" +
" \"aws.cloudtrail.event_name\": {\n" +
" \"type\": \"alias\",\n" +
" \"path\": \"aws.cloudtrail.event_name\"\n" +
" }\n" +
" }\n" +
" }\n" +
"}"
);

Response response = client().performRequest(createMappingRequest);
assertEquals(RestStatus.OK.getStatus(), response.getStatusLine().getStatusCode());

Detector s3AccessLogsDetector = randomDetectorWithInputsAndTriggersAndType(List.of(new DetectorInput("s3 access logs detector for security analytics", List.of(indexName), List.of(),
getPrePackagedRules("s3").stream().map(DetectorRule::new).collect(Collectors.toList()))),
List.of(new DetectorTrigger(null, "test-trigger", "1", List.of("s3"), List.of(), List.of(), List.of(), List.of(), List.of())), "s3");

Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(s3AccessLogsDetector));
Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse));

Map<String, Object> responseBody = asMap(createResponse);

String createdId = responseBody.get("_id").toString();

String request = "{\n" +
" \"query\" : {\n" +
" \"match\":{\n" +
" \"_id\": \"" + createdId + "\"\n" +
" }\n" +
" }\n" +
"}";
List<SearchHit> hits = executeSearch(Detector.DETECTORS_INDEX, request);
SearchHit hit = hits.get(0);

return ((List<String>) ((Map<String, Object>) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0);
}

static class LogIndices {
String vpcFlowsIndex;
String adLdapLogsIndex;
String windowsIndex;
String appLogsIndex;
String s3AccessLogsIndex;
}
}
Loading
Loading