Skip to content

Commit

Permalink
Fix get mappings view API incorrectly returning ecs path (opensearch-…
Browse files Browse the repository at this point in the history 
…project#867)

* add logic and integ tests to not add duplicate to log-types-config index

Signed-off-by: Joanne Wang <[email protected]>

* change naming for existingFieldMapping and change contains to equals

Signed-off-by: Joanne Wang <[email protected]>

---------

Signed-off-by: Joanne Wang <[email protected]>
  • Loading branch information
@jowg-amazon
jowg-amazon committed Mar 14, 2024
1 parent 1b948ac commit 840a8bf
Show file tree
Hide file tree
Showing 3 changed files with 148 additions and 23 deletions.
23 changes: 17 additions & 6 deletions src/main/java/org/opensearch/securityanalytics/logtype/LogTypeService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -348,15 +348,26 @@ private List<FieldMappingDoc> mergeFieldMappings(List<FieldMappingDoc> existingF
List<FieldMappingDoc> newFieldMappings = new ArrayList<>();
fieldMappingDocs.forEach( newFieldMapping -> {
Optional<FieldMappingDoc> foundFieldMappingDoc = Optional.empty();
for (FieldMappingDoc e: existingFieldMappings) {
if (e.getRawField().equals(newFieldMapping.getRawField())) {
for (FieldMappingDoc existingFieldMapping: existingFieldMappings) {
if (existingFieldMapping.getRawField().equals(newFieldMapping.getRawField())) {
if ((
e.get(defaultSchemaField) != null && newFieldMapping.get(defaultSchemaField) != null &&
e.get(defaultSchemaField).equals(newFieldMapping.get(defaultSchemaField))
existingFieldMapping.get(defaultSchemaField) != null && newFieldMapping.get(defaultSchemaField) != null &&
existingFieldMapping.get(defaultSchemaField).equals(newFieldMapping.get(defaultSchemaField))
) || (
e.get(defaultSchemaField) == null && newFieldMapping.get(defaultSchemaField) == null
existingFieldMapping.get(defaultSchemaField) == null && newFieldMapping.get(defaultSchemaField) == null
)) {
foundFieldMappingDoc = Optional.of(e);
foundFieldMappingDoc = Optional.of(existingFieldMapping);
}
// Grabs the right side of the ID with "|" as the delimiter if present representing the ecs field from predefined mappings
// Additional check to see if raw field path + log type combination is already in existingFieldMappings so a new one is not indexed
} else {
String id = existingFieldMapping.getId();
int indexOfPipe = id.indexOf("|");
if (indexOfPipe != -1) {
String ecsIdField = id.substring(indexOfPipe + 1);
if (ecsIdField.equals(newFieldMapping.getRawField()) && existingFieldMapping.getLogTypes().containsAll(newFieldMapping.getLogTypes())) {
foundFieldMappingDoc = Optional.of(existingFieldMapping);
}
}
}
}
Expand Down
102 changes: 85 additions & 17 deletions src/test/java/org/opensearch/securityanalytics/TestHelpers.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -239,8 +239,8 @@ public static String randomRule() {
"level: high";
}

public static String randomNullRule() {
return "title: null field\n" +
public static String randomRuleWithRawField() {
return "title: Remote Encrypting File System Abuse\n" +
"id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" +
"description: Detects remote RPC calls to possibly abuse remote encryption service via MS-EFSR\n" +
"references:\n" +
Expand All @@ -258,18 +258,50 @@ public static String randomNullRule() {
"logsource:\n" +
" product: rpc_firewall\n" +
" category: application\n" +
" definition: 'Requirements: install and apply the RPC Firew all to all processes with \"audit:true action:block uuid:df1941c5-fe89-4e79-bf10-463657acf44d or c681d488-d850-11d0-8c52-00c04fd90f7e'\n" +
" definition: 'Requirements: install and apply the RPC Firewall to all processes with \"audit:true action:block uuid:df1941c5-fe89-4e79-bf10-463657acf44d or c681d488-d850-11d0-8c52-00c04fd90f7e'\n" +
"detection:\n" +
" selection:\n" +
" EventID: 22\n" +
" RecordNumber: null\n" +
" eventName: testinghere\n" +
" condition: selection\n" +
"falsepositives:\n" +
" - Legitimate usage of remote file encryption\n" +
"level: high";
}

public static String randomRuleForMappingView(String field) {
public static String randomRuleWithNotCondition() {
return "title: Remote Encrypting File System Abuse\n" +
"id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" +
"description: Detects remote RPC calls to possibly abuse remote encryption service via MS-EFSR\n" +
"references:\n" +
" - https://attack.mitre.org/tactics/TA0008/\n" +
" - https://msrc.microsoft.com/update-guide/vulnerability/CVE-2021-36942\n" +
" - https://github.com/jsecurity101/MSRPC-to-ATTACK/blob/main/documents/MS-EFSR.md\n" +
" - https://github.com/zeronetworks/rpcfirewall\n" +
" - https://zeronetworks.com/blog/stopping_lateral_movement_via_the_rpc_firewall/\n" +
"tags:\n" +
" - attack.defense_evasion\n" +
"status: experimental\n" +
"author: Sagie Dulce, Dekel Paz\n" +
"date: 2022/01/01\n" +
"modified: 2022/01/01\n" +
"logsource:\n" +
" product: rpc_firewall\n" +
" category: application\n" +
" definition: 'Requirements: install and apply the RPC Firewall to all processes with \"audit:true action:block uuid:df1941c5-fe89-4e79-bf10-463657acf44d or c681d488-d850-11d0-8c52-00c04fd90f7e'\n" +
"detection:\n" +
" selection1:\n" +
" AccountType: TestAccountType\n" +
" selection2:\n" +
" AccountName: TestAccountName\n" +
" selection3:\n" +
" EventID: 22\n" +
" condition: (not selection1 and not selection2) and selection3\n" +
"falsepositives:\n" +
" - Legitimate usage of remote file encryption\n" +
"level: high";
}

public static String randomRuleWithNotConditionBoolAndNum() {
return "title: Remote Encrypting File System Abuse\n" +
"id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" +
"description: Detects remote RPC calls to possibly abuse remote encryption service via MS-EFSR\n" +
Expand All @@ -290,8 +322,42 @@ public static String randomRuleForMappingView(String field) {
" category: application\n" +
" definition: 'Requirements: install and apply the RPC Firewall to all processes with \"audit:true action:block uuid:df1941c5-fe89-4e79-bf10-463657acf44d or c681d488-d850-11d0-8c52-00c04fd90f7e'\n" +
"detection:\n" +
" selection1:\n" +
" Initiated: \"false\"\n" +
" selection2:\n" +
" AccountName: TestAccountName\n" +
" selection3:\n" +
" EventID: 21\n" +
" condition: not selection1 and not selection3\n" +
"falsepositives:\n" +
" - Legitimate usage of remote file encryption\n" +
"level: high";
}

public static String randomNullRule() {
return "title: null field\n" +
"id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" +
"description: Detects remote RPC calls to possibly abuse remote encryption service via MS-EFSR\n" +
"references:\n" +
" - https://attack.mitre.org/tactics/TA0008/\n" +
" - https://msrc.microsoft.com/update-guide/vulnerability/CVE-2021-36942\n" +
" - https://github.com/jsecurity101/MSRPC-to-ATTACK/blob/main/documents/MS-EFSR.md\n" +
" - https://github.com/zeronetworks/rpcfirewall\n" +
" - https://zeronetworks.com/blog/stopping_lateral_movement_via_the_rpc_firewall/\n" +
"tags:\n" +
" - attack.defense_evasion\n" +
"status: experimental\n" +
"author: Sagie Dulce, Dekel Paz\n" +
"date: 2022/01/01\n" +
"modified: 2022/01/01\n" +
"logsource:\n" +
" product: rpc_firewall\n" +
" category: application\n" +
" definition: 'Requirements: install and apply the RPC Firew all to all processes with \"audit:true action:block uuid:df1941c5-fe89-4e79-bf10-463657acf44d or c681d488-d850-11d0-8c52-00c04fd90f7e'\n" +
"detection:\n" +
" selection:\n" +
" "+ field + ": 'ACL'\n" +
" EventID: 22\n" +
" RecordNumber: null\n" +
" condition: selection\n" +
"falsepositives:\n" +
" - Legitimate usage of remote file encryption\n" +
Expand Down Expand Up @@ -1635,31 +1701,31 @@ public static String windowsIndexMappingOnlyNumericAndText() {
" }";
}

public static String randomDocWithNullField() {
return "{\n" +
"\"@timestamp\":\"2020-02-04T14:59:39.343541+00:00\",\n" +

public static String randomDoc(int severity, int version, String opCode) {
String doc = "{\n" +
"\"EventTime\":\"2020-02-04T14:59:39.343541+00:00\",\n" +
"\"HostName\":\"EC2AMAZ-EPO7HKA\",\n" +
"\"Keywords\":\"9223372036854775808\",\n" +
"\"SeverityValue\":2,\n" +
"\"SeverityValue\":%s,\n" +
"\"Severity\":\"INFO\",\n" +
"\"EventID\":22,\n" +
"\"SourceName\":\"Microsoft-Windows-Sysmon\",\n" +
"\"ProviderGuid\":\"{5770385F-C22A-43E0-BF4C-06F5698FFBD9}\",\n" +
"\"Version\":5,\n" +
"\"Version\":%s,\n" +
"\"TaskValue\":22,\n" +
"\"OpcodeValue\":0,\n" +
"\"RecordNumber\":null,\n" +
"\"RecordNumber\":9532,\n" +
"\"ExecutionProcessID\":1996,\n" +
"\"ExecutionThreadID\":2616,\n" +
"\"Channel\":\"Microsoft-Windows-Sysmon/Operational\",\n" +
"\"Domain\":\"NTAUTHORITY\",\n" +
"\"Domain\":\"NT AUTHORITY\",\n" +
"\"AccountName\":\"SYSTEM\",\n" +
"\"UserID\":\"S-1-5-18\",\n" +
"\"AccountType\":\"User\",\n" +
"\"Message\":\"Dns query:\\r\\nRuleName: \\r\\nUtcTime: 2020-02-04 14:59:38.349\\r\\nProcessGuid: {b3c285a4-3cda-5dc0-0000-001077270b00}\\r\\nProcessId: 1904\\r\\nQueryName: EC2AMAZ-EPO7HKA\\r\\nQueryStatus: 0\\r\\nQueryResults: 172.31.46.38;\\r\\nImage: C:\\\\Program Files\\\\nxlog\\\\nxlog.exe\",\n" +
"\"Category\":\"Dns query (rule: DnsQuery)\",\n" +
"\"Opcode\":\"Info\",\n" +
"\"Opcode\":\"%s\",\n" +
"\"UtcTime\":\"2020-02-04 14:59:38.349\",\n" +
"\"ProcessGuid\":\"{b3c285a4-3cda-5dc0-0000-001077270b00}\",\n" +
"\"ProcessId\":\"1904\",\"QueryName\":\"EC2AMAZ-EPO7HKA\",\"QueryStatus\":\"0\",\n" +
Expand All @@ -1671,9 +1737,11 @@ public static String randomDocWithNullField() {
"\"CommandLine\": \"eachtest\",\n" +
"\"Initiated\": \"true\"\n" +
"}";
return String.format(Locale.ROOT, doc, severity, version, opCode);

}

public static String randomDoc(int severity, int version, String opCode) {
public static String randomDocForNotCondition(int severity, int version, String opCode) {
String doc = "{\n" +
"\"EventTime\":\"2020-02-04T14:59:39.343541+00:00\",\n" +
"\"HostName\":\"EC2AMAZ-EPO7HKA\",\n" +
Expand All @@ -1691,7 +1759,6 @@ public static String randomDoc(int severity, int version, String opCode) {
"\"ExecutionThreadID\":2616,\n" +
"\"Channel\":\"Microsoft-Windows-Sysmon/Operational\",\n" +
"\"Domain\":\"NT AUTHORITY\",\n" +
"\"AccountName\":\"SYSTEM\",\n" +
"\"UserID\":\"S-1-5-18\",\n" +
"\"AccountType\":\"User\",\n" +
"\"Message\":\"Dns query:\\r\\nRuleName: \\r\\nUtcTime: 2020-02-04 14:59:38.349\\r\\nProcessGuid: {b3c285a4-3cda-5dc0-0000-001077270b00}\\r\\nProcessId: 1904\\r\\nQueryName: EC2AMAZ-EPO7HKA\\r\\nQueryStatus: 0\\r\\nQueryResults: 172.31.46.38;\\r\\nImage: C:\\\\Program Files\\\\nxlog\\\\nxlog.exe\",\n" +
Expand Down Expand Up @@ -1783,6 +1850,7 @@ public static String randomVpcFlowDoc() {
" \"srcport\": 9000,\n" +
" \"dstport\": 8000,\n" +
" \"severity_id\": \"-1\",\n" +
" \"id.orig_h\": \"1.2.3.4\",\n" +
" \"class_name\": \"Network Activity\"\n" +
"}";
}
Expand Down
46 changes: 46 additions & 0 deletions src/test/java/org/opensearch/securityanalytics/resthandler/OCSFDetectorRestApiIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -439,6 +439,52 @@ public void testOCSFCloudtrailGetMappingsViewApi() throws IOException {
assertEquals(24, unmappedFieldAliases.size());
}

@SuppressWarnings("unchecked")
public void testOCSFCloudtrailGetMappingsViewApiWithCustomRule() throws IOException {
String index = createTestIndex("cloudtrail", ocsfCloudtrailMappings());

Request request = new Request("GET", SecurityAnalyticsPlugin.MAPPINGS_VIEW_BASE_URI);
// both req params and req body are supported
request.addParameter("index_name", index);
request.addParameter("rule_topic", "cloudtrail");
Response response = client().performRequest(request);
assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode());
Map<String, Object> respMap = responseAsMap(response);
// Verify alias mappings
Map<String, Object> props = (Map<String, Object>) respMap.get("properties");
Assert.assertEquals(18, props.size());
// Verify unmapped index fields
List<String> unmappedIndexFields = (List<String>) respMap.get("unmapped_index_fields");
assertEquals(20, unmappedIndexFields.size());
// Verify unmapped field aliases
List<String> unmappedFieldAliases = (List<String>) respMap.get("unmapped_field_aliases");
assertEquals(25, unmappedFieldAliases.size());

// create a cloudtrail rule with a raw field
String rule = randomRuleWithRawField();
Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.RULE_BASE_URI, Collections.singletonMap("category", "cloudtrail"),
new StringEntity(rule), new BasicHeader("Content-Type", "application/json"));
Assert.assertEquals("Create rule failed", RestStatus.CREATED, restStatus(createResponse));

// check the mapping view API again to ensure it's the same after rule is created
Response response2 = client().performRequest(request);
assertEquals(HttpStatus.SC_OK, response2.getStatusLine().getStatusCode());
Map<String, Object> respMap2 = responseAsMap(response2);
// Verify alias mappings
Map<String, Object> props2 = (Map<String, Object>) respMap2.get("properties");
Assert.assertEquals(18, props2.size());
// Verify unmapped index fields
List<String> unmappedIndexFields2 = (List<String>) respMap2.get("unmapped_index_fields");
assertEquals(20, unmappedIndexFields2.size());
// Verify unmapped field aliases
List<String> unmappedFieldAliases2 = (List<String>) respMap2.get("unmapped_field_aliases");
assertEquals(25, unmappedFieldAliases2.size());
// Verify that first response and second response are the same after rule was indexed
assertEquals(props, props2);
assertEquals(unmappedIndexFields, unmappedIndexFields2);
assertEquals(unmappedFieldAliases, unmappedFieldAliases2);
}

@SuppressWarnings("unchecked")
public void testOCSFVpcflowGetMappingsViewApi() throws IOException {
String index = createTestIndex("vpcflow", ocsfVpcflowMappings());
Expand Down

0 comments on commit 840a8bf

Please sign in to comment.