Skip to content

Commit

Permalink
Fix get mappings view API incorrectly returning ecs path (#867)
Browse files Browse the repository at this point in the history 
* add logic and integ tests to not add duplicate to log-types-config index

Signed-off-by: Joanne Wang <[email protected]>

* change naming for existingFieldMapping and change contains to equals

Signed-off-by: Joanne Wang <[email protected]>

---------

Signed-off-by: Joanne Wang <[email protected]>
  • Loading branch information
@jowg-amazon @riysaxen-amzn
jowg-amazon authored and riysaxen-amzn committed Mar 18, 2024
1 parent 1f630c1 commit dd53fa4
Show file tree
Hide file tree
Showing 3 changed files with 158 additions and 6 deletions.
23 changes: 17 additions & 6 deletions src/main/java/org/opensearch/securityanalytics/logtype/LogTypeService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -344,15 +344,26 @@ private List<FieldMappingDoc> mergeFieldMappings(List<FieldMappingDoc> existingF
List<FieldMappingDoc> newFieldMappings = new ArrayList<>();
fieldMappingDocs.forEach( newFieldMapping -> {
Optional<FieldMappingDoc> foundFieldMappingDoc = Optional.empty();
for (FieldMappingDoc e: existingFieldMappings) {
if (e.getRawField().equals(newFieldMapping.getRawField())) {
for (FieldMappingDoc existingFieldMapping: existingFieldMappings) {
if (existingFieldMapping.getRawField().equals(newFieldMapping.getRawField())) {
if ((
e.get(defaultSchemaField) != null && newFieldMapping.get(defaultSchemaField) != null &&
e.get(defaultSchemaField).equals(newFieldMapping.get(defaultSchemaField))
existingFieldMapping.get(defaultSchemaField) != null && newFieldMapping.get(defaultSchemaField) != null &&
existingFieldMapping.get(defaultSchemaField).equals(newFieldMapping.get(defaultSchemaField))
) || (
e.get(defaultSchemaField) == null && newFieldMapping.get(defaultSchemaField) == null
existingFieldMapping.get(defaultSchemaField) == null && newFieldMapping.get(defaultSchemaField) == null
)) {
foundFieldMappingDoc = Optional.of(e);
foundFieldMappingDoc = Optional.of(existingFieldMapping);
}
// Grabs the right side of the ID with "|" as the delimiter if present representing the ecs field from predefined mappings
// Additional check to see if raw field path + log type combination is already in existingFieldMappings so a new one is not indexed
} else {
String id = existingFieldMapping.getId();
int indexOfPipe = id.indexOf("|");
if (indexOfPipe != -1) {
String ecsIdField = id.substring(indexOfPipe + 1);
if (ecsIdField.equals(newFieldMapping.getRawField()) && existingFieldMapping.getLogTypes().containsAll(newFieldMapping.getLogTypes())) {
foundFieldMappingDoc = Optional.of(existingFieldMapping);
}
}
}
}
Expand Down
95 changes: 95 additions & 0 deletions src/test/java/org/opensearch/securityanalytics/TestHelpers.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -209,6 +209,101 @@ public static String randomRule() {
"level: high";
}

public static String randomRuleWithRawField() {
return "title: Remote Encrypting File System Abuse\n" +
"id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" +
"description: Detects remote RPC calls to possibly abuse remote encryption service via MS-EFSR\n" +
"references:\n" +
" - https://attack.mitre.org/tactics/TA0008/\n" +
" - https://msrc.microsoft.com/update-guide/vulnerability/CVE-2021-36942\n" +
" - https://github.com/jsecurity101/MSRPC-to-ATTACK/blob/main/documents/MS-EFSR.md\n" +
" - https://github.com/zeronetworks/rpcfirewall\n" +
" - https://zeronetworks.com/blog/stopping_lateral_movement_via_the_rpc_firewall/\n" +
"tags:\n" +
" - attack.defense_evasion\n" +
"status: experimental\n" +
"author: Sagie Dulce, Dekel Paz\n" +
"date: 2022/01/01\n" +
"modified: 2022/01/01\n" +
"logsource:\n" +
" product: rpc_firewall\n" +
" category: application\n" +
" definition: 'Requirements: install and apply the RPC Firewall to all processes with \"audit:true action:block uuid:df1941c5-fe89-4e79-bf10-463657acf44d or c681d488-d850-11d0-8c52-00c04fd90f7e'\n" +
"detection:\n" +
" selection:\n" +
" eventName: testinghere\n" +
" condition: selection\n" +
"falsepositives:\n" +
" - Legitimate usage of remote file encryption\n" +
"level: high";
}

public static String randomRuleWithNotCondition() {
return "title: Remote Encrypting File System Abuse\n" +
"id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" +
"description: Detects remote RPC calls to possibly abuse remote encryption service via MS-EFSR\n" +
"references:\n" +
" - https://attack.mitre.org/tactics/TA0008/\n" +
" - https://msrc.microsoft.com/update-guide/vulnerability/CVE-2021-36942\n" +
" - https://github.com/jsecurity101/MSRPC-to-ATTACK/blob/main/documents/MS-EFSR.md\n" +
" - https://github.com/zeronetworks/rpcfirewall\n" +
" - https://zeronetworks.com/blog/stopping_lateral_movement_via_the_rpc_firewall/\n" +
"tags:\n" +
" - attack.defense_evasion\n" +
"status: experimental\n" +
"author: Sagie Dulce, Dekel Paz\n" +
"date: 2022/01/01\n" +
"modified: 2022/01/01\n" +
"logsource:\n" +
" product: rpc_firewall\n" +
" category: application\n" +
" definition: 'Requirements: install and apply the RPC Firewall to all processes with \"audit:true action:block uuid:df1941c5-fe89-4e79-bf10-463657acf44d or c681d488-d850-11d0-8c52-00c04fd90f7e'\n" +
"detection:\n" +
" selection1:\n" +
" AccountType: TestAccountType\n" +
" selection2:\n" +
" AccountName: TestAccountName\n" +
" selection3:\n" +
" EventID: 22\n" +
" condition: (not selection1 and not selection2) and selection3\n" +
"falsepositives:\n" +
" - Legitimate usage of remote file encryption\n" +
"level: high";
}

public static String randomRuleWithNotConditionBoolAndNum() {
return "title: Remote Encrypting File System Abuse\n" +
"id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" +
"description: Detects remote RPC calls to possibly abuse remote encryption service via MS-EFSR\n" +
"references:\n" +
" - https://attack.mitre.org/tactics/TA0008/\n" +
" - https://msrc.microsoft.com/update-guide/vulnerability/CVE-2021-36942\n" +
" - https://github.com/jsecurity101/MSRPC-to-ATTACK/blob/main/documents/MS-EFSR.md\n" +
" - https://github.com/zeronetworks/rpcfirewall\n" +
" - https://zeronetworks.com/blog/stopping_lateral_movement_via_the_rpc_firewall/\n" +
"tags:\n" +
" - attack.defense_evasion\n" +
"status: experimental\n" +
"author: Sagie Dulce, Dekel Paz\n" +
"date: 2022/01/01\n" +
"modified: 2022/01/01\n" +
"logsource:\n" +
" product: rpc_firewall\n" +
" category: application\n" +
" definition: 'Requirements: install and apply the RPC Firewall to all processes with \"audit:true action:block uuid:df1941c5-fe89-4e79-bf10-463657acf44d or c681d488-d850-11d0-8c52-00c04fd90f7e'\n" +
"detection:\n" +
" selection1:\n" +
" Initiated: \"false\"\n" +
" selection2:\n" +
" AccountName: TestAccountName\n" +
" selection3:\n" +
" EventID: 21\n" +
" condition: not selection1 and not selection3\n" +
"falsepositives:\n" +
" - Legitimate usage of remote file encryption\n" +
"level: high";
}

public static String randomNullRule() {
return "title: null field\n" +
"id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" +
Expand Down
46 changes: 46 additions & 0 deletions src/test/java/org/opensearch/securityanalytics/resthandler/OCSFDetectorRestApiIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -439,6 +439,52 @@ public void testOCSFCloudtrailGetMappingsViewApi() throws IOException {
assertEquals(24, unmappedFieldAliases.size());
}

@SuppressWarnings("unchecked")
public void testOCSFCloudtrailGetMappingsViewApiWithCustomRule() throws IOException {
String index = createTestIndex("cloudtrail", ocsfCloudtrailMappings());

Request request = new Request("GET", SecurityAnalyticsPlugin.MAPPINGS_VIEW_BASE_URI);
// both req params and req body are supported
request.addParameter("index_name", index);
request.addParameter("rule_topic", "cloudtrail");
Response response = client().performRequest(request);
assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode());
Map<String, Object> respMap = responseAsMap(response);
// Verify alias mappings
Map<String, Object> props = (Map<String, Object>) respMap.get("properties");
Assert.assertEquals(18, props.size());
// Verify unmapped index fields
List<String> unmappedIndexFields = (List<String>) respMap.get("unmapped_index_fields");
assertEquals(20, unmappedIndexFields.size());
// Verify unmapped field aliases
List<String> unmappedFieldAliases = (List<String>) respMap.get("unmapped_field_aliases");
assertEquals(25, unmappedFieldAliases.size());

// create a cloudtrail rule with a raw field
String rule = randomRuleWithRawField();
Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.RULE_BASE_URI, Collections.singletonMap("category", "cloudtrail"),
new StringEntity(rule), new BasicHeader("Content-Type", "application/json"));
Assert.assertEquals("Create rule failed", RestStatus.CREATED, restStatus(createResponse));

// check the mapping view API again to ensure it's the same after rule is created
Response response2 = client().performRequest(request);
assertEquals(HttpStatus.SC_OK, response2.getStatusLine().getStatusCode());
Map<String, Object> respMap2 = responseAsMap(response2);
// Verify alias mappings
Map<String, Object> props2 = (Map<String, Object>) respMap2.get("properties");
Assert.assertEquals(18, props2.size());
// Verify unmapped index fields
List<String> unmappedIndexFields2 = (List<String>) respMap2.get("unmapped_index_fields");
assertEquals(20, unmappedIndexFields2.size());
// Verify unmapped field aliases
List<String> unmappedFieldAliases2 = (List<String>) respMap2.get("unmapped_field_aliases");
assertEquals(25, unmappedFieldAliases2.size());
// Verify that first response and second response are the same after rule was indexed
assertEquals(props, props2);
assertEquals(unmappedIndexFields, unmappedIndexFields2);
assertEquals(unmappedFieldAliases, unmappedFieldAliases2);
}

@SuppressWarnings("unchecked")
public void testOCSFVpcflowGetMappingsViewApi() throws IOException {
String index = createTestIndex("vpcflow", ocsfVpcflowMappings());
Expand Down

0 comments on commit dd53fa4

Please sign in to comment.