Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

OCSF1.1 Fixes #1439

Merged
merged 3 commits into from
Jan 7, 2025
+164 −13
Merged

OCSF1.1 Fixes #1439

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
d2e2617
OCSF1.1 Fixes
toepkerd-zz Dec 23, 2024
f2b08c4
reverting var declare ordering
toepkerd-zz Dec 31, 2024
06188c7
adding brief comment explaining importance of the OCSF check ordering
toepkerd-zz Jan 3, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 10 additions & 8 deletions src/main/java/org/opensearch/securityanalytics/mapper/MapperService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -232,10 +232,12 @@ public void onResponse(List<LogType.Mapping> mappings) {
for (LogType.Mapping mapping : mappings) {
if (indexFields.contains(mapping.getRawField())) {
aliasMappingFields.put(mapping.getEcs(), Map.of("type", "alias", "path", mapping.getRawField()));
} else if (indexFields.contains(mapping.getOcsf())) {
aliasMappingFields.put(mapping.getEcs(), Map.of("type", "alias", "path", mapping.getOcsf()));
} else if (indexFields.contains(mapping.getOcsf11())) {
// it's important to first check for OCSF1.1 before checking for OCSF1.0
// changing this order leads to multiple ECS fields mapping to the same OCSF1.1 field
aliasMappingFields.put(mapping.getEcs(), Map.of("type", "alias", "path", mapping.getOcsf11()));
} else if (indexFields.contains(mapping.getOcsf())) {
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Add a comment explaining why the first fallback should be OCSF 1.1.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

adding

aliasMappingFields.put(mapping.getEcs(), Map.of("type", "alias", "path", mapping.getOcsf()));
}
}
aliasMappingsObj.field("properties", aliasMappingFields);
Expand Down Expand Up @@ -497,12 +499,12 @@ public void onResponse(GetMappingsResponse getMappingsResponse) {
}
pathsOfApplyableAliases.add(rawPath);
}
} else if (allFieldsFromIndex.contains(ocsfPath)) {
applyableAliases.add(alias);
pathsOfApplyableAliases.add(ocsfPath);
} else if (allFieldsFromIndex.contains(ocsf11Path)) {
applyableAliases.add(alias);
pathsOfApplyableAliases.add(ocsf11Path);
} else if (allFieldsFromIndex.contains(ocsfPath)) {
applyableAliases.add(alias);
pathsOfApplyableAliases.add(ocsfPath);
} else if ((alias == null && allFieldsFromIndex.contains(rawPath) == false) || allFieldsFromIndex.contains(alias) == false) {
if (alias != null) {
// we don't want to send back aliases which have same name as existing field in index
Expand All @@ -524,10 +526,10 @@ public void onResponse(GetMappingsResponse getMappingsResponse) {
Map<String, Map<String, String>> aliasMappingFields = new HashMap<>();
XContentBuilder aliasMappingsObj = XContentFactory.jsonBuilder().startObject();
for (LogType.Mapping mapping : requiredFields) {
if (allFieldsFromIndex.contains(mapping.getOcsf())) {
aliasMappingFields.put(mapping.getEcs(), Map.of("type", "alias", "path", mapping.getOcsf()));
} else if (allFieldsFromIndex.contains(mapping.getOcsf11())) {
if (allFieldsFromIndex.contains(mapping.getOcsf11())) {
aliasMappingFields.put(mapping.getEcs(), Map.of("type", "alias", "path", mapping.getOcsf11()));
} else if (allFieldsFromIndex.contains(mapping.getOcsf())) {
aliasMappingFields.put(mapping.getEcs(), Map.of("type", "alias", "path", mapping.getOcsf()));
} else if (mapping.getEcs() != null) {
shouldUpdateEcsMappingAndMaybeUpdates(mapping, aliasMappingFields, pathsOfApplyableAliases);
} else if (mapping.getEcs() == null) {
Expand Down
4 changes: 2 additions & 2 deletions src/main/resources/OSMapping/waf_logtype.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,12 +57,12 @@
{
"raw_field":"httpRequest.headers.value",
"ecs":"waf.request.headers.value",
"ocsf": "http_request.http_headers[].value"
"ocsf": "http_request.http_headers.value"
},
{
"raw_field":"httpRequest.headers.name",
"ecs":"waf.request.headers.name",
"ocsf": "http_request.http_headers[].name"
"ocsf": "http_request.http_headers.name"
}
]
}
155 changes: 152 additions & 3 deletions src/test/java/org/opensearch/securityanalytics/resthandler/OCSFDetectorRestApiIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,11 @@
import org.opensearch.securityanalytics.model.Detector;
import org.opensearch.securityanalytics.model.DetectorInput;
import org.opensearch.securityanalytics.model.DetectorRule;
import org.opensearch.securityanalytics.model.DetectorTrigger;

import java.io.IOException;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
Expand Down Expand Up @@ -433,7 +435,7 @@ public void testOCSFCloudtrailGetMappingsViewApi() throws IOException {
Assert.assertEquals(18, props.size());
// Verify unmapped index fields
List<String> unmappedIndexFields = (List<String>) respMap.get("unmapped_index_fields");
assertEquals(20, unmappedIndexFields.size());
assertEquals(21, unmappedIndexFields.size());
// Verify unmapped field aliases
List<String> unmappedFieldAliases = (List<String>) respMap.get("unmapped_field_aliases");
assertEquals(24, unmappedFieldAliases.size());
Expand All @@ -455,7 +457,8 @@ public void testOCSFCloudtrailGetMappingsViewApiWithCustomRule() throws IOExcept
Assert.assertEquals(18, props.size());
// Verify unmapped index fields
List<String> unmappedIndexFields = (List<String>) respMap.get("unmapped_index_fields");
assertEquals(20, unmappedIndexFields.size());

assertEquals(21, unmappedIndexFields.size());
// Verify unmapped field aliases
List<String> unmappedFieldAliases = (List<String>) respMap.get("unmapped_field_aliases");
assertEquals(24, unmappedFieldAliases.size());
Expand All @@ -475,7 +478,7 @@ public void testOCSFCloudtrailGetMappingsViewApiWithCustomRule() throws IOExcept
Assert.assertEquals(18, props2.size());
// Verify unmapped index fields
List<String> unmappedIndexFields2 = (List<String>) respMap2.get("unmapped_index_fields");
assertEquals(20, unmappedIndexFields2.size());
assertEquals(21, unmappedIndexFields2.size());
// Verify unmapped field aliases
List<String> unmappedFieldAliases2 = (List<String>) respMap2.get("unmapped_field_aliases");
assertEquals(24, unmappedFieldAliases2.size());
Expand Down Expand Up @@ -595,6 +598,93 @@ public void testRawRoute53GetMappingsViewApi() throws IOException {
assertEquals(8, unmappedFieldAliases.size());
}

public void testCloudtrailPrincipalIdAndArnFieldsGenerateFinding() throws IOException {
// create an index with OCSF1.1 fields actor.user.uid and actor.user.uid_alt
String indexName = "test_index";
String index = createTestIndex(indexName, ocsf11ReducedCloudtrailMappings());

// create the cloudtrail mappings
createMappingsAPI(indexName, "cloudtrail");

// create the custom rule
String rule = ocsf11Rule();

Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.RULE_BASE_URI, Collections.singletonMap("category", "cloudtrail"),
new StringEntity(rule), new BasicHeader("Content-Type", "application/json"));
Assert.assertEquals("Create rule failed", RestStatus.CREATED, restStatus(createResponse));

Map<String, Object> responseBody = asMap(createResponse);
String ruleId = responseBody.get("_id").toString();

// create the detector that uses only the custom rule
Detector detector = randomDetector(
"cloudtrail-detector",
"cloudtrail",
null,
List.of(
new DetectorInput(
"cloudtrail detector for security analytics",
List.of(indexName),
List.of(new DetectorRule(ruleId)),
List.of()
)
),
List.of(
new DetectorTrigger(
null,
"cloudtrail-trigger",
"1",
List.of("cloudtrail"),
List.of(ruleId),
List.of(),
List.of(),
List.of(),
List.of()
)
),
null,
true,
null,
null,
false
);

createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector));
Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse));

responseBody = asMap(createResponse);

String detectorId = responseBody.get("_id").toString();

// get the underlying alerting monitor for the detector so we can manually execute it
String request = "{\n" +
" \"query\" : {\n" +
" \"match\":{\n" +
" \"_id\": \"" + detectorId + "\"\n" +
" }\n" +
" }\n" +
"}";
List<SearchHit> hits = executeSearch(Detector.DETECTORS_INDEX, request);
SearchHit hit = hits.get(0);

String detectorType = (String) ((Map<String, Object>) hit.getSourceAsMap().get("detector")).get("detector_type");
Assert.assertEquals("Detector type incorrect", "cloudtrail", detectorType.toLowerCase(Locale.ROOT));

String monitorId = ((List<String>) ((Map<String, Object>) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0);

// index a document that should trigger a finding
indexDoc(index, "1", ocsf11Doc());

// execute detector by executing its underlying monitor
executeAlertingMonitor(monitorId, Collections.emptyMap());

Map<String, String> params = new HashMap<>();
params.put("detector_id", detectorId);
Response getFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.FINDINGS_BASE_URI + "/_search", params, null);
Map<String, Object> getFindingsBody = entityAsMap(getFindingsResponse);
Assert.assertEquals(1, getFindingsBody.get("total_findings"));
}

private String rawCloudtrailDoc() {
return "{\n" +
" \"eventVersion\": \"1.03\",\n" +
Expand Down Expand Up @@ -2810,4 +2900,63 @@ private String rawVpcFlowMappings() {
" }\n" +
" }";
}

private String ocsf11ReducedCloudtrailMappings() {
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good job adding integ tests!

return "\"properties\": {\n" +
" \"actor.user.uid_alt\": {\n" +
" \"type\": \"text\",\n" +
" \"fields\": {\n" +
" \"keyword\": {\n" +
" \"type\": \"keyword\",\n" +
" \"ignore_above\": 256\n" +
" }\n" +
" }\n" +
" },\n" +
" \"actor.user.uid\": {\n" +
" \"type\": \"text\",\n" +
" \"fields\": {\n" +
" \"keyword\": {\n" +
" \"type\": \"keyword\",\n" +
" \"ignore_above\": 256\n" +
" }\n" +
" }\n" +
" }\n" +
" }";
}

private String ocsf11Rule() {
return "title: Cloudtrail Principal ID Rule\n" +
"id: 5f92fff9-82e2-48eb-8fc1-8b133556a123\n" +
"description: A rule that checks specifically for the cloudtrail principal ID field\n" +
"references:\n" +
" - https://attack.mitre.org/tactics/TA0008/\n" +
" - https://msrc.microsoft.com/update-guide/vulnerability/CVE-2021-36942\n" +
" - https://github.com/jsecurity101/MSRPC-to-ATTACK/blob/main/documents/MS-EFSR.md\n" +
" - https://github.com/zeronetworks/rpcfirewall\n" +
" - https://zeronetworks.com/blog/stopping_lateral_movement_via_the_rpc_firewall/\n" +
"tags:\n" +
" - attack.defense_evasion\n" +
"status: experimental\n" +
"author: Sagie Dulce, Dekel Paz\n" +
"date: 2022/01/01\n" +
"modified: 2022/01/01\n" +
"logsource:\n" +
" product: rpc_firewall\n" +
" category: application\n" +
" definition: 'Requirements: install and apply the RPC Firewall to all processes with \"audit:true action:block uuid:df1941c5-fe89-4e79-bf10-463657acf44d or c681d488-d850-11d0-8c52-00c04fd90f7e'\n" +
"detection:\n" +
" selection:\n" +
" aws.cloudtrail.user_identity.principalId: abc\n" +
" condition: selection\n" +
"falsepositives:\n" +
" - Legitimate usage of remote file encryption\n" +
"level: high";
}

public String ocsf11Doc() {
return "{\n" +
"\"actor.user.uid_alt\":\"abc\",\n" +
"\"actor.user.uid\":\"def\"\n" +
"}";
}
}
Loading