From af12c26c8c05b564ea70b6a468188ba75963cea2 Mon Sep 17 00:00:00 2001 From: Marius Iversen Date: Mon, 3 May 2021 15:21:55 +0200 Subject: [PATCH] Cherry-pick #25421 to 7.12: [Filebeat] Allow m365 defender to receive incidents with no alerts (#25485) * [Filebeat] Allow m365 defender to receive incidents with no alerts (#25421) * allowing incidents with no alerts to be parsed, updating some typos and making the pipeline safer in terms of null def references * adding changelog entry * updating local test files * reverting changes to json decoding, back to beat (cherry picked from commit 34837f553d5b21adb8113c4bc7879c9ad199581f) * Update CHANGELOG.next.asciidoc --- CHANGELOG.next.asciidoc | 1 + .../m365_defender/ingest/pipeline.yml | 39 ++++++++++++------- .../test/m365_defender-test-empty.ndjson.log | 1 + ...fender-test-empty.ndjson.log-expected.json | 30 ++++++++++++++ .../test/m365_defender-test.ndjson.log | 2 +- ...365_defender-test.ndjson.log-expected.json | 29 +++++++------- 6 files changed, 75 insertions(+), 27 deletions(-) create mode 100644 x-pack/filebeat/module/microsoft/m365_defender/test/m365_defender-test-empty.ndjson.log create mode 100644 x-pack/filebeat/module/microsoft/m365_defender/test/m365_defender-test-empty.ndjson.log-expected.json diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 8f8cead1aaad..5ab890ab82c2 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -125,6 +125,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - system/package: Fix an error that can occur while trying to persist package metadata. {issue}18536[18536] {pull}18887[18887] - system/socket: Fix dataset using 100% CPU and becoming unresponsive in some scenarios. {pull}19033[19033] {pull}19764[19764] - system/socket: Fixed tracking of long-running connections. {pull}19033[19033] +- Fix issue with m365_defender, when parsing incidents that has no alerts attached: {pull}25421[25421] *Filebeat* diff --git a/x-pack/filebeat/module/microsoft/m365_defender/ingest/pipeline.yml b/x-pack/filebeat/module/microsoft/m365_defender/ingest/pipeline.yml index ae33c77d6d5e..824363d5ffb8 100644 --- a/x-pack/filebeat/module/microsoft/m365_defender/ingest/pipeline.yml +++ b/x-pack/filebeat/module/microsoft/m365_defender/ingest/pipeline.yml @@ -10,6 +10,11 @@ processors: - json.comments - host ignore_missing: true +- remove: + field: + - json.alerts + ignore_missing: true + if: 'ctx?.json?.alerts == null || ctx?.json?.alerts instanceof List' ######################### ## ECS General Mapping ## @@ -45,11 +50,16 @@ processors: - set: field: '@timestamp' value: '{{json.lastUpdateTime}}' - if: ctx.json?.lastUpdateTime != null + if: ctx?.json?.lastUpdateTime != null - rename: field: json.alerts.title target_field: message ignore_missing: true +- rename: + field: json.incidentName + target_field: message + ignore_missing: true + if: ctx?.message == null ####################### ## ECS Event Mapping ## @@ -144,7 +154,7 @@ processors: - set: field: threat.framework value: MITRE ATT&CK - if: ctx.json?.alerts?.category != null + if: ctx?.json?.alerts?.category != null - rename: field: json.alerts.category target_field: threat.technique.name @@ -153,7 +163,7 @@ processors: field: json.alerts.description target_field: rule.description ignore_missing: true - if: ctx.json?.alerts?.description.length() < 1020 + if: ctx?.json?.alerts != null && ctx.json?.alerts?.description.length() < 1020 ###################### ## ECS File Mapping ## @@ -220,8 +230,11 @@ processors: field: json.alerts.entities.url target_field: url.full ignore_missing: true - if: ctx?.json?.entities?.url != null - + if: ctx?.json?.alerts?.entities?.url != null +- uri_parts: + field: url.full + ignore_failure: true + if: ctx?.url?.full != null ###################### ## ECS User Mapping ## ###################### @@ -261,23 +274,23 @@ processors: - append: field: related.ip value: '{{json.alerts.entities.ipAddress}}' - if: ctx.json?.entities?.ipAddress != null + if: ctx?.json?.alerts?.entities?.ipAddress != null - append: field: related.user value: '{{user.name}}' - if: ctx.user?.name != null + if: ctx?.user?.name != null - append: field: related.hash value: '{{file.hash.sha1}}' - if: ctx.file?.hash?.sha1 != null + if: ctx?.file?.hash?.sha1 != null - append: field: related.hash value: '{{file.hash.sha256}}' - if: ctx.file?.hash?.sha256 != null + if: ctx?.file?.hash?.sha256 != null - append: field: related.hosts value: '{{host.hostname}}' - if: ctx.host?.hostname != null + if: ctx?.host?.hostname != null ############# ## Cleanup ## @@ -293,15 +306,15 @@ processors: - remove: field: json.alerts.mitreTechniques ignore_missing: true - if: ctx?.json?.alerts?.mitreTechniques.isEmpty() + if: 'ctx?.json?.alerts != null && ctx?.json?.alerts?.mitreTechniques.isEmpty()' - remove: field: json.alerts.devices ignore_missing: true - if: ctx?.json?.alerts?.devices.isEmpty() + if: 'ctx?.json?.alerts != null && ctx?.json?.alerts?.devices.isEmpty()' - remove: field: json.tags ignore_missing: true - if: ctx?.json?.tags.isEmpty() + if: 'ctx?.json?.alerts != null && ctx?.json?.tags.isEmpty()' - remove: ignore_missing: true field: diff --git a/x-pack/filebeat/module/microsoft/m365_defender/test/m365_defender-test-empty.ndjson.log b/x-pack/filebeat/module/microsoft/m365_defender/test/m365_defender-test-empty.ndjson.log new file mode 100644 index 000000000000..8bd804528332 --- /dev/null +++ b/x-pack/filebeat/module/microsoft/m365_defender/test/m365_defender-test-empty.ndjson.log @@ -0,0 +1 @@ +{"incidentId":1111,"redirectIncidentId":1107,"incidentName":"Impossible travel activity involving one user","createdTime":"2021-04-12T11:18:28.86Z","lastUpdateTime":"2021-04-12T11:18:30.4033333Z","assignedTo":null,"classification":"Unknown","determination":"NotAvailable","status":"Redirected","severity":"UnSpecified","tags":[],"comments":[],"alerts":[]} diff --git a/x-pack/filebeat/module/microsoft/m365_defender/test/m365_defender-test-empty.ndjson.log-expected.json b/x-pack/filebeat/module/microsoft/m365_defender/test/m365_defender-test-empty.ndjson.log-expected.json new file mode 100644 index 000000000000..6b59f0f8c945 --- /dev/null +++ b/x-pack/filebeat/module/microsoft/m365_defender/test/m365_defender-test-empty.ndjson.log-expected.json @@ -0,0 +1,30 @@ +[ + { + "@timestamp": "2021-04-12T11:18:30.4033333Z", + "cloud.provider": "azure", + "event.category": [ + "host" + ], + "event.dataset": "microsoft.m365_defender", + "event.kind": "alert", + "event.module": "microsoft", + "event.timezone": "UTC", + "fileset.name": "m365_defender", + "input.type": "log", + "log.offset": 0, + "message": "Impossible travel activity involving one user", + "microsoft.m365_defender.classification": "Unknown", + "microsoft.m365_defender.determination": "NotAvailable", + "microsoft.m365_defender.incidentId": "1111", + "microsoft.m365_defender.redirectIncidentId": 1107, + "microsoft.m365_defender.status": "Redirected", + "microsoft.m365_defender.tags": [], + "observer.product": "365 Defender", + "observer.vendor": "Microsoft", + "service.type": "microsoft", + "tags": [ + "m365-defender", + "forwarded" + ] + } +] \ No newline at end of file diff --git a/x-pack/filebeat/module/microsoft/m365_defender/test/m365_defender-test.ndjson.log b/x-pack/filebeat/module/microsoft/m365_defender/test/m365_defender-test.ndjson.log index 4fc4cf141b6b..0fd241be2e3f 100644 --- a/x-pack/filebeat/module/microsoft/m365_defender/test/m365_defender-test.ndjson.log +++ b/x-pack/filebeat/module/microsoft/m365_defender/test/m365_defender-test.ndjson.log @@ -5,5 +5,5 @@ {"assignedTo":"elastic@elasticuser.com","classification":"Unknown","createdTime":"2020-06-30T09:32:31.85Z","redirectIncidentId":null,"severity":"Low","status":"Resolved","tags":[],"alerts":{"assignedTo":"elastic@elasticuser.com","determination":null,"serviceSource":"MicrosoftDefenderATP","severity":"Low","alertId":"da637291086161511365_-2075772905","classification":"FalsePositive","creationTime":"2020-06-30T10:10:16.1355657Z","description":"Malware and unwanted software are undesirable applications that perform annoying, disruptive, or harmful actions on affected machines. Some of these undesirable applications can replicate and spread from one machine to another. Others are able to receive commands from remote attackers and perform activities associated with cyber attacks.\n\nA malware is considered active if it is found running on the machine or it already has persistence mechanisms in place. Active malware detections are assigned higher severity ratings.\n\nBecause this malware was active, take precautionary measures and check for residual signs of infection.","entities":{"deviceId":"75a63a39f9bc5a964f417c11f6277d5bf9489f0d","entityType":"Process","processCreationTime":"2020-06-30T10:31:04.1092404Z","processId":6720},"mitreTechniques":[],"title":"Suspicious 'AccessibilityEscalation' behavior was detected","category":"SuspiciousActivity","devices":[{"aadDeviceId":null,"mdatpDeviceId":"75a63a39f9bc5a964f417c11f6277d5bf9489f0d","osProcessor":"x64","riskScore":"High","osPlatform":"Other","rbacGroupId":0,"rbacGroupName":null,"version":"Other","deviceDnsName":"testserver4","firstSeen":"2020-06-30T08:55:08.8320449Z","healthStatus":"Inactive","osBuild":17763}],"firstActivity":"2020-06-30T10:09:10.8889583Z","investigationState":"UnsupportedAlertType","status":"Resolved","detectionSource":"WindowsDefenderAv","incidentId":12,"investigationId":null,"lastActivity":"2020-06-30T10:31:09.4165785Z","lastUpdatedTime":"2020-09-23T19:44:37.9666667Z","resolvedTime":"2020-09-23T19:44:36.1092821Z","threatFamilyName":null,"actorName":null},"determination":"NotAvailable","incidentId":12,"incidentName":"12","lastUpdateTime":"2020-09-23T19:44:36.29Z"} {"determination":"NotAvailable","severity":"Low","classification":"Unknown","createdTime":"2020-06-30T09:32:31.85Z","incidentId":12,"incidentName":"12","lastUpdateTime":"2020-09-23T19:44:36.29Z","redirectIncidentId":null,"alerts":{"lastActivity":"2020-06-30T10:31:09.4165785Z","lastUpdatedTime":"2020-09-23T19:44:37.9666667Z","actorName":null,"description":"Malware and unwanted software are undesirable applications that perform annoying, disruptive, or harmful actions on affected machines. Some of these undesirable applications can replicate and spread from one machine to another. Others are able to receive commands from remote attackers and perform activities associated with cyber attacks.\n\nA malware is considered active if it is found running on the machine or it already has persistence mechanisms in place. Active malware detections are assigned higher severity ratings.\n\nBecause this malware was active, take precautionary measures and check for residual signs of infection.","determination":null,"entities":{"accountName":"","entityType":"User"},"firstActivity":"2020-06-30T10:09:10.8889583Z","investigationState":"UnsupportedAlertType","serviceSource":"MicrosoftDefenderATP","status":"Resolved","title":"Suspicious 'AccessibilityEscalation' behavior was detected","classification":"FalsePositive","devices":[{"aadDeviceId":null,"healthStatus":"Inactive","osPlatform":"Other","osProcessor":"x64","riskScore":"High","deviceDnsName":"testserver4","firstSeen":"2020-06-30T08:55:08.8320449Z","mdatpDeviceId":"75a63a39f9bc5a964f417c11f6277d5bf9489f0d","osBuild":17763,"rbacGroupId":0,"rbacGroupName":null,"version":"Other"}],"mitreTechniques":[],"severity":"Low","threatFamilyName":null,"creationTime":"2020-06-30T10:10:16.1355657Z","detectionSource":"WindowsDefenderAv","incidentId":12,"alertId":"da637291086161511365_-2075772905","assignedTo":"elastic@elasticuser.com","category":"SuspiciousActivity","investigationId":null,"resolvedTime":"2020-09-23T19:44:36.1092821Z"},"assignedTo":"elastic@elasticuser.com","status":"Resolved","tags":[]} {"determination":"NotAvailable","lastUpdateTime":"2020-09-23T19:44:36.29Z","tags":[],"alerts":{"investigationState":"UnsupportedAlertType","status":"Resolved","alertId":"da637291086161511365_-2075772905","assignedTo":"elastic@elasticuser.com","determination":null,"firstActivity":"2020-06-30T10:09:10.8889583Z","mitreTechniques":[],"resolvedTime":"2020-09-23T19:44:36.1092821Z","severity":"Low","actorName":null,"category":"SuspiciousActivity","description":"Malware and unwanted software are undesirable applications that perform annoying, disruptive, or harmful actions on affected machines. Some of these undesirable applications can replicate and spread from one machine to another. Others are able to receive commands from remote attackers and perform activities associated with cyber attacks.\n\nA malware is considered active if it is found running on the machine or it already has persistence mechanisms in place. Active malware detections are assigned higher severity ratings.\n\nBecause this malware was active, take precautionary measures and check for residual signs of infection.","lastUpdatedTime":"2020-09-23T19:44:37.9666667Z","title":"Suspicious 'AccessibilityEscalation' behavior was detected","classification":"FalsePositive","creationTime":"2020-06-30T10:10:16.1355657Z","entities":{"deviceId":"75a63a39f9bc5a964f417c11f6277d5bf9489f0d","entityType":"Process","processCreationTime":"2020-06-30T10:09:10.5747992Z","processId":1324},"incidentId":12,"serviceSource":"MicrosoftDefenderATP","threatFamilyName":null,"detectionSource":"WindowsDefenderAv","devices":[{"osPlatform":"Other","osProcessor":"x64","rbacGroupId":0,"riskScore":"High","version":"Other","aadDeviceId":null,"deviceDnsName":"testserver4","mdatpDeviceId":"75a63a39f9bc5a964f417c11f6277d5bf9489f0d","rbacGroupName":null,"firstSeen":"2020-06-30T08:55:08.8320449Z","healthStatus":"Inactive","osBuild":17763}],"investigationId":null,"lastActivity":"2020-06-30T10:31:09.4165785Z"},"assignedTo":"elastic@elasticuser.com","classification":"Unknown","createdTime":"2020-06-30T09:32:31.85Z","status":"Resolved","incidentId":12,"incidentName":"12","redirectIncidentId":null,"severity":"Low"} -{"incidentId":14,"incidentName":"Activity from infrequent country","redirectIncidentId":null,"tags":[],"alerts":{"category":"SuspiciousActivity","entities":{"aadUserId":"8e24c50a-a77c-4782-813f-965009b5ddf3","accountName":"brent","entityType":"User","userPrincipalName":"brent@elasticbv.onmicrosoft.com"},"incidentId":14,"investigationState":"UnsupportedAlertType","status":"New","actorName":null,"classification":"FalsePositive","description":"Brent Murphy (brent@elasticbv.onmicrosoft.com) performed an activity. No activity was performed in United States in the past 41 days.","investigationId":null,"lastActivity":"2020-07-27T15:47:22.088Z","lastUpdatedTime":"2020-09-23T19:32:17.5433333Z","mitreTechniques":[],"serviceSource":"MicrosoftCloudAppSecurity","severity":"Medium","threatFamilyName":null,"title":"Activity from infrequent country","assignedTo":"elastic@elasticuser.com","detectionSource":"MCAS","devices":[],"alertId":"caA214771F-6AB0-311D-B2B0-BECD3B4A967B","creationTime":"2020-07-27T15:54:20.52207Z","determination":null,"firstActivity":"2020-07-27T15:47:22.088Z","resolvedTime":null},"classification":"Unknown","determination":"NotAvailable","lastUpdateTime":"2020-09-23T19:32:05.8366667Z","severity":"Medium","status":"Active","assignedTo":"elastic@elasticuser.com","createdTime":"2020-07-27T15:54:21.58Z"} {"incidentId":14,"incidentName":"Activity from infrequent country","severity":"Medium","status":"Active","tags":[],"alerts":{"description":"Brent Murphy (brent@elasticbv.onmicrosoft.com) performed an activity. No activity was performed in United States in the past 41 days.","detectionSource":"MCAS","firstActivity":"2020-07-27T15:47:22.088Z","investigationId":null,"investigationState":"UnsupportedAlertType","severity":"Medium","alertId":"caA214771F-6AB0-311D-B2B0-BECD3B4A967B","category":"SuspiciousActivity","classification":"FalsePositive","determination":null,"entities":{"entityType":"Ip","ipAddress":"73.172.171.53"},"incidentId":14,"serviceSource":"MicrosoftCloudAppSecurity","status":"New","actorName":null,"title":"Activity from infrequent country","devices":[],"lastActivity":"2020-07-27T15:47:22.088Z","lastUpdatedTime":"2020-09-23T19:32:17.5433333Z","creationTime":"2020-07-27T15:54:20.52207Z","mitreTechniques":[],"resolvedTime":null,"threatFamilyName":null,"assignedTo":"elastic@elasticuser.com"},"createdTime":"2020-07-27T15:54:21.58Z","determination":"NotAvailable","lastUpdateTime":"2020-09-23T19:32:05.8366667Z","redirectIncidentId":null,"assignedTo":"elastic@elasticuser.com","classification":"Unknown"} +{"incidentId":14,"incidentName":"Activity from infrequent country","redirectIncidentId":null,"tags":[],"alerts":{"category":"SuspiciousActivity","entities":{"aadUserId":"8e24c50a-a77c-4782-813f-965009b5ddf3","accountName":"brent","entityType":"User","userPrincipalName":"brent@elasticbv.onmicrosoft.com"},"incidentId":14,"investigationState":"UnsupportedAlertType","status":"New","actorName":null,"classification":"FalsePositive","description":"Brent Murphy (brent@elasticbv.onmicrosoft.com) performed an activity. No activity was performed in United States in the past 41 days.","investigationId":null,"lastActivity":"2020-07-27T15:47:22.088Z","lastUpdatedTime":"2020-09-23T19:32:17.5433333Z","mitreTechniques":[],"serviceSource":"MicrosoftCloudAppSecurity","severity":"Medium","threatFamilyName":null,"title":"Activity from infrequent country","assignedTo":"elastic@elasticuser.com","detectionSource":"MCAS","devices":[],"alertId":"caA214771F-6AB0-311D-B2B0-BECD3B4A967B","creationTime":"2020-07-27T15:54:20.52207Z","determination":null,"firstActivity":"2020-07-27T15:47:22.088Z","resolvedTime":null},"classification":"Unknown","determination":"NotAvailable","lastUpdateTime":"2020-09-23T19:32:05.8366667Z","severity":"Medium","status":"Active","assignedTo":"elastic@elasticuser.com","createdTime":"2020-07-27T15:54:21.58Z"} diff --git a/x-pack/filebeat/module/microsoft/m365_defender/test/m365_defender-test.ndjson.log-expected.json b/x-pack/filebeat/module/microsoft/m365_defender/test/m365_defender-test.ndjson.log-expected.json index edd4b8ad091d..7091b8b456dd 100644 --- a/x-pack/filebeat/module/microsoft/m365_defender/test/m365_defender-test.ndjson.log-expected.json +++ b/x-pack/filebeat/module/microsoft/m365_defender/test/m365_defender-test.ndjson.log-expected.json @@ -521,8 +521,6 @@ "event.start": "2020-07-27T15:47:22.088Z", "event.timezone": "UTC", "fileset.name": "m365_defender", - "host.user.id": "8e24c50a-a77c-4782-813f-965009b5ddf3", - "host.user.name": "brent@elasticbv.onmicrosoft.com", "input.type": "log", "log.offset": 14764, "message": "Activity from infrequent country", @@ -530,8 +528,8 @@ "microsoft.m365_defender.alerts.classification": "FalsePositive", "microsoft.m365_defender.alerts.creationTime": "2020-07-27T15:54:20.52207Z", "microsoft.m365_defender.alerts.detectionSource": "MCAS", - "microsoft.m365_defender.alerts.entities.accountName": "brent", - "microsoft.m365_defender.alerts.entities.entityType": "User", + "microsoft.m365_defender.alerts.entities.entityType": "Ip", + "microsoft.m365_defender.alerts.entities.ipAddress": "73.172.171.53", "microsoft.m365_defender.alerts.incidentId": "14", "microsoft.m365_defender.alerts.investigationState": "UnsupportedAlertType", "microsoft.m365_defender.alerts.lastUpdatedTime": "2020-09-23T19:32:17.5433333Z", @@ -546,8 +544,8 @@ "observer.name": "MicrosoftCloudAppSecurity", "observer.product": "365 Defender", "observer.vendor": "Microsoft", - "related.user": [ - "brent@elasticbv.onmicrosoft.com" + "related.ip": [ + "73.172.171.53" ], "rule.description": "Brent Murphy (brent@elasticbv.onmicrosoft.com) performed an activity. No activity was performed in United States in the past 41 days.", "service.type": "microsoft", @@ -556,9 +554,7 @@ "forwarded" ], "threat.framework": "MITRE ATT&CK", - "threat.technique.name": "SuspiciousActivity", - "user.id": "8e24c50a-a77c-4782-813f-965009b5ddf3", - "user.name": "brent@elasticbv.onmicrosoft.com" + "threat.technique.name": "SuspiciousActivity" }, { "@timestamp": "2020-09-23T19:32:05.8366667Z", @@ -578,15 +574,17 @@ "event.start": "2020-07-27T15:47:22.088Z", "event.timezone": "UTC", "fileset.name": "m365_defender", + "host.user.id": "8e24c50a-a77c-4782-813f-965009b5ddf3", + "host.user.name": "brent@elasticbv.onmicrosoft.com", "input.type": "log", - "log.offset": 16091, + "log.offset": 15990, "message": "Activity from infrequent country", "microsoft.m365_defender.alerts.assignedTo": "elastic@elasticuser.com", "microsoft.m365_defender.alerts.classification": "FalsePositive", "microsoft.m365_defender.alerts.creationTime": "2020-07-27T15:54:20.52207Z", "microsoft.m365_defender.alerts.detectionSource": "MCAS", - "microsoft.m365_defender.alerts.entities.entityType": "Ip", - "microsoft.m365_defender.alerts.entities.ipAddress": "73.172.171.53", + "microsoft.m365_defender.alerts.entities.accountName": "brent", + "microsoft.m365_defender.alerts.entities.entityType": "User", "microsoft.m365_defender.alerts.incidentId": "14", "microsoft.m365_defender.alerts.investigationState": "UnsupportedAlertType", "microsoft.m365_defender.alerts.lastUpdatedTime": "2020-09-23T19:32:17.5433333Z", @@ -601,6 +599,9 @@ "observer.name": "MicrosoftCloudAppSecurity", "observer.product": "365 Defender", "observer.vendor": "Microsoft", + "related.user": [ + "brent@elasticbv.onmicrosoft.com" + ], "rule.description": "Brent Murphy (brent@elasticbv.onmicrosoft.com) performed an activity. No activity was performed in United States in the past 41 days.", "service.type": "microsoft", "tags": [ @@ -608,6 +609,8 @@ "forwarded" ], "threat.framework": "MITRE ATT&CK", - "threat.technique.name": "SuspiciousActivity" + "threat.technique.name": "SuspiciousActivity", + "user.id": "8e24c50a-a77c-4782-813f-965009b5ddf3", + "user.name": "brent@elasticbv.onmicrosoft.com" } ] \ No newline at end of file