Cherry-pick elastic#25421 to 7.12: [Filebeat] Allow m365 defender to …

…receive incidents with no alerts (elastic#25485) * [Filebeat] Allow m365 defender to receive incidents with no alerts (elastic#25421) * allowing incidents with no alerts to be parsed, updating some typos and making the pipeline safer in terms of null def references * adding changelog entry * updating local test files * reverting changes to json decoding, back to beat (cherry picked from commit 34837f5) * Update CHANGELOG.next.asciidoc
leweafan · May 3, 2021 · af12c26 · af12c26
1 parent 5db5113
commit af12c26
Show file tree

Hide file tree

Showing 6 changed files with 75 additions and 27 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -125,6 +125,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - system/package: Fix an error that can occur while trying to persist package metadata. {issue}18536[18536] {pull}18887[18887]
 - system/socket: Fix dataset using 100% CPU and becoming unresponsive in some scenarios. {pull}19033[19033] {pull}19764[19764]
 - system/socket: Fixed tracking of long-running connections. {pull}19033[19033]
+- Fix issue with m365_defender, when parsing incidents that has no alerts attached: {pull}25421[25421]
 
 *Filebeat*
 

diff --git a/x-pack/filebeat/module/microsoft/m365_defender/ingest/pipeline.yml b/x-pack/filebeat/module/microsoft/m365_defender/ingest/pipeline.yml
@@ -10,6 +10,11 @@ processors:
       - json.comments
       - host
     ignore_missing: true
+- remove:
+    field:
+      - json.alerts
+    ignore_missing: true
+    if: 'ctx?.json?.alerts == null || ctx?.json?.alerts instanceof List'
 
 #########################
 ## ECS General Mapping ##
@@ -45,11 +50,16 @@ processors:
 - set:
     field: '@timestamp'
     value: '{{json.lastUpdateTime}}'
-    if: ctx.json?.lastUpdateTime != null
+    if: ctx?.json?.lastUpdateTime != null
 - rename:
     field: json.alerts.title
     target_field: message
     ignore_missing: true
+- rename:
+    field: json.incidentName
+    target_field: message
+    ignore_missing: true
+    if: ctx?.message == null
 
 #######################
 ## ECS Event Mapping ##
@@ -144,7 +154,7 @@ processors:
 - set:
     field: threat.framework
     value: MITRE ATT&CK
-    if: ctx.json?.alerts?.category != null
+    if: ctx?.json?.alerts?.category != null
 - rename:
     field: json.alerts.category
     target_field: threat.technique.name
@@ -153,7 +163,7 @@ processors:
     field: json.alerts.description
     target_field: rule.description
     ignore_missing: true
-    if: ctx.json?.alerts?.description.length() < 1020
+    if: ctx?.json?.alerts != null && ctx.json?.alerts?.description.length() < 1020
 
 ######################
 ## ECS File Mapping ##
@@ -220,8 +230,11 @@ processors:
     field: json.alerts.entities.url
     target_field: url.full
     ignore_missing: true
-    if: ctx?.json?.entities?.url != null
-
+    if: ctx?.json?.alerts?.entities?.url != null
+- uri_parts:
+    field: url.full
+    ignore_failure: true
+    if: ctx?.url?.full != null
 ######################
 ## ECS User Mapping ##
 ######################
@@ -261,23 +274,23 @@ processors:
 - append:
     field: related.ip
     value: '{{json.alerts.entities.ipAddress}}'
-    if: ctx.json?.entities?.ipAddress != null
+    if: ctx?.json?.alerts?.entities?.ipAddress != null
 - append:
     field: related.user
     value: '{{user.name}}'
-    if: ctx.user?.name != null
+    if: ctx?.user?.name != null
 - append:
     field: related.hash
     value: '{{file.hash.sha1}}'
-    if: ctx.file?.hash?.sha1 != null
+    if: ctx?.file?.hash?.sha1 != null
 - append:
     field: related.hash
     value: '{{file.hash.sha256}}'
-    if: ctx.file?.hash?.sha256 != null
+    if: ctx?.file?.hash?.sha256 != null
 - append:
     field: related.hosts
     value: '{{host.hostname}}'
-    if: ctx.host?.hostname != null
+    if: ctx?.host?.hostname != null
 
 #############
 ## Cleanup ##
@@ -293,15 +306,15 @@ processors:
 - remove:
     field: json.alerts.mitreTechniques
     ignore_missing: true
-    if: ctx?.json?.alerts?.mitreTechniques.isEmpty()
+    if: 'ctx?.json?.alerts != null && ctx?.json?.alerts?.mitreTechniques.isEmpty()'
 - remove:
     field: json.alerts.devices
     ignore_missing: true
-    if: ctx?.json?.alerts?.devices.isEmpty()
+    if: 'ctx?.json?.alerts != null && ctx?.json?.alerts?.devices.isEmpty()'
 - remove:
     field: json.tags
     ignore_missing: true
-    if: ctx?.json?.tags.isEmpty()
+    if: 'ctx?.json?.alerts != null && ctx?.json?.tags.isEmpty()'
 - remove:
     ignore_missing: true
     field:

diff --git a/x-pack/filebeat/module/microsoft/m365_defender/test/m365_defender-test-empty.ndjson.log b/x-pack/filebeat/module/microsoft/m365_defender/test/m365_defender-test-empty.ndjson.log
@@ -0,0 +1 @@
+{"incidentId":1111,"redirectIncidentId":1107,"incidentName":"Impossible travel activity involving one user","createdTime":"2021-04-12T11:18:28.86Z","lastUpdateTime":"2021-04-12T11:18:30.4033333Z","assignedTo":null,"classification":"Unknown","determination":"NotAvailable","status":"Redirected","severity":"UnSpecified","tags":[],"comments":[],"alerts":[]}
diff --git a/...eat/module/microsoft/m365_defender/test/m365_defender-test-empty.ndjson.log-expected.json b/...eat/module/microsoft/m365_defender/test/m365_defender-test-empty.ndjson.log-expected.json
@@ -0,0 +1,30 @@
+[
+    {
+        "@timestamp": "2021-04-12T11:18:30.4033333Z",
+        "cloud.provider": "azure",
+        "event.category": [
+            "host"
+        ],
+        "event.dataset": "microsoft.m365_defender",
+        "event.kind": "alert",
+        "event.module": "microsoft",
+        "event.timezone": "UTC",
+        "fileset.name": "m365_defender",
+        "input.type": "log",
+        "log.offset": 0,
+        "message": "Impossible travel activity involving one user",
+        "microsoft.m365_defender.classification": "Unknown",
+        "microsoft.m365_defender.determination": "NotAvailable",
+        "microsoft.m365_defender.incidentId": "1111",
+        "microsoft.m365_defender.redirectIncidentId": 1107,
+        "microsoft.m365_defender.status": "Redirected",
+        "microsoft.m365_defender.tags": [],
+        "observer.product": "365 Defender",
+        "observer.vendor": "Microsoft",
+        "service.type": "microsoft",
+        "tags": [
+            "m365-defender",
+            "forwarded"
+        ]
+    }
+]
diff --git a/x-pack/filebeat/module/microsoft/m365_defender/test/m365_defender-test.ndjson.log b/x-pack/filebeat/module/microsoft/m365_defender/test/m365_defender-test.ndjson.log
@@ -5,5 +5,5 @@
 {"assignedTo":"[email protected]","classification":"Unknown","createdTime":"2020-06-30T09:32:31.85Z","redirectIncidentId":null,"severity":"Low","status":"Resolved","tags":[],"alerts":{"assignedTo":"[email protected]","determination":null,"serviceSource":"MicrosoftDefenderATP","severity":"Low","alertId":"da637291086161511365_-2075772905","classification":"FalsePositive","creationTime":"2020-06-30T10:10:16.1355657Z","description":"Malware and unwanted software are undesirable applications that perform annoying, disruptive, or harmful actions on affected machines. Some of these undesirable applications can replicate and spread from one machine to another. Others are able to receive commands from remote attackers and perform activities associated with cyber attacks.\n\nA malware is considered active if it is found running on the machine or it already has persistence mechanisms in place. Active malware detections are assigned higher severity ratings.\n\nBecause this malware was active, take precautionary measures and check for residual signs of infection.","entities":{"deviceId":"75a63a39f9bc5a964f417c11f6277d5bf9489f0d","entityType":"Process","processCreationTime":"2020-06-30T10:31:04.1092404Z","processId":6720},"mitreTechniques":[],"title":"Suspicious 'AccessibilityEscalation' behavior was detected","category":"SuspiciousActivity","devices":[{"aadDeviceId":null,"mdatpDeviceId":"75a63a39f9bc5a964f417c11f6277d5bf9489f0d","osProcessor":"x64","riskScore":"High","osPlatform":"Other","rbacGroupId":0,"rbacGroupName":null,"version":"Other","deviceDnsName":"testserver4","firstSeen":"2020-06-30T08:55:08.8320449Z","healthStatus":"Inactive","osBuild":17763}],"firstActivity":"2020-06-30T10:09:10.8889583Z","investigationState":"UnsupportedAlertType","status":"Resolved","detectionSource":"WindowsDefenderAv","incidentId":12,"investigationId":null,"lastActivity":"2020-06-30T10:31:09.4165785Z","lastUpdatedTime":"2020-09-23T19:44:37.9666667Z","resolvedTime":"2020-09-23T19:44:36.1092821Z","threatFamilyName":null,"actorName":null},"determination":"NotAvailable","incidentId":12,"incidentName":"12","lastUpdateTime":"2020-09-23T19:44:36.29Z"}
 {"determination":"NotAvailable","severity":"Low","classification":"Unknown","createdTime":"2020-06-30T09:32:31.85Z","incidentId":12,"incidentName":"12","lastUpdateTime":"2020-09-23T19:44:36.29Z","redirectIncidentId":null,"alerts":{"lastActivity":"2020-06-30T10:31:09.4165785Z","lastUpdatedTime":"2020-09-23T19:44:37.9666667Z","actorName":null,"description":"Malware and unwanted software are undesirable applications that perform annoying, disruptive, or harmful actions on affected machines. Some of these undesirable applications can replicate and spread from one machine to another. Others are able to receive commands from remote attackers and perform activities associated with cyber attacks.\n\nA malware is considered active if it is found running on the machine or it already has persistence mechanisms in place. Active malware detections are assigned higher severity ratings.\n\nBecause this malware was active, take precautionary measures and check for residual signs of infection.","determination":null,"entities":{"accountName":"","entityType":"User"},"firstActivity":"2020-06-30T10:09:10.8889583Z","investigationState":"UnsupportedAlertType","serviceSource":"MicrosoftDefenderATP","status":"Resolved","title":"Suspicious 'AccessibilityEscalation' behavior was detected","classification":"FalsePositive","devices":[{"aadDeviceId":null,"healthStatus":"Inactive","osPlatform":"Other","osProcessor":"x64","riskScore":"High","deviceDnsName":"testserver4","firstSeen":"2020-06-30T08:55:08.8320449Z","mdatpDeviceId":"75a63a39f9bc5a964f417c11f6277d5bf9489f0d","osBuild":17763,"rbacGroupId":0,"rbacGroupName":null,"version":"Other"}],"mitreTechniques":[],"severity":"Low","threatFamilyName":null,"creationTime":"2020-06-30T10:10:16.1355657Z","detectionSource":"WindowsDefenderAv","incidentId":12,"alertId":"da637291086161511365_-2075772905","assignedTo":"[email protected]","category":"SuspiciousActivity","investigationId":null,"resolvedTime":"2020-09-23T19:44:36.1092821Z"},"assignedTo":"[email protected]","status":"Resolved","tags":[]}
 {"determination":"NotAvailable","lastUpdateTime":"2020-09-23T19:44:36.29Z","tags":[],"alerts":{"investigationState":"UnsupportedAlertType","status":"Resolved","alertId":"da637291086161511365_-2075772905","assignedTo":"[email protected]","determination":null,"firstActivity":"2020-06-30T10:09:10.8889583Z","mitreTechniques":[],"resolvedTime":"2020-09-23T19:44:36.1092821Z","severity":"Low","actorName":null,"category":"SuspiciousActivity","description":"Malware and unwanted software are undesirable applications that perform annoying, disruptive, or harmful actions on affected machines. Some of these undesirable applications can replicate and spread from one machine to another. Others are able to receive commands from remote attackers and perform activities associated with cyber attacks.\n\nA malware is considered active if it is found running on the machine or it already has persistence mechanisms in place. Active malware detections are assigned higher severity ratings.\n\nBecause this malware was active, take precautionary measures and check for residual signs of infection.","lastUpdatedTime":"2020-09-23T19:44:37.9666667Z","title":"Suspicious 'AccessibilityEscalation' behavior was detected","classification":"FalsePositive","creationTime":"2020-06-30T10:10:16.1355657Z","entities":{"deviceId":"75a63a39f9bc5a964f417c11f6277d5bf9489f0d","entityType":"Process","processCreationTime":"2020-06-30T10:09:10.5747992Z","processId":1324},"incidentId":12,"serviceSource":"MicrosoftDefenderATP","threatFamilyName":null,"detectionSource":"WindowsDefenderAv","devices":[{"osPlatform":"Other","osProcessor":"x64","rbacGroupId":0,"riskScore":"High","version":"Other","aadDeviceId":null,"deviceDnsName":"testserver4","mdatpDeviceId":"75a63a39f9bc5a964f417c11f6277d5bf9489f0d","rbacGroupName":null,"firstSeen":"2020-06-30T08:55:08.8320449Z","healthStatus":"Inactive","osBuild":17763}],"investigationId":null,"lastActivity":"2020-06-30T10:31:09.4165785Z"},"assignedTo":"[email protected]","classification":"Unknown","createdTime":"2020-06-30T09:32:31.85Z","status":"Resolved","incidentId":12,"incidentName":"12","redirectIncidentId":null,"severity":"Low"}
-{"incidentId":14,"incidentName":"Activity from infrequent country","redirectIncidentId":null,"tags":[],"alerts":{"category":"SuspiciousActivity","entities":{"aadUserId":"8e24c50a-a77c-4782-813f-965009b5ddf3","accountName":"brent","entityType":"User","userPrincipalName":"[email protected]"},"incidentId":14,"investigationState":"UnsupportedAlertType","status":"New","actorName":null,"classification":"FalsePositive","description":"Brent Murphy ([email protected]) performed an activity. No activity was performed in United States in the past 41 days.","investigationId":null,"lastActivity":"2020-07-27T15:47:22.088Z","lastUpdatedTime":"2020-09-23T19:32:17.5433333Z","mitreTechniques":[],"serviceSource":"MicrosoftCloudAppSecurity","severity":"Medium","threatFamilyName":null,"title":"Activity from infrequent country","assignedTo":"[email protected]","detectionSource":"MCAS","devices":[],"alertId":"caA214771F-6AB0-311D-B2B0-BECD3B4A967B","creationTime":"2020-07-27T15:54:20.52207Z","determination":null,"firstActivity":"2020-07-27T15:47:22.088Z","resolvedTime":null},"classification":"Unknown","determination":"NotAvailable","lastUpdateTime":"2020-09-23T19:32:05.8366667Z","severity":"Medium","status":"Active","assignedTo":"[email protected]","createdTime":"2020-07-27T15:54:21.58Z"}
 {"incidentId":14,"incidentName":"Activity from infrequent country","severity":"Medium","status":"Active","tags":[],"alerts":{"description":"Brent Murphy ([email protected]) performed an activity. No activity was performed in United States in the past 41 days.","detectionSource":"MCAS","firstActivity":"2020-07-27T15:47:22.088Z","investigationId":null,"investigationState":"UnsupportedAlertType","severity":"Medium","alertId":"caA214771F-6AB0-311D-B2B0-BECD3B4A967B","category":"SuspiciousActivity","classification":"FalsePositive","determination":null,"entities":{"entityType":"Ip","ipAddress":"73.172.171.53"},"incidentId":14,"serviceSource":"MicrosoftCloudAppSecurity","status":"New","actorName":null,"title":"Activity from infrequent country","devices":[],"lastActivity":"2020-07-27T15:47:22.088Z","lastUpdatedTime":"2020-09-23T19:32:17.5433333Z","creationTime":"2020-07-27T15:54:20.52207Z","mitreTechniques":[],"resolvedTime":null,"threatFamilyName":null,"assignedTo":"[email protected]"},"createdTime":"2020-07-27T15:54:21.58Z","determination":"NotAvailable","lastUpdateTime":"2020-09-23T19:32:05.8366667Z","redirectIncidentId":null,"assignedTo":"[email protected]","classification":"Unknown"}
+{"incidentId":14,"incidentName":"Activity from infrequent country","redirectIncidentId":null,"tags":[],"alerts":{"category":"SuspiciousActivity","entities":{"aadUserId":"8e24c50a-a77c-4782-813f-965009b5ddf3","accountName":"brent","entityType":"User","userPrincipalName":"[email protected]"},"incidentId":14,"investigationState":"UnsupportedAlertType","status":"New","actorName":null,"classification":"FalsePositive","description":"Brent Murphy ([email protected]) performed an activity. No activity was performed in United States in the past 41 days.","investigationId":null,"lastActivity":"2020-07-27T15:47:22.088Z","lastUpdatedTime":"2020-09-23T19:32:17.5433333Z","mitreTechniques":[],"serviceSource":"MicrosoftCloudAppSecurity","severity":"Medium","threatFamilyName":null,"title":"Activity from infrequent country","assignedTo":"[email protected]","detectionSource":"MCAS","devices":[],"alertId":"caA214771F-6AB0-311D-B2B0-BECD3B4A967B","creationTime":"2020-07-27T15:54:20.52207Z","determination":null,"firstActivity":"2020-07-27T15:47:22.088Z","resolvedTime":null},"classification":"Unknown","determination":"NotAvailable","lastUpdateTime":"2020-09-23T19:32:05.8366667Z","severity":"Medium","status":"Active","assignedTo":"[email protected]","createdTime":"2020-07-27T15:54:21.58Z"}
diff --git a/.../filebeat/module/microsoft/m365_defender/test/m365_defender-test.ndjson.log-expected.json b/.../filebeat/module/microsoft/m365_defender/test/m365_defender-test.ndjson.log-expected.json
@@ -521,17 +521,15 @@
         "event.start": "2020-07-27T15:47:22.088Z",
         "event.timezone": "UTC",
         "fileset.name": "m365_defender",
-        "host.user.id": "8e24c50a-a77c-4782-813f-965009b5ddf3",
-        "host.user.name": "[email protected]",
         "input.type": "log",
         "log.offset": 14764,
         "message": "Activity from infrequent country",
         "microsoft.m365_defender.alerts.assignedTo": "[email protected]",
         "microsoft.m365_defender.alerts.classification": "FalsePositive",
         "microsoft.m365_defender.alerts.creationTime": "2020-07-27T15:54:20.52207Z",
         "microsoft.m365_defender.alerts.detectionSource": "MCAS",
-        "microsoft.m365_defender.alerts.entities.accountName": "brent",
-        "microsoft.m365_defender.alerts.entities.entityType": "User",
+        "microsoft.m365_defender.alerts.entities.entityType": "Ip",
+        "microsoft.m365_defender.alerts.entities.ipAddress": "73.172.171.53",
         "microsoft.m365_defender.alerts.incidentId": "14",
         "microsoft.m365_defender.alerts.investigationState": "UnsupportedAlertType",
         "microsoft.m365_defender.alerts.lastUpdatedTime": "2020-09-23T19:32:17.5433333Z",
@@ -546,8 +544,8 @@
         "observer.name": "MicrosoftCloudAppSecurity",
         "observer.product": "365 Defender",
         "observer.vendor": "Microsoft",
-        "related.user": [
-            "[email protected]"
+        "related.ip": [
+            "73.172.171.53"
         ],
         "rule.description": "Brent Murphy ([email protected]) performed an activity. No activity was performed in United States in the past 41 days.",
         "service.type": "microsoft",
@@ -556,9 +554,7 @@
             "forwarded"
         ],
         "threat.framework": "MITRE ATT&CK",
-        "threat.technique.name": "SuspiciousActivity",
-        "user.id": "8e24c50a-a77c-4782-813f-965009b5ddf3",
-        "user.name": "[email protected]"
+        "threat.technique.name": "SuspiciousActivity"
     },
     {
         "@timestamp": "2020-09-23T19:32:05.8366667Z",
@@ -578,15 +574,17 @@
         "event.start": "2020-07-27T15:47:22.088Z",
         "event.timezone": "UTC",
         "fileset.name": "m365_defender",
+        "host.user.id": "8e24c50a-a77c-4782-813f-965009b5ddf3",
+        "host.user.name": "[email protected]",
         "input.type": "log",
-        "log.offset": 16091,
+        "log.offset": 15990,
         "message": "Activity from infrequent country",
         "microsoft.m365_defender.alerts.assignedTo": "[email protected]",
         "microsoft.m365_defender.alerts.classification": "FalsePositive",
         "microsoft.m365_defender.alerts.creationTime": "2020-07-27T15:54:20.52207Z",
         "microsoft.m365_defender.alerts.detectionSource": "MCAS",
-        "microsoft.m365_defender.alerts.entities.entityType": "Ip",
-        "microsoft.m365_defender.alerts.entities.ipAddress": "73.172.171.53",
+        "microsoft.m365_defender.alerts.entities.accountName": "brent",
+        "microsoft.m365_defender.alerts.entities.entityType": "User",
         "microsoft.m365_defender.alerts.incidentId": "14",
         "microsoft.m365_defender.alerts.investigationState": "UnsupportedAlertType",
         "microsoft.m365_defender.alerts.lastUpdatedTime": "2020-09-23T19:32:17.5433333Z",
@@ -601,13 +599,18 @@
         "observer.name": "MicrosoftCloudAppSecurity",
         "observer.product": "365 Defender",
         "observer.vendor": "Microsoft",
+        "related.user": [
+            "[email protected]"
+        ],
         "rule.description": "Brent Murphy ([email protected]) performed an activity. No activity was performed in United States in the past 41 days.",
         "service.type": "microsoft",
         "tags": [
             "m365-defender",
             "forwarded"
         ],
         "threat.framework": "MITRE ATT&CK",
-        "threat.technique.name": "SuspiciousActivity"
+        "threat.technique.name": "SuspiciousActivity",
+        "user.id": "8e24c50a-a77c-4782-813f-965009b5ddf3",
+        "user.name": "[email protected]"
     }
 ]
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		{"incidentId":1111,"redirectIncidentId":1107,"incidentName":"Impossible travel activity involving one user","createdTime":"2021-04-12T11:18:28.86Z","lastUpdateTime":"2021-04-12T11:18:30.4033333Z","assignedTo":null,"classification":"Unknown","determination":"NotAvailable","status":"Redirected","severity":"UnSpecified","tags":[],"comments":[],"alerts":[]}