Improve ECS field mappings for CEF module (elastic#16338)

- related.hash - related.ip - related.user - fix description Closes elastic#16157 Closes elastic#16289
leehinman · Mar 12, 2020 · 3e6edf2 · 3e6edf2
1 parent 3c13de5
commit 3e6edf2
Show file tree

Hide file tree

Showing 5 changed files with 139 additions and 1 deletion.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -178,6 +178,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Improve ECS categorization field mapping in icinga module. {issue}16164[16164] {pull}16533[16533]
 - Improve ECS categorization field mappings in ibmmq module. {issue}16163[16163] {pull}16532[16532]
 - Improve ECS categorization, host field mappings in elasticsearch module. {issue}16160[16160] {pull}16469[16469]
+- Add ECS related fields to CEF module {issue}16157[16157] {pull}16338[16338]
 
 *Heartbeat*
 

diff --git a/x-pack/filebeat/module/cef/log/ingest/pipeline.yml b/x-pack/filebeat/module/cef/log/ingest/pipeline.yml
@@ -1,5 +1,5 @@
 ---
-description: Pipeline for Filebeat NetFlow
+description: Pipeline for Filebeat CEF
 
 processors:
   # IP Geolocation Lookup
@@ -45,6 +45,38 @@ processors:
         field: destination.as.organization_name
         target_field: destination.as.organization.name
         ignore_missing: true
+  - append:
+        field: related.hash
+        value: "{{cef.extensions.fileHash}}"
+        if: "ctx?.cef?.extensions?.fileHash != null"
+  - append:
+        field: related.hash
+        value: "{{cef.extensions.oldFileHash}}"
+        if: "ctx?.cef?.extensions?.oldFileHash != null"
+  - append:
+        field: related.ip
+        value: "{{destination.ip}}"
+        if: "ctx?.destination?.ip != null"
+  - append:
+        field: related.ip
+        value: "{{destination.nat.ip}}"
+        if: "ctx?.destination?.nat?.ip != null"
+  - append:
+        field: related.ip
+        value: "{{source.ip}}"
+        if: "ctx?.source?.ip != null"
+  - append:
+        field: related.ip
+        value: "{{source.nat.ip}}"
+        if: "ctx?.source?.nat?.ip != null"
+  - append:
+        field: related.user
+        value: "{{destination.user.name}}"
+        if: "ctx?.destination?.user?.name != null"
+  - append:
+        field: related.user
+        value: "{{source.user.name}}"
+        if: "ctx?.source?.user?.name != null"
   - pipeline:
         name: '{< IngestPipeline "fp-pipeline" >}'
         if: "ctx.cef?.device?.vendor == 'FORCEPOINT'"

diff --git a/x-pack/filebeat/module/cef/log/test/cef.log b/x-pack/filebeat/module/cef/log/test/cef.log
@@ -1 +1,2 @@
 CEF:0|Elastic|Vaporware|1.0.0-alpha|18|Web request|low|eventId=3457 requestMethod=POST slat=38.915 slong=-77.511 proto=TCP sourceServiceName=httpd requestContext=https://www.google.com src=6.7.8.9 spt=33876 dst=192.168.10.1 dpt=443 request=https://www.example.com/cart
+CEF:0|Elastic|Vaporware|1.0.0-alpha|18|Authentication|low|eventId=123 src=6.7.8.9 spt=33876 dst=1.2.3.4 dpt=443 duser=alice suser=bob destinationTranslatedAddress=10.10.10.10 fileHash=bc8bbe52f041fd17318f08a0f73762ce oldFileHash=a9796280592f86b74b27e370662d41eb
diff --git a/x-pack/filebeat/module/cef/log/test/cef.log-expected.json b/x-pack/filebeat/module/cef/log/test/cef.log-expected.json
@@ -38,6 +38,10 @@
         "observer.product": "Vaporware",
         "observer.vendor": "Elastic",
         "observer.version": "1.0.0-alpha",
+        "related.ip": [
+            "192.168.10.1",
+            "6.7.8.9"
+        ],
         "service.type": "cef",
         "source.geo.continent_name": "North America",
         "source.geo.country_iso_code": "US",
@@ -50,5 +54,72 @@
             "cef"
         ],
         "url.original": "https://www.example.com/cart"
+    },
+    {
+        "cef.device.event_class_id": "18",
+        "cef.device.product": "Vaporware",
+        "cef.device.vendor": "Elastic",
+        "cef.device.version": "1.0.0-alpha",
+        "cef.extensions.destinationAddress": "1.2.3.4",
+        "cef.extensions.destinationPort": 443,
+        "cef.extensions.destinationTranslatedAddress": "10.10.10.10",
+        "cef.extensions.destinationUserName": "alice",
+        "cef.extensions.eventId": 123,
+        "cef.extensions.fileHash": "bc8bbe52f041fd17318f08a0f73762ce",
+        "cef.extensions.oldFileHash": "a9796280592f86b74b27e370662d41eb",
+        "cef.extensions.sourceAddress": "6.7.8.9",
+        "cef.extensions.sourcePort": 33876,
+        "cef.extensions.sourceUserName": "bob",
+        "cef.name": "Authentication",
+        "cef.severity": "low",
+        "cef.version": "0",
+        "destination.geo.city_name": "Moscow",
+        "destination.geo.continent_name": "Europe",
+        "destination.geo.country_iso_code": "RU",
+        "destination.geo.location.lat": 55.7527,
+        "destination.geo.location.lon": 37.6172,
+        "destination.geo.region_iso_code": "RU-MOW",
+        "destination.geo.region_name": "Moscow",
+        "destination.ip": "1.2.3.4",
+        "destination.nat.ip": "10.10.10.10",
+        "destination.port": 443,
+        "destination.user.name": "alice",
+        "event.code": "18",
+        "event.dataset": "cef.log",
+        "event.id": 123,
+        "event.module": "cef",
+        "event.original": "CEF:0|Elastic|Vaporware|1.0.0-alpha|18|Authentication|low|eventId=123 src=6.7.8.9 spt=33876 dst=1.2.3.4 dpt=443 duser=alice suser=bob destinationTranslatedAddress=10.10.10.10 fileHash=bc8bbe52f041fd17318f08a0f73762ce oldFileHash=a9796280592f86b74b27e370662d41eb",
+        "event.severity": 0,
+        "fileset.name": "log",
+        "input.type": "log",
+        "log.offset": 269,
+        "message": "Authentication",
+        "observer.product": "Vaporware",
+        "observer.vendor": "Elastic",
+        "observer.version": "1.0.0-alpha",
+        "related.hash": [
+            "bc8bbe52f041fd17318f08a0f73762ce",
+            "a9796280592f86b74b27e370662d41eb"
+        ],
+        "related.ip": [
+            "1.2.3.4",
+            "10.10.10.10",
+            "6.7.8.9"
+        ],
+        "related.user": [
+            "alice",
+            "bob"
+        ],
+        "service.type": "cef",
+        "source.geo.continent_name": "North America",
+        "source.geo.country_iso_code": "US",
+        "source.geo.location.lat": 37.751,
+        "source.geo.location.lon": -97.822,
+        "source.ip": "6.7.8.9",
+        "source.port": 33876,
+        "source.user.name": "bob",
+        "tags": [
+            "cef"
+        ]
     }
 ]
diff --git a/x-pack/filebeat/module/cef/log/test/fp-ngfw-smc.log-expected.json b/x-pack/filebeat/module/cef/log/test/fp-ngfw-smc.log-expected.json
@@ -106,6 +106,10 @@
         "observer.product": "Firewall",
         "observer.vendor": "FORCEPOINT",
         "observer.version": "6.6.1",
+        "related.ip": [
+            "10.1.1.40",
+            "10.37.205.252"
+        ],
         "rule.id": "2097157.1",
         "service.type": "cef",
         "source.ip": "10.37.205.252",
@@ -154,6 +158,10 @@
         "observer.product": "Firewall",
         "observer.vendor": "FORCEPOINT",
         "observer.version": "unknown",
+        "related.ip": [
+            "255.255.255.255",
+            "172.16.1.1"
+        ],
         "rule.id": "605.0",
         "service.type": "cef",
         "source.ip": "172.16.1.1",
@@ -202,6 +210,10 @@
         "observer.product": "Firewall",
         "observer.vendor": "FORCEPOINT",
         "observer.version": "unknown",
+        "related.ip": [
+            "192.168.1.1",
+            "172.16.1.1"
+        ],
         "rule.id": "601.0",
         "service.type": "cef",
         "source.ip": "172.16.1.1",
@@ -247,6 +259,9 @@
         "observer.product": "Firewall",
         "observer.vendor": "FORCEPOINT",
         "observer.version": "unknown",
+        "related.user": [
+            "alice"
+        ],
         "service.type": "cef",
         "source.bytes": 32526,
         "source.user.name": "alice",
@@ -283,6 +298,12 @@
         "observer.product": "Firewall",
         "observer.vendor": "FORCEPOINT",
         "observer.version": "unknown",
+        "related.ip": [
+            "192.168.1.1"
+        ],
+        "related.user": [
+            "bob"
+        ],
         "service.type": "cef",
         "source.ip": "192.168.1.1",
         "source.user.name": "bob",
@@ -319,6 +340,12 @@
         "observer.product": "Firewall",
         "observer.vendor": "FORCEPOINT",
         "observer.version": "unknown",
+        "related.ip": [
+            "192.168.1.1"
+        ],
+        "related.user": [
+            "bob"
+        ],
         "service.type": "cef",
         "source.ip": "192.168.1.1",
         "source.user.name": "bob",
@@ -355,6 +382,12 @@
         "observer.product": "Firewall",
         "observer.vendor": "FORCEPOINT",
         "observer.version": "unknown",
+        "related.ip": [
+            "172.16.2.1"
+        ],
+        "related.user": [
+            "alice"
+        ],
         "service.type": "cef",
         "source.ip": "172.16.2.1",
         "source.user.name": "alice",
Original file line number	Diff line number	Diff line change
		@@ -1 +1,2 @@
		CEF:0\|Elastic\|Vaporware\|1.0.0-alpha\|18\|Web request\|low\|eventId=3457 requestMethod=POST slat=38.915 slong=-77.511 proto=TCP sourceServiceName=httpd requestContext=https://www.google.com src=6.7.8.9 spt=33876 dst=192.168.10.1 dpt=443 request=https://www.example.com/cart
		CEF:0\|Elastic\|Vaporware\|1.0.0-alpha\|18\|Authentication\|low\|eventId=123 src=6.7.8.9 spt=33876 dst=1.2.3.4 dpt=443 duser=alice suser=bob destinationTranslatedAddress=10.10.10.10 fileHash=bc8bbe52f041fd17318f08a0f73762ce oldFileHash=a9796280592f86b74b27e370662d41eb