Improve ECS categorization field mappings in misp module. (elastic#17344

)

- event.id
- event.kind
- registry.key if indicator type regkey
- rule.category
- rule.description
- rule.id
- rule.uuid
- user.name if indicator type github-username

Closes elastic#16026

(cherry picked from commit fee1f15)

CHANGELOG.next.asciidoc

@@ -331,6 +331,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Release Google Cloud module as GA. {pull}17511[17511]
 - Update filebeat httpjson input to support pagination via Header and Okta module. {pull}16354[16354]
 - Enhance `elasticsearch/server` fileset to handle ECS-compatible logs emitted by Elasticsearch. {issue}17715[17715] {pull}17714[17714]
+- Improve ECS categorization field mappings in misp module. {issue}16026[16026] {pull}17344[17344]
 
 *Heartbeat*
 

x-pack/filebeat/module/misp/threat/config/pipeline.js

@@ -13,6 +13,7 @@ var threat = (function () {
     var categorizeEvent = new processor.AddFields({
         target: "event",
         fields: {
+            kind: "event",
             category: "threat-intel",
             type: "indicator",
         },
@@ -24,8 +25,13 @@ var threat = (function () {
 
     var convertFields = new processor.Convert({
         fields: [
+            { from: "json.Event.id", to: "rule.id" },
             { from: "json.Event.info", to: "misp.threat_indicator.description" },
+            { from: "json.Event.info", to: "rule.description" },
             { from: "json.Event.uuid", to: "misp.threat_indicator.id" },
+            { from: "json.Event.uuid", to: "rule.uuid" },
+            { from: "json.category", to: "rule.category" },
+            { from: "json.uuid", to: "event.id" },
         ],
         mode: "rename",
         ignore_missing: true,
@@ -116,6 +122,7 @@ var threat = (function () {
             case 'github-username':
                 attackPattern = '[' + 'user:name = ' + '\'' + v + '\'' + ']';
                 attackPatternKQL = 'user.name: ' + '"' + v + '"';
+                evt.Put("user.name", v);
                 break;
             case "hostname":
                 attackPattern = '[' + 'source:domain = ' + '\'' + v + '\'' + ' OR destination:domain = ' + '\'' + v + '\'' + ']';
@@ -155,6 +162,7 @@ var threat = (function () {
             case 'regkey':
                 attackPattern = '[' + 'regkey = ' + '\'' + v + '\'' + ']';
                 attackPatternKQL = 'regkey: ' + '"' + v + '"';
+                evt.Put("registry.key", v);
                 break;
             case "sha1":
                 attackPattern = '[' + 'file:hash:sha1 = ' + '\'' + v + '\'' + ']';

x-pack/filebeat/module/misp/threat/test/misp-test.json.log-expected.json

@@ -11,6 +11,8 @@
         "destination.ip": "98.235.162.24",
         "event.category": "threat-intel",
         "event.dataset": "misp.threat",
+        "event.id": "5d2cb906-eff4-40f0-9f1d-10eb7d6a0c26",
+        "event.kind": "event",
         "event.module": "misp",
         "event.type": "indicator",
         "fileset.name": "threat",
@@ -23,12 +25,18 @@
         "misp.threat_indicator.feed": "misp",
         "misp.threat_indicator.id": "58dcfe62-ed84-4e5e-b293-4991950d210f",
         "misp.threat_indicator.type": "ip-dst",
+        "rule.category": "Network activity",
+        "rule.description": "Tor exit nodes feed",
+        "rule.id": "1",
+        "rule.uuid": "58dcfe62-ed84-4e5e-b293-4991950d210f",
         "service.type": "misp"
     },
     {
         "@timestamp": "2017-03-30T12:54:26.000Z",
         "event.category": "threat-intel",
         "event.dataset": "misp.threat",
+        "event.id": "5d159be2-d4b4-4d97-9e14-406a02de0b81",
+        "event.kind": "event",
         "event.module": "misp",
         "event.type": "indicator",
         "file.hash.md5": "89357a1b2e32f2b9bddff94b8136810b",
@@ -42,12 +50,18 @@
         "misp.threat_indicator.feed": "misp",
         "misp.threat_indicator.id": "5d159be2-d4b4-4d97-9e14-406a02de0b81",
         "misp.threat_indicator.type": "md5",
+        "rule.category": "Payload delivery",
+        "rule.description": "OSINT - OSX/Linker: New Mac malware attempts zero-day Gatekeeper bypass",
+        "rule.id": "1",
+        "rule.uuid": "5d159be2-d4b4-4d97-9e14-406a02de0b81",
         "service.type": "misp"
     },
     {
         "@timestamp": "2017-03-30T12:54:26.000Z",
         "event.category": "threat-intel",
         "event.dataset": "misp.threat",
+        "event.id": "5d159be2-d4b4-4d97-9e14-406a02de0b81",
+        "event.kind": "event",
         "event.module": "misp",
         "event.type": "indicator",
         "file.path": "f6bf5b8bb2400aad4ac844f2b94a4e556907f35b44c5ff462fb4e70c0208c9de",
@@ -61,12 +75,18 @@
         "misp.threat_indicator.feed": "misp",
         "misp.threat_indicator.id": "5d159be2-d4b4-4d97-9e14-406a02de0b81",
         "misp.threat_indicator.type": "filename",
+        "rule.category": "Payload delivery",
+        "rule.description": "OSINT - OSX/Linker: New Mac malware attempts zero-day Gatekeeper bypass",
+        "rule.id": "1",
+        "rule.uuid": "5d159be2-d4b4-4d97-9e14-406a02de0b81",
         "service.type": "misp"
     },
     {
         "@timestamp": "2017-03-30T12:54:26.000Z",
         "event.category": "threat-intel",
         "event.dataset": "misp.threat",
+        "event.id": "563b3ea6-b26c-401f-a68b-4d84950d210b",
+        "event.kind": "event",
         "event.module": "misp",
         "event.type": "indicator",
         "fileset.name": "threat",
@@ -79,6 +99,10 @@
         "misp.threat_indicator.feed": "misp",
         "misp.threat_indicator.id": "563b3ea6-b26c-401f-a68b-4d84950d210b",
         "misp.threat_indicator.type": "domain",
+        "rule.category": "Bad Domain",
+        "rule.description": "OSINT Expansion on Systematic cyber attacks against Israeli and Palestinian targets going on for a year by Norman",
+        "rule.id": "4",
+        "rule.uuid": "563b3ea6-b26c-401f-a68b-4d84950d210b",
         "service.type": "misp"
     }
 ]