[Filebeat] Improve ECS categorization field mappings in googlecloud m…

…odule (elastic#16500) + audit - event.id - event.action - event.kind + firewall - event.kind - event.category - event.type - event.action - event.id - rule.name + vpcflow - event.kind - event.category - event.type - event.id Closes elastic#16030 Closes elastic#15651
leehinman · Feb 24, 2020 · e1fa198 · e1fa198
1 parent 3cd1351
commit e1fa198
Show file tree

Hide file tree

Showing 8 changed files with 634 additions and 306 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -133,6 +133,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add an SSL config example in config.yml for filebeat MISP module. {pull}16320[16320]
 - Improve ECS categorization, container & process field mappings in auditd module. {issue}16153[16153] {pull}16280[16280]
 - Improve ECS field mappings in aws module. {issue}16154[16154] {pull}16307[16307]
+- Improve ECS categorization field mappings in googlecloud module. {issue}16030[16030] {pull}16500[16500]
 
 *Heartbeat*
 

diff --git a/x-pack/filebeat/module/googlecloud/audit/config/pipeline.js b/x-pack/filebeat/module/googlecloud/audit/config/pipeline.js
@@ -38,6 +38,7 @@ function Audit(keep_original_message) {
     var saveMetadata = new processor.Convert({
         fields: [
             {from: "json.logName", to: "log.logger"},
+            {from: "json.insertId", to: "event.id"},
         ],
         ignore_missing: true
     });
@@ -103,6 +104,7 @@ function Audit(keep_original_message) {
             {from: "googlecloud.audit.authentication_info.principal_email", to: "user.email"},
             {from: "googlecloud.audit.service_name", to: "service.name"},
             {from: "googlecloud.audit.request_metadata.caller_supplied_user_agent", to: "user_agent.original"},
+            {from: "googlecloud.audit.method_name", to: "event.action"},
         ],
         fail_on_error: false,
     });
@@ -123,8 +125,8 @@ function Audit(keep_original_message) {
         }
     };
 
-    // Set event.outcome based on authentication_info and status.
-    var setEventOutcome = function(evt) {
+    // Set ECS categorization fields.
+    var setECSCategorization = function(evt) {
         if (evt.Get("googlecloud.audit.status.code") == null) {
             var authorization_info = evt.Get("googlecloud.audit.authorization_info");
             if (authorization_info.length === 1) {
@@ -143,6 +145,7 @@ function Audit(keep_original_message) {
         } else {
             evt.Put("event.outcome", "failure");
         }
+        evt.Put("event.kind", "event");
     };
 
     var pipeline = new processor.Chain()
@@ -157,7 +160,7 @@ function Audit(keep_original_message) {
         .Add(copyFields)
         .Add(dropExtraFields)
         .Add(RenameNestedFields)
-        .Add(setEventOutcome)
+        .Add(setECSCategorization)
         .Build();
 
     return {

diff --git a/x-pack/filebeat/module/googlecloud/audit/test/audit-log-entries.json.log-expected.json b/x-pack/filebeat/module/googlecloud/audit/test/audit-log-entries.json.log-expected.json
@@ -2,7 +2,10 @@
     {
         "@timestamp": "2019-12-19T00:49:36.086Z",
         "cloud.project.id": "elastic-beats",
+        "event.action": "GetResourceBillingInfo",
         "event.dataset": "googlecloud.audit",
+        "event.id": "-uihnmjctwo",
+        "event.kind": "event",
         "event.module": "googlecloud",
         "event.outcome": "success",
         "fileset.name": "audit",
@@ -33,7 +36,10 @@
     {
         "@timestamp": "2019-12-19T00:45:51.228Z",
         "cloud.project.id": "elastic-beats",
+        "event.action": "beta.compute.machineTypes.aggregatedList",
         "event.dataset": "googlecloud.audit",
+        "event.id": "-h6onuze1h7dg",
+        "event.kind": "event",
         "event.module": "googlecloud",
         "event.outcome": "failure",
         "fileset.name": "audit",
@@ -78,7 +84,10 @@
     {
         "@timestamp": "2019-12-19T00:44:25.051Z",
         "cloud.project.id": "elastic-beats",
+        "event.action": "beta.compute.instances.aggregatedList",
         "event.dataset": "googlecloud.audit",
+        "event.id": "yonau2dg2zi",
+        "event.kind": "event",
         "event.module": "googlecloud",
         "event.outcome": "success",
         "fileset.name": "audit",
@@ -123,7 +132,10 @@
     {
         "@timestamp": "2019-12-19T00:44:25.051Z",
         "cloud.project.id": "elastic-beats",
+        "event.action": "beta.compute.instances.aggregatedList",
         "event.dataset": "googlecloud.audit",
+        "event.id": "yonau3dc2zi",
+        "event.kind": "event",
         "event.module": "googlecloud",
         "event.outcome": "failure",
         "fileset.name": "audit",

diff --git a/x-pack/filebeat/module/googlecloud/firewall/config/pipeline.js b/x-pack/filebeat/module/googlecloud/firewall/config/pipeline.js
@@ -101,15 +101,18 @@ function FirewallProcessor(keep_original_message, debug) {
     builder.Add("categorizeEvent", new processor.AddFields({
         target: "event",
         fields: {
-            category: "firewall-rule",
-            type: "firewall"
+            kind: "event",
+            category: "network",
+            type: "connection",
+            action: "firewall-rule"
         },
     }));
 
     builder.Add("saveMetadata", new processor.Convert({
         fields: [
             {from: "json.logName", to: "log.logger"},
-            {from: "json.resource.labels.subnetwork_name", to: "network.name"}
+            {from: "json.resource.labels.subnetwork_name", to: "network.name"},
+            {from: "json.insertId", to: "event.id"}
         ],
         ignore_missing: true
     }));
@@ -125,15 +128,12 @@ function FirewallProcessor(keep_original_message, debug) {
         mode: "rename"
     }));
 
-    builder.Add("addOutcome", makeMapper({
-        from: "json.disposition",
-        to: "event.outcome",
-        mappings: {
-            ALLOWED: "allow",
-            DENIED: "deny"
-        },
-        default: "unknown"
-    }));
+    builder.Add("addType", function(evt) {
+        var disp = evt.Get("json.disposition");
+        if (disp != null) {
+            evt.AppendTo("event.type", disp.toLowerCase());
+        }
+    });
 
     builder.Add("addDirection", makeMapper({
         from: "json.rule_details.direction",
@@ -228,7 +228,7 @@ function FirewallProcessor(keep_original_message, debug) {
             {from: "json.dest_vpc", to: "googlecloud.destination.vpc"},
             {from: "json.src_instance", to: "googlecloud.source.instance"},
             {from: "json.src_vpc", to: "googlecloud.source.vpc"},
-
+            {from: "json.rule_details.reference", to: "rule.name"},
             {from: "json", to: "googlecloud.firewall"},
         ],
         mode: "rename",

diff --git a/x-pack/filebeat/module/googlecloud/firewall/test/rare.log-expected.json b/x-pack/filebeat/module/googlecloud/firewall/test/rare.log-expected.json
@@ -5,11 +5,16 @@
         "destination.domain": "local-adrian-test",
         "destination.ip": "10.128.0.16",
         "destination.port": 80,
-        "event.category": "firewall-rule",
+        "event.action": "firewall-rule",
+        "event.category": "network",
         "event.dataset": "googlecloud.firewall",
+        "event.id": "1dobeotg13df9f5",
+        "event.kind": "event",
         "event.module": "googlecloud",
-        "event.outcome": "deny",
-        "event.type": "firewall",
+        "event.type": [
+            "connection",
+            "denied"
+        ],
         "fileset.name": "firewall",
         "googlecloud.destination.instance.project_id": "local-test",
         "googlecloud.destination.instance.region": "us-central1",
@@ -29,7 +34,6 @@
             }
         ],
         "googlecloud.firewall.rule_details.priority": 1000,
-        "googlecloud.firewall.rule_details.reference": "network:default/firewall:adrian-test-3",
         "googlecloud.firewall.rule_details.source_range": [
             "0.0.0.0/0"
         ],
@@ -52,6 +56,7 @@
             "10.142.0.10",
             "10.128.0.16"
         ],
+        "rule.name": "network:default/firewall:adrian-test-3",
         "service.type": "googlecloud",
         "source.address": "10.142.0.10",
         "source.domain": "test-es",
@@ -64,11 +69,16 @@
         "destination.domain": "test-es",
         "destination.ip": "10.128.0.10",
         "destination.port": 57794,
-        "event.category": "firewall-rule",
+        "event.action": "firewall-rule",
+        "event.category": "network",
         "event.dataset": "googlecloud.firewall",
+        "event.id": "1dobeotg13df9f7",
+        "event.kind": "event",
         "event.module": "googlecloud",
-        "event.outcome": "deny",
-        "event.type": "firewall",
+        "event.type": [
+            "connection",
+            "denied"
+        ],
         "fileset.name": "firewall",
         "googlecloud.destination.instance.project_id": "remote-beats",
         "googlecloud.destination.instance.region": "us-east1",
@@ -88,7 +98,6 @@
             }
         ],
         "googlecloud.firewall.rule_details.priority": 1000,
-        "googlecloud.firewall.rule_details.reference": "network:default/firewall:adrian-test-3",
         "googlecloud.firewall.rule_details.source_range": [
             "0.0.0.0/0"
         ],
@@ -111,6 +120,7 @@
             "10.142.0.16",
             "10.128.0.10"
         ],
+        "rule.name": "network:default/firewall:adrian-test-3",
         "service.type": "googlecloud",
         "source.address": "10.142.0.16",
         "source.domain": "local-adrian-test",