Skip to content

Commit

Permalink
[Filebeat] Improve ECS categorization field mappings in googlecloud m…
Browse files Browse the repository at this point in the history
…odule (elastic#16500)

+ audit
  - event.id
  - event.action
  - event.kind

+ firewall
  - event.kind
  - event.category
  - event.type
  - event.action
  - event.id
  - rule.name

+ vpcflow
  - event.kind
  - event.category
  - event.type
  - event.id

Closes elastic#16030
Closes elastic#15651
  • Loading branch information
leehinman authored Feb 24, 2020
1 parent 3cd1351 commit e1fa198
Show file tree
Hide file tree
Showing 8 changed files with 634 additions and 306 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.next.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
- Add an SSL config example in config.yml for filebeat MISP module. {pull}16320[16320]
- Improve ECS categorization, container & process field mappings in auditd module. {issue}16153[16153] {pull}16280[16280]
- Improve ECS field mappings in aws module. {issue}16154[16154] {pull}16307[16307]
- Improve ECS categorization field mappings in googlecloud module. {issue}16030[16030] {pull}16500[16500]

*Heartbeat*

Expand Down
9 changes: 6 additions & 3 deletions x-pack/filebeat/module/googlecloud/audit/config/pipeline.js
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ function Audit(keep_original_message) {
var saveMetadata = new processor.Convert({
fields: [
{from: "json.logName", to: "log.logger"},
{from: "json.insertId", to: "event.id"},
],
ignore_missing: true
});
Expand Down Expand Up @@ -103,6 +104,7 @@ function Audit(keep_original_message) {
{from: "googlecloud.audit.authentication_info.principal_email", to: "user.email"},
{from: "googlecloud.audit.service_name", to: "service.name"},
{from: "googlecloud.audit.request_metadata.caller_supplied_user_agent", to: "user_agent.original"},
{from: "googlecloud.audit.method_name", to: "event.action"},
],
fail_on_error: false,
});
Expand All @@ -123,8 +125,8 @@ function Audit(keep_original_message) {
}
};

// Set event.outcome based on authentication_info and status.
var setEventOutcome = function(evt) {
// Set ECS categorization fields.
var setECSCategorization = function(evt) {
if (evt.Get("googlecloud.audit.status.code") == null) {
var authorization_info = evt.Get("googlecloud.audit.authorization_info");
if (authorization_info.length === 1) {
Expand All @@ -143,6 +145,7 @@ function Audit(keep_original_message) {
} else {
evt.Put("event.outcome", "failure");
}
evt.Put("event.kind", "event");
};

var pipeline = new processor.Chain()
Expand All @@ -157,7 +160,7 @@ function Audit(keep_original_message) {
.Add(copyFields)
.Add(dropExtraFields)
.Add(RenameNestedFields)
.Add(setEventOutcome)
.Add(setECSCategorization)
.Build();

return {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,10 @@
{
"@timestamp": "2019-12-19T00:49:36.086Z",
"cloud.project.id": "elastic-beats",
"event.action": "GetResourceBillingInfo",
"event.dataset": "googlecloud.audit",
"event.id": "-uihnmjctwo",
"event.kind": "event",
"event.module": "googlecloud",
"event.outcome": "success",
"fileset.name": "audit",
Expand Down Expand Up @@ -33,7 +36,10 @@
{
"@timestamp": "2019-12-19T00:45:51.228Z",
"cloud.project.id": "elastic-beats",
"event.action": "beta.compute.machineTypes.aggregatedList",
"event.dataset": "googlecloud.audit",
"event.id": "-h6onuze1h7dg",
"event.kind": "event",
"event.module": "googlecloud",
"event.outcome": "failure",
"fileset.name": "audit",
Expand Down Expand Up @@ -78,7 +84,10 @@
{
"@timestamp": "2019-12-19T00:44:25.051Z",
"cloud.project.id": "elastic-beats",
"event.action": "beta.compute.instances.aggregatedList",
"event.dataset": "googlecloud.audit",
"event.id": "yonau2dg2zi",
"event.kind": "event",
"event.module": "googlecloud",
"event.outcome": "success",
"fileset.name": "audit",
Expand Down Expand Up @@ -123,7 +132,10 @@
{
"@timestamp": "2019-12-19T00:44:25.051Z",
"cloud.project.id": "elastic-beats",
"event.action": "beta.compute.instances.aggregatedList",
"event.dataset": "googlecloud.audit",
"event.id": "yonau3dc2zi",
"event.kind": "event",
"event.module": "googlecloud",
"event.outcome": "failure",
"fileset.name": "audit",
Expand Down
26 changes: 13 additions & 13 deletions x-pack/filebeat/module/googlecloud/firewall/config/pipeline.js
Original file line number Diff line number Diff line change
Expand Up @@ -101,15 +101,18 @@ function FirewallProcessor(keep_original_message, debug) {
builder.Add("categorizeEvent", new processor.AddFields({
target: "event",
fields: {
category: "firewall-rule",
type: "firewall"
kind: "event",
category: "network",
type: "connection",
action: "firewall-rule"
},
}));

builder.Add("saveMetadata", new processor.Convert({
fields: [
{from: "json.logName", to: "log.logger"},
{from: "json.resource.labels.subnetwork_name", to: "network.name"}
{from: "json.resource.labels.subnetwork_name", to: "network.name"},
{from: "json.insertId", to: "event.id"}
],
ignore_missing: true
}));
Expand All @@ -125,15 +128,12 @@ function FirewallProcessor(keep_original_message, debug) {
mode: "rename"
}));

builder.Add("addOutcome", makeMapper({
from: "json.disposition",
to: "event.outcome",
mappings: {
ALLOWED: "allow",
DENIED: "deny"
},
default: "unknown"
}));
builder.Add("addType", function(evt) {
var disp = evt.Get("json.disposition");
if (disp != null) {
evt.AppendTo("event.type", disp.toLowerCase());
}
});

builder.Add("addDirection", makeMapper({
from: "json.rule_details.direction",
Expand Down Expand Up @@ -228,7 +228,7 @@ function FirewallProcessor(keep_original_message, debug) {
{from: "json.dest_vpc", to: "googlecloud.destination.vpc"},
{from: "json.src_instance", to: "googlecloud.source.instance"},
{from: "json.src_vpc", to: "googlecloud.source.vpc"},

{from: "json.rule_details.reference", to: "rule.name"},
{from: "json", to: "googlecloud.firewall"},
],
mode: "rename",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,16 @@
"destination.domain": "local-adrian-test",
"destination.ip": "10.128.0.16",
"destination.port": 80,
"event.category": "firewall-rule",
"event.action": "firewall-rule",
"event.category": "network",
"event.dataset": "googlecloud.firewall",
"event.id": "1dobeotg13df9f5",
"event.kind": "event",
"event.module": "googlecloud",
"event.outcome": "deny",
"event.type": "firewall",
"event.type": [
"connection",
"denied"
],
"fileset.name": "firewall",
"googlecloud.destination.instance.project_id": "local-test",
"googlecloud.destination.instance.region": "us-central1",
Expand All @@ -29,7 +34,6 @@
}
],
"googlecloud.firewall.rule_details.priority": 1000,
"googlecloud.firewall.rule_details.reference": "network:default/firewall:adrian-test-3",
"googlecloud.firewall.rule_details.source_range": [
"0.0.0.0/0"
],
Expand All @@ -52,6 +56,7 @@
"10.142.0.10",
"10.128.0.16"
],
"rule.name": "network:default/firewall:adrian-test-3",
"service.type": "googlecloud",
"source.address": "10.142.0.10",
"source.domain": "test-es",
Expand All @@ -64,11 +69,16 @@
"destination.domain": "test-es",
"destination.ip": "10.128.0.10",
"destination.port": 57794,
"event.category": "firewall-rule",
"event.action": "firewall-rule",
"event.category": "network",
"event.dataset": "googlecloud.firewall",
"event.id": "1dobeotg13df9f7",
"event.kind": "event",
"event.module": "googlecloud",
"event.outcome": "deny",
"event.type": "firewall",
"event.type": [
"connection",
"denied"
],
"fileset.name": "firewall",
"googlecloud.destination.instance.project_id": "remote-beats",
"googlecloud.destination.instance.region": "us-east1",
Expand All @@ -88,7 +98,6 @@
}
],
"googlecloud.firewall.rule_details.priority": 1000,
"googlecloud.firewall.rule_details.reference": "network:default/firewall:adrian-test-3",
"googlecloud.firewall.rule_details.source_range": [
"0.0.0.0/0"
],
Expand All @@ -111,6 +120,7 @@
"10.142.0.16",
"10.128.0.10"
],
"rule.name": "network:default/firewall:adrian-test-3",
"service.type": "googlecloud",
"source.address": "10.142.0.16",
"source.domain": "local-adrian-test",
Expand Down
Loading

0 comments on commit e1fa198

Please sign in to comment.