Skip to content

Commit

Permalink
Add additional fields to address issue #18465 for googlecloud audit l…
Browse files Browse the repository at this point in the history
…og (#18472)

* Fix issue 18465

* Update test data for googlecloud.audit dataset

* Update CHANGELOG.next.asciidoc
  • Loading branch information
Lei Qiu authored May 14, 2020
1 parent 8694f47 commit c5bd3e4
Show file tree
Hide file tree
Showing 7 changed files with 140 additions and 17 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.next.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
- Fix PANW module wrong mappings for bytes and packets counters. {issue}18522[18522] {pull}18525[18525]
- Fixed ingestion of some Cisco ASA and FTD messages when a hostname was used instead of an IP for NAT fields. {issue}14034[14034] {pull}18376[18376]
- Fix a rate limit related issue in httpjson input for Okta module. {issue}18530[18530] {pull}18534[18534]
- Fix `googlecloud.audit` pipeline to only take in fields that are explicitly defined by the dataset. {issue}18465[18465] {pull}18472[18472]

*Heartbeat*

Expand Down
74 changes: 74 additions & 0 deletions filebeat/docs/fields.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -22897,6 +22897,80 @@ type: ip
The user agent of the caller. This information is not authenticated and should be treated accordingly.
type: keyword
--
[float]
=== response
The operation response.
*`googlecloud.audit.response.proto_name`*::
+
--
Type property of the response.
type: keyword
--
[float]
=== details
The details of the response.
*`googlecloud.audit.response.details.group`*::
+
--
The name of the group.
type: keyword
--
*`googlecloud.audit.response.details.kind`*::
+
--
The kind of the response details.
type: keyword
--
*`googlecloud.audit.response.details.name`*::
+
--
The name of the response details.
type: keyword
--
*`googlecloud.audit.response.details.uid`*::
+
--
The uid of the response details.
type: keyword
--
*`googlecloud.audit.response.status`*::
+
--
Status of the response.
type: keyword
--
Expand Down
34 changes: 34 additions & 0 deletions x-pack/filebeat/module/googlecloud/audit/_meta/fields.yml
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,40 @@
description: >
The user agent of the caller. This information is not authenticated and
should be treated accordingly.
- name: response
type: group
description: >
The operation response.
fields:
- name: proto_name
type: keyword
description: >
Type property of the response.
- name: details
type: group
description: >
The details of the response.
fields:
- name: group
type: keyword
description: >
The name of the group.
- name: kind
type: keyword
description: >
The kind of the response details.
- name: name
type: keyword
description: >
The name of the response details.
- name: uid
type: keyword
description: >
The uid of the response details.
- name: status
type: keyword
description: >
Status of the response.
- name: resource_name
type: keyword
description: >
Expand Down
36 changes: 22 additions & 14 deletions x-pack/filebeat/module/googlecloud/audit/config/pipeline.js
Original file line number Diff line number Diff line change
Expand Up @@ -63,35 +63,43 @@ function Audit(keep_original_message) {
// contains the structured audit log fields.
var convertProtoPayload = new processor.Convert({
fields: [
{from: "json.@type", to: "json.type"},
{from: "json.@type", to: "googlecloud.audit.type"},

{from: "json.authenticationInfo.principalEmail", to: "json.authenticationInfo.principal_email"},
{from: "json.authenticationInfo.authoritySelector", to: "json.authenticationInfo.authority_selector"},
{from: "json.authenticationInfo", to: "json.authentication_info"},
{from: "json.authenticationInfo", to: "googlecloud.audit.authentication_info"},

{from: "json.authorizationInfo", to: "json.authorization_info"},
{from: "json.authorizationInfo", to: "googlecloud.audit.authorization_info"},

{from: "json.methodName", to: "json.method_name"},
{from: "json.methodName", to: "googlecloud.audit.method_name"},

{from: "json.numResponseItems", to: "json.num_response_items", type: "long"},
{from: "json.numResponseItems", to: "googlecloud.audit.num_response_items", type: "long"},

{from: "json.request.@type", to: "json.request.proto_name"},
{from: "json.request.filter", to: "json.request.filter"},
{from: "json.request.name", to: "json.request.name"},
{from: "json.request.resourceName", to: "json.request.resource_name"},
{from: "json.request.@type", to: "googlecloud.audit.request.proto_name"},
{from: "json.request.filter", to: "googlecloud.audit.request.filter"},
{from: "json.request.name", to: "googlecloud.audit.request.name"},
{from: "json.request.resourceName", to: "googlecloud.audit.request.resource_name"},

{from: "json.requestMetadata.callerIp", to: "json.requestMetadata.caller_ip", type: "ip"},
{from: "json.requestMetadata.callerSuppliedUserAgent", to: "json.requestMetadata.caller_supplied_user_agent"},
{from: "json.requestMetadata", to: "json.request_metadata"},
{from: "json.requestMetadata", to: "googlecloud.audit.request_metadata"},

{from: "json.resourceName", to: "json.resource_name"},
{from: "json.response.@type", to: "googlecloud.audit.response.proto_name"},
{from: "json.response.status", to: "googlecloud.audit.response.status"},
{from: "json.response.details.group", to: "googlecloud.audit.response.details.group"},
{from: "json.response.details.kind", to: "googlecloud.audit.response.details.kind"},
{from: "json.response.details.name", to: "googlecloud.audit.response.details.name"},
{from: "json.response.details.uid", to: "googlecloud.audit.response.details.uid"},

{from: "json.resourceName", to: "googlecloud.audit.resource_name"},

{from: "json.resourceLocation.currentLocations", to: "json.resourceLocation.current_locations"},
{from: "json.resourceLocation", to: "json.resource_location"},
{from: "json.resourceLocation", to: "googlecloud.audit.resource_location"},

{from: "json.serviceName", to: "googlecloud.audit.service_name"},

{from: "json.serviceName", to: "json.service_name"},
{from: "json.status", to: "googlecloud.audit.status"},

{from: "json", to: "googlecloud.audit"},
],
mode: "rename",
ignore_missing: true,
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
{"insertId":"-uihnmjctwo","logName":"projects/elastic-beats/logs/cloudaudit.googleapis.com%2Fdata_access","protoPayload":{"@type":"type.googleapis.com/google.cloud.audit.AuditLog","authenticationInfo":{"principalEmail":"[email protected]"},"authorizationInfo":[{"granted":true,"permission":"resourcemanager.projects.get","resource":"projects/elastic-beats","resourceAttributes":{}}],"methodName":"GetResourceBillingInfo","request":{"@type":"type.googleapis.com/google.internal.cloudbilling.billingaccount.v1.GetResourceBillingInfoRequest","resourceName":"projects/189716325846"},"requestMetadata":{"callerIp":"192.168.1.1","destinationAttributes":{},"requestAttributes":{}},"resourceName":"projects/elastic-beats","serviceName":"cloudbilling.googleapis.com","status":{}},"receiveTimestamp":"2019-12-19T00:49:36.313482371Z","resource":{"labels":{"project_id":"elastic-beats"},"type":"project"},"severity":"INFO","timestamp":"2019-12-19T00:49:36.086Z"}
{"insertId":"-h6onuze1h7dg","logName":"projects/elastic-beats/logs/cloudaudit.googleapis.com%2Fdata_access","protoPayload":{"@type":"type.googleapis.com/google.cloud.audit.AuditLog","authenticationInfo":{"principalEmail":"[email protected]"},"authorizationInfo":[{"granted":false,"permission":"compute.machineTypes.list","resourceAttributes":{"name":"projects/elastic-beats","service":"resourcemanager","type":"resourcemanager.projects"}}],"methodName":"beta.compute.machineTypes.aggregatedList","numResponseItems":"71","request":{"@type":"type.googleapis.com/compute.machineTypes.aggregatedList"},"requestMetadata":{"callerIp":"192.168.1.1","callerSuppliedUserAgent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:71.0) Gecko/20100101 Firefox/71.0,gzip(gfe),gzip(gfe)","destinationAttributes":{},"requestAttributes":{"auth":{},"time":"2019-12-19T00:45:51.711Z"}},"resourceLocation":{"currentLocations":["global"]},"resourceName":"projects/elastic-beats/global/machineTypes","serviceName":"compute.googleapis.com"},"receiveTimestamp":"2019-12-19T00:45:52.367887078Z","resource":{"labels":{"location":"global","method":"compute.machineTypes.aggregatedList","project_id":"elastic-beats","service":"compute.googleapis.com","version":"beta"},"type":"api"},"severity":"INFO","timestamp":"2019-12-19T00:45:51.228Z"}
{"insertId":"yonau2dg2zi","logName":"projects/elastic-beats/logs/cloudaudit.googleapis.com%2Fdata_access","protoPayload":{"@type":"type.googleapis.com/google.cloud.audit.AuditLog","authenticationInfo":{"principalEmail":"[email protected]"},"authorizationInfo":[{"granted":true,"permission":"compute.instances.list","resourceAttributes":{"name":"projects/elastic-beats","service":"resourcemanager","type":"resourcemanager.projects"}}],"methodName":"beta.compute.instances.aggregatedList","numResponseItems":"61","request":{"@type":"type.googleapis.com/compute.instances.aggregatedList"},"requestMetadata":{"callerIp":"192.168.1.1","callerSuppliedUserAgent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:71.0) Gecko/20100101 Firefox/71.0,gzip(gfe),gzip(gfe)","destinationAttributes":{},"requestAttributes":{"auth":{},"time":"2019-12-19T00:44:25.198Z"}},"resourceLocation":{"currentLocations":["global"]},"resourceName":"projects/elastic-beats/global/instances","serviceName":"compute.googleapis.com"},"receiveTimestamp":"2019-12-19T00:44:25.262379373Z","resource":{"labels":{"location":"global","method":"compute.instances.aggregatedList","project_id":"elastic-beats","service":"compute.googleapis.com","version":"beta"},"type":"api"},"severity":"INFO","timestamp":"2019-12-19T00:44:25.051Z"}
{"insertId":"yonau2dg2zi","logName":"projects/elastic-beats/logs/cloudaudit.googleapis.com%2Fdata_access","protoPayload":{"@type":"type.googleapis.com/google.cloud.audit.AuditLog","authenticationInfo":{"principalEmail":"[email protected]"},"authorizationInfo":[{"granted":true,"permission":"compute.instances.list","resourceAttributes":{"name":"projects/elastic-beats","service":"resourcemanager","type":"resourcemanager.projects"}}],"methodName":"beta.compute.instances.aggregatedList","numResponseItems":"61","request":{"@type":"type.googleapis.com/compute.instances.aggregatedList"},"requestMetadata":{"callerIp":"192.168.1.1","callerSuppliedUserAgent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:71.0) Gecko/20100101 Firefox/71.0,gzip(gfe),gzip(gfe)","destinationAttributes":{},"requestAttributes":{"auth":{},"time":"2019-12-19T00:44:25.198Z"}},"response":{"@type":"core.k8s.io/v1.Status","apiVersion":"v1","details":{"group":"batch","kind":"jobs","name":"gsuite-exporter-1589294700","uid":"2beff34a-945f-11ea-bacf-42010a80007f"},"kind":"Status","metadata":{},"status":"Success"},"resourceLocation":{"currentLocations":["global"]},"resourceName":"projects/elastic-beats/global/instances","serviceName":"compute.googleapis.com"},"receiveTimestamp":"2019-12-19T00:44:25.262379373Z","resource":{"labels":{"location":"global","method":"compute.instances.aggregatedList","project_id":"elastic-beats","service":"compute.googleapis.com","version":"beta"},"type":"api"},"severity":"INFO","timestamp":"2019-12-19T00:44:25.051Z"}
{"insertId":"yonau3dc2zi","logName":"projects/elastic-beats/logs/cloudaudit.googleapis.com%2Fdata_access","protoPayload":{"@type":"type.googleapis.com/google.cloud.audit.AuditLog","authenticationInfo":{"principalEmail":"[email protected]"},"authorizationInfo":[{"permission":"compute.instances.list","resourceAttributes":{"name":"projects/elastic-beats","service":"resourcemanager","type":"resourcemanager.projects"}}],"methodName":"beta.compute.instances.aggregatedList","numResponseItems":"61","request":{"@type":"type.googleapis.com/compute.instances.aggregatedList"},"requestMetadata":{"callerIp":"192.168.1.1","callerSuppliedUserAgent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:71.0) Gecko/20100101 Firefox/71.0,gzip(gfe),gzip(gfe)","destinationAttributes":{},"requestAttributes":{"auth":{},"time":"2019-12-19T00:44:25.198Z"}},"resourceLocation":{"currentLocations":["global"]},"resourceName":"projects/elastic-beats/global/instances","serviceName":"compute.googleapis.com","status":{"code":7,"message":"PERMISSION_DENIED"}},"receiveTimestamp":"2019-12-19T00:44:25.262379373Z","resource":{"labels":{"location":"global","method":"compute.instances.aggregatedList","project_id":"elastic-beats","service":"compute.googleapis.com","version":"beta"},"type":"api"},"severity":"INFO","timestamp":"2019-12-19T00:44:25.051Z"}
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,12 @@
"global"
],
"googlecloud.audit.resource_name": "projects/elastic-beats/global/instances",
"googlecloud.audit.response.details.group": "batch",
"googlecloud.audit.response.details.kind": "jobs",
"googlecloud.audit.response.details.name": "gsuite-exporter-1589294700",
"googlecloud.audit.response.details.uid": "2beff34a-945f-11ea-bacf-42010a80007f",
"googlecloud.audit.response.proto_name": "core.k8s.io/v1.Status",
"googlecloud.audit.response.status": "Success",
"googlecloud.audit.service_name": "compute.googleapis.com",
"googlecloud.audit.type": "type.googleapis.com/google.cloud.audit.AuditLog",
"input.type": "log",
Expand Down Expand Up @@ -165,7 +171,7 @@
"googlecloud.audit.type": "type.googleapis.com/google.cloud.audit.AuditLog",
"input.type": "log",
"log.logger": "projects/elastic-beats/logs/cloudaudit.googleapis.com%2Fdata_access",
"log.offset": 3541,
"log.offset": 3776,
"service.name": "compute.googleapis.com",
"service.type": "googlecloud",
"source.ip": "192.168.1.1",
Expand Down
2 changes: 1 addition & 1 deletion x-pack/filebeat/module/googlecloud/fields.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

0 comments on commit c5bd3e4

Please sign in to comment.