Cherry-pick #18223 to 7.x: Disable host fields for "cloud", panw, cef…

… modules (#19074) This changes the default configuration of Filebeat to not add `host` fields to events that originated in other places. The `host` field is defined in ECS as "host on which the event happened" but for data pulled from cloud APIs for data forwarded to Filebeat from other sources (PANW, CEF) this `host` field is inaccurate. The affected "cloud" modules are azure, aws, googlecloud, o365, and okta. By default they will tag events with `forwarded`. This will cause the module to not add `host.name` at the input state. And then the default configuration for Filebeat was updated to add a `when` condition to the `add_host_metadata` processors to skip events containing the `forwarded` tag. For PANW and CEF when data is forwarded to Filebeat from another host/device (this is most of the time) you don't want Filebeat to add `host`. So by default this modules add a `forwarded` tag to events. If you configure the module to not include the `forwarded` tag (e.g. `var.tags: [my_tag]`) then Filebeat will add the `host.*` fields. And for PANW I added some additional static `observer.*` fields. Relates: #13920 (cherry picked from commit e990740)
elastic · Jun 9, 2020 · 8af0424 · 8af0424
1 parent da9cb87
commit 8af0424
Show file tree

Hide file tree

Showing 127 changed files with 4,008 additions and 581 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -39,6 +39,13 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Improve ECS categorization field mappings for nginx module. http.request.referrer only populated when nginx sets a value {issue}16174[16174] {pull}17844[17844]
 - Improve ECS field mappings in santa module. move hash.sha256 to process.hash.sha256 & move certificate fields to santa.certificate . {issue}16180[16180] {pull}17982[17982]
 - Preserve case of http.request.method.  ECS prior to 1.6 specified normalizing to lowercase, which lost information. Affects filesets: apache/access, elasticsearch/audit, iis/access, iis/error, nginx/access, nginx/ingress_controller, aws/elb, suricata/eve, zeek/http. {issue}18154[18154] {pull}18359[18359]
+- With the default configuration the cloud modules (aws, azure, googlecloud, o365, okta)
+will no longer send the `host` field that contains information about the host Filebeat is
+running on. This is because the `host` field specifies the host on which the event
+happened. {issue}13920[13920] {pull}18223[18223]
+- With the default configuration the cef and panw modules will no longer send the `host`
+field. You can revert this change by configuring tags for the module and omitting
+`forwarded` from the list. {issue}13920[13920] {pull}18223[18223]
 
 *Heartbeat*
 
@@ -434,6 +441,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add dashboard for Google Cloud Audit and AWS CloudTrail. {pull}17379[17379]
 - Added http_endpoint input{pull}18298[18298]
 - Add support for array parsing in azure-eventhub input. {pull}18585[18585]
+- Added `observer.vendor`, `observer.product`, and `observer.type` to PANW module events. {pull}18223[18223]
 - The `logstash` module can now automatically detect the log file format (JSON or plaintext) and process it accordingly. {issue}9964[9964] {pull}18095[18095]
 - Improve ECS categorization field mappings in coredns module. {issue}16159[16159] {pull}18424[18424]
 - Improve ECS categorization field mappings in envoyproxy module. {issue}16161[16161] {pull}18395[18395]

diff --git a/filebeat/_meta/config/processors.yml.tmpl b/filebeat/_meta/config/processors.yml.tmpl
@@ -0,0 +1,7 @@
+{{header "Processors"}}
+processors:
+  - add_host_metadata:
+      when.not.contains.tags: forwarded
+  - add_cloud_metadata: ~
+  - add_docker_metadata: ~
+  - add_kubernetes_metadata: ~
diff --git a/filebeat/docs/modules/cef.asciidoc b/filebeat/docs/modules/cef.asciidoc
@@ -40,6 +40,12 @@ The UDP port to listen for syslog traffic. Defaults to `9003`
 
 NOTE: Ports below 1024 require Filebeat to run as root.
 
+*`var.tags`*::
+
+A list of tags to include in events. Including `forwarded` indicates that the
+events did not originate on this host and causes `host.name` to not be added to
+events. Defaults to `[cef, forwarded]`.
+
 [float]
 ==== Forcepoint NGFW Security Management Center
 

diff --git a/filebeat/filebeat.yml b/filebeat/filebeat.yml
@@ -172,16 +172,13 @@ output.elasticsearch:
   #ssl.key: "/etc/pki/client/cert.key"
 
 # ================================= Processors =================================
-
-# Configure processors to enhance or manipulate events generated by the beat.
-
 processors:
-  - add_host_metadata: ~
+  - add_host_metadata:
+      when.not.contains.tags: forwarded
   - add_cloud_metadata: ~
   - add_docker_metadata: ~
   - add_kubernetes_metadata: ~
 
-
 # ================================== Logging ===================================
 
 # Sets log level. The default log level is info.

diff --git a/filebeat/fileset/fileset.go b/filebeat/fileset/fileset.go
@@ -27,6 +27,7 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"reflect"
 	"runtime"
 	"strings"
 	"text/template"
@@ -297,6 +298,18 @@ func getTemplateFunctions(vars map[string]interface{}) (template.FuncMap, error)
 	}
 
 	return template.FuncMap{
+		"inList": func(collection []interface{}, item string) bool {
+			for _, h := range collection {
+				if reflect.DeepEqual(item, h) {
+					return true
+				}
+			}
+			return false
+		},
+		"tojson": func(v interface{}) (string, error) {
+			bytes, err := json.Marshal(v)
+			return string(bytes), err
+		},
 		"IngestPipeline": func(shortID string) string {
 			return formatPipelineID(
 				builtinVars["prefix"].(string),

diff --git a/filebeat/fileset/fileset_test.go b/filebeat/fileset/fileset_test.go
@@ -288,6 +288,7 @@ func TestGetTemplateFunctions(t *testing.T) {
 	templateFunctions, err := getTemplateFunctions(vars)
 	assert.NoError(t, err)
 	assert.IsType(t, template.FuncMap{}, templateFunctions)
-	assert.Len(t, templateFunctions, 1)
+	assert.Contains(t, templateFunctions, "inList")
+	assert.Contains(t, templateFunctions, "tojson")
 	assert.Contains(t, templateFunctions, "IngestPipeline")
 }
diff --git a/x-pack/filebeat/filebeat.yml b/x-pack/filebeat/filebeat.yml
@@ -172,16 +172,13 @@ output.elasticsearch:
   #ssl.key: "/etc/pki/client/cert.key"
 
 # ================================= Processors =================================
-
-# Configure processors to enhance or manipulate events generated by the beat.
-
 processors:
-  - add_host_metadata: ~
+  - add_host_metadata:
+      when.not.contains.tags: forwarded
   - add_cloud_metadata: ~
   - add_docker_metadata: ~
   - add_kubernetes_metadata: ~
 
-
 # ================================== Logging ===================================
 
 # Sets log level. The default log level is info.

diff --git a/x-pack/filebeat/module/aws/cloudtrail/config/file.yml b/x-pack/filebeat/module/aws/cloudtrail/config/file.yml
@@ -4,6 +4,9 @@ paths:
   - {{$path}}
     {{ end }}
 exclude_files: [".gz$"]
+tags: {{.tags | tojson}}
+publisher_pipeline.disable_host: {{ inList .tags "forwarded" }}
+
 processors:
   - add_fields:
       target: ''

diff --git a/x-pack/filebeat/module/aws/cloudtrail/config/s3.yml b/x-pack/filebeat/module/aws/cloudtrail/config/s3.yml
@@ -38,6 +38,9 @@ session_token: {{ .session_token }}
 role_arn: {{ .role_arn }}
 {{ end }}
 
+tags: {{.tags | tojson}}
+publisher_pipeline.disable_host: {{ inList .tags "forwarded" }}
+
 processors:
   - add_fields:
       target: ''

diff --git a/x-pack/filebeat/module/aws/cloudtrail/manifest.yml b/x-pack/filebeat/module/aws/cloudtrail/manifest.yml
@@ -13,6 +13,8 @@ var:
   - name: secret_access_key
   - name: session_token
   - name: role_arn
+  - name: tags
+    default: [forwarded]
 
 ingest_pipeline: ingest/pipeline.yml
 input: config/{{.input}}.yml
diff --git a/x-pack/filebeat/module/aws/cloudtrail/test/add-user-to-group-json.log-expected.json b/x-pack/filebeat/module/aws/cloudtrail/test/add-user-to-group-json.log-expected.json
@@ -33,6 +33,9 @@
         "service.type": "aws",
         "source.address": "127.0.0.1",
         "source.ip": "127.0.0.1",
+        "tags": [
+            "forwarded"
+        ],
         "user.id": "EX_PRINCIPAL_ID",
         "user.name": "Alice",
         "user_agent.device.name": "Other",

diff --git a/x-pack/filebeat/module/aws/cloudtrail/test/assume-role-json.log-expected.json b/x-pack/filebeat/module/aws/cloudtrail/test/assume-role-json.log-expected.json
@@ -41,6 +41,9 @@
         "source.geo.region_iso_code": "CN-CQ",
         "source.geo.region_name": "Chongqing",
         "source.ip": "123.145.67.89",
+        "tags": [
+            "forwarded"
+        ],
         "user.id": "AROAIN5ATK5U7KEXAMPLE:JohnRole1",
         "user_agent.device.name": "Spider",
         "user_agent.name": "aws-cli",

diff --git a/x-pack/filebeat/module/aws/cloudtrail/test/change-password-json.log-expected.json b/x-pack/filebeat/module/aws/cloudtrail/test/change-password-json.log-expected.json
@@ -32,6 +32,9 @@
         "service.type": "aws",
         "source.address": "127.0.0.1",
         "source.ip": "127.0.0.1",
+        "tags": [
+            "forwarded"
+        ],
         "user.id": "0123456789012",
         "user.name": "Alice",
         "user_agent.device.name": "Spider",
@@ -70,6 +73,9 @@
         "service.type": "aws",
         "source.address": "127.0.0.1",
         "source.ip": "127.0.0.1",
+        "tags": [
+            "forwarded"
+        ],
         "user.id": "0123456789012",
         "user.name": "Alice",
         "user_agent.device.name": "Spider",

diff --git a/x-pack/filebeat/module/aws/cloudtrail/test/console-login-json.log-expected.json b/x-pack/filebeat/module/aws/cloudtrail/test/console-login-json.log-expected.json
@@ -31,6 +31,9 @@
         "service.type": "aws",
         "source.address": "192.0.2.110",
         "source.ip": "192.0.2.110",
+        "tags": [
+            "forwarded"
+        ],
         "user.id": "AIDACKCEVSQ6C2EXAMPLE",
         "user.name": "JohnDoe",
         "user_agent.device.name": "Other",
@@ -74,6 +77,9 @@
         "service.type": "aws",
         "source.address": "192.0.2.100",
         "source.ip": "192.0.2.100",
+        "tags": [
+            "forwarded"
+        ],
         "user.id": "AIDACKCEVSQ6C2EXAMPLE",
         "user.name": "JaneDoe",
         "user_agent.device.name": "Other",
@@ -123,6 +129,9 @@
         "service.type": "aws",
         "source.address": "192.0.2.100",
         "source.ip": "192.0.2.100",
+        "tags": [
+            "forwarded"
+        ],
         "user.id": "AROAIDPPEZS35WEXAMPLE:AssumedRoleSessionName",
         "user.name": "RoleToBeAssumed",
         "user_agent.device.name": "Other",

diff --git a/x-pack/filebeat/module/aws/cloudtrail/test/create-access-key-json.log-expected.json b/x-pack/filebeat/module/aws/cloudtrail/test/create-access-key-json.log-expected.json
@@ -38,6 +38,9 @@
         "service.type": "aws",
         "source.address": "127.0.0.1",
         "source.ip": "127.0.0.1",
+        "tags": [
+            "forwarded"
+        ],
         "user.id": "EXAMPLE_ID",
         "user.name": "Alice",
         "user_agent.device.name": "Other",

diff --git a/x-pack/filebeat/module/aws/cloudtrail/test/create-group-json.log-expected.json b/x-pack/filebeat/module/aws/cloudtrail/test/create-group-json.log-expected.json
@@ -35,6 +35,9 @@
         "service.type": "aws",
         "source.address": "127.0.0.1",
         "source.ip": "127.0.0.1",
+        "tags": [
+            "forwarded"
+        ],
         "user.id": "0123456789012",
         "user.name": "Alice",
         "user_agent.device.name": "Other",
@@ -75,6 +78,9 @@
         "service.type": "aws",
         "source.address": "127.0.0.1",
         "source.ip": "127.0.0.1",
+        "tags": [
+            "forwarded"
+        ],
         "user.id": "0123456789012",
         "user.name": "Alice",
         "user_agent.device.name": "Spider",

diff --git a/x-pack/filebeat/module/aws/cloudtrail/test/create-key-pair-json.log-expected.json b/x-pack/filebeat/module/aws/cloudtrail/test/create-key-pair-json.log-expected.json
@@ -40,6 +40,9 @@
         "source.geo.region_iso_code": "US-VA",
         "source.geo.region_name": "Virginia",
         "source.ip": "72.21.198.64",
+        "tags": [
+            "forwarded"
+        ],
         "user.id": "EX_PRINCIPAL_ID",
         "user.name": "Alice",
         "user_agent.device.name": "Other",

diff --git a/x-pack/filebeat/module/aws/cloudtrail/test/create-trail-json.log-expected.json b/x-pack/filebeat/module/aws/cloudtrail/test/create-trail-json.log-expected.json
@@ -30,6 +30,9 @@
         "service.type": "aws",
         "source.address": "127.0.0.1",
         "source.ip": "127.0.0.1",
+        "tags": [
+            "forwarded"
+        ],
         "user.id": "EXAMPLE_ID",
         "user.name": "Alice",
         "user_agent.device.name": "Other",

diff --git a/x-pack/filebeat/module/aws/cloudtrail/test/create-user-json.log-expected.json b/x-pack/filebeat/module/aws/cloudtrail/test/create-user-json.log-expected.json
@@ -32,6 +32,9 @@
         "service.type": "aws",
         "source.address": "127.0.0.1",
         "source.ip": "127.0.0.1",
+        "tags": [
+            "forwarded"
+        ],
         "user.id": "EX_PRINCIPAL_ID",
         "user.name": "Alice",
         "user_agent.device.name": "Other",

diff --git a/x-pack/filebeat/module/aws/cloudtrail/test/create-virtual-mfa-device-json.log-expected.json b/x-pack/filebeat/module/aws/cloudtrail/test/create-virtual-mfa-device-json.log-expected.json
@@ -34,6 +34,9 @@
         "service.type": "aws",
         "source.address": "127.0.0.1",
         "source.ip": "127.0.0.1",
+        "tags": [
+            "forwarded"
+        ],
         "user.id": "EXAMPLE_ID",
         "user.name": "Alice",
         "user_agent.device.name": "Other",

diff --git a/x-pack/filebeat/module/aws/cloudtrail/test/deactivate-mfa-device-json.log-expected.json b/x-pack/filebeat/module/aws/cloudtrail/test/deactivate-mfa-device-json.log-expected.json
@@ -37,6 +37,9 @@
         "service.type": "aws",
         "source.address": "127.0.0.1",
         "source.ip": "127.0.0.1",
+        "tags": [
+            "forwarded"
+        ],
         "user.id": "EXAMPLE_ID",
         "user.name": "Alice",
         "user_agent.device.name": "Other",

diff --git a/x-pack/filebeat/module/aws/cloudtrail/test/delete-access-key-json.log-expected.json b/x-pack/filebeat/module/aws/cloudtrail/test/delete-access-key-json.log-expected.json
@@ -37,6 +37,9 @@
         "service.type": "aws",
         "source.address": "127.0.0.1",
         "source.ip": "127.0.0.1",
+        "tags": [
+            "forwarded"
+        ],
         "user.id": "EXAMPLE_ID",
         "user.name": "Alice",
         "user_agent.device.name": "Other",

diff --git a/x-pack/filebeat/module/aws/cloudtrail/test/delete-bucket-json.log-expected.json b/x-pack/filebeat/module/aws/cloudtrail/test/delete-bucket-json.log-expected.json
@@ -32,6 +32,9 @@
         "service.type": "aws",
         "source.address": "192.0.2.1",
         "source.ip": "192.0.2.1",
+        "tags": [
+            "forwarded"
+        ],
         "user.id": "AIDAQRSTUVWXYZEXAMPLE:devdsk",
         "user_agent.device.name": "Spider",
         "user_agent.name": "aws-cli",

diff --git a/x-pack/filebeat/module/aws/cloudtrail/test/delete-group-json.log-expected.json b/x-pack/filebeat/module/aws/cloudtrail/test/delete-group-json.log-expected.json
@@ -34,6 +34,9 @@
         "service.type": "aws",
         "source.address": "127.0.0.1",
         "source.ip": "127.0.0.1",
+        "tags": [
+            "forwarded"
+        ],
         "user.id": "0123456789012",
         "user.name": "Alice",
         "user_agent.device.name": "Other",
@@ -74,6 +77,9 @@
         "service.type": "aws",
         "source.address": "127.0.0.1",
         "source.ip": "127.0.0.1",
+        "tags": [
+            "forwarded"
+        ],
         "user.id": "EXAMPLE_PRINCIPLE",
         "user.name": "Alice",
         "user_agent.device.name": "Spider",

diff --git a/x-pack/filebeat/module/aws/cloudtrail/test/delete-ssh-public-key-json.log-expected.json b/x-pack/filebeat/module/aws/cloudtrail/test/delete-ssh-public-key-json.log-expected.json
@@ -37,6 +37,9 @@
         "service.type": "aws",
         "source.address": "127.0.0.1",
         "source.ip": "127.0.0.1",
+        "tags": [
+            "forwarded"
+        ],
         "user.id": "EXAMPLE_ID",
         "user.name": "Alice",
         "user_agent.device.name": "Other",

diff --git a/x-pack/filebeat/module/aws/cloudtrail/test/delete-trail-json.log-expected.json b/x-pack/filebeat/module/aws/cloudtrail/test/delete-trail-json.log-expected.json
@@ -26,6 +26,9 @@
         "service.type": "aws",
         "source.address": "127.0.0.1",
         "source.ip": "127.0.0.1",
+        "tags": [
+            "forwarded"
+        ],
         "user.id": "EXAMPLE_ID",
         "user.name": "Alice",
         "user_agent.device.name": "Spider",

diff --git a/x-pack/filebeat/module/aws/cloudtrail/test/delete-user-json.log-expected.json b/x-pack/filebeat/module/aws/cloudtrail/test/delete-user-json.log-expected.json
@@ -37,6 +37,9 @@
         "service.type": "aws",
         "source.address": "127.0.0.1",
         "source.ip": "127.0.0.1",
+        "tags": [
+            "forwarded"
+        ],
         "user.id": "EX_PRINCIPAL_ID",
         "user.name": "Alice",
         "user_agent.device.name": "Other",

diff --git a/x-pack/filebeat/module/aws/cloudtrail/test/delete-virtual-mfa-device-json.log-expected.json b/x-pack/filebeat/module/aws/cloudtrail/test/delete-virtual-mfa-device-json.log-expected.json
@@ -34,6 +34,9 @@
         "service.type": "aws",
         "source.address": "127.0.0.1",
         "source.ip": "127.0.0.1",
+        "tags": [
+            "forwarded"
+        ],
         "user.id": "EXAMPLE_ID",
         "user.name": "Alice",
         "user_agent.device.name": "Other",

diff --git a/x-pack/filebeat/module/aws/cloudtrail/test/enable-mfa-device-json.log-expected.json b/x-pack/filebeat/module/aws/cloudtrail/test/enable-mfa-device-json.log-expected.json
@@ -36,6 +36,9 @@
         "service.type": "aws",
         "source.address": "127.0.0.1",
         "source.ip": "127.0.0.1",
+        "tags": [
+            "forwarded"
+        ],
         "user.id": "EXAMPLE_ID",
         "user.name": "Alice",
         "user_agent.device.name": "Other",

diff --git a/x-pack/filebeat/module/aws/cloudtrail/test/remove-user-from-group-json.log-expected.json b/x-pack/filebeat/module/aws/cloudtrail/test/remove-user-from-group-json.log-expected.json
@@ -37,6 +37,9 @@
         "service.type": "aws",
         "source.address": "127.0.0.1",
         "source.ip": "127.0.0.1",
+        "tags": [
+            "forwarded"
+        ],
         "user.id": "EXAMPLE_ID",
         "user.name": "Alice",
         "user_agent.device.name": "Other",