From b20374ffed3d0683488fdd23137deb020e5b3618 Mon Sep 17 00:00:00 2001 From: Adrian Serrano Date: Tue, 9 Jun 2020 17:54:55 +0200 Subject: [PATCH] Cherry-pick #18948 to 7.x: Filebeat: Fix o365 module issues (#19061) * Filebeat: Fix o365 module issues (#18948) - Fix scary data-loss warning on startup - Avoid API errors being processed by the JS pipeline - Fix dissect error about overiding client.port - Fix module passing API settings to the input - Document max_period using the right units (cherry picked from commit 83bbd57548e71272597e0bd846eeeee14f817f22) --- CHANGELOG.next.asciidoc | 1 + x-pack/filebeat/filebeat.reference.yml | 2 +- x-pack/filebeat/input/o365audit/state.go | 5 +---- x-pack/filebeat/input/o365audit/state_test.go | 8 ++------ x-pack/filebeat/module/o365/_meta/config.yml | 2 +- x-pack/filebeat/module/o365/audit/config/input.yml | 3 ++- x-pack/filebeat/module/o365/audit/config/pipeline.js | 5 ++++- x-pack/filebeat/modules.d/o365.yml.disabled | 2 +- 8 files changed, 13 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index d23ee3039cc..96de111a207 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -185,6 +185,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fixed ingestion of some Cisco ASA and FTD messages when a hostname was used instead of an IP for NAT fields. {issue}14034[14034] {pull}18376[18376] - Fix `o365.audit` failing to ingest events when ip address is surrounded by square brackets. {issue}18587[18587] {pull}18591[18591] - Fix Kubernetes Watcher goroutine leaks when input config is invalid and `input.reload` is enabled. {issue}18629[18629] {pull}18630[18630] +- Fix `o365` module ignoring `var.api` settings. {pull}18948[18948] *Heartbeat* diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index 2f59e5ad606..c9d7dbc8f6a 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -852,7 +852,7 @@ filebeat.modules: # authentication_endpoint: "https://login.microsoftonline.us/" # resource: "https://manage.office365.us" # - # max_retention: 7d + # max_retention: 168h # max_requests_per_minute: 2000 # poll_interval: 3m diff --git a/x-pack/filebeat/input/o365audit/state.go b/x-pack/filebeat/input/o365audit/state.go index ecdb8fc89ff..6992437ccab 100644 --- a/x-pack/filebeat/input/o365audit/state.go +++ b/x-pack/filebeat/input/o365audit/state.go @@ -114,10 +114,7 @@ func (s *stateStorage) Load(key stream) (cursor, error) { } cur, err := s.persister.Load(key) if err != nil { - if err != errStateNotFound { - return cur, err - } - cur = newCursor(key, time.Time{}) + return newCursor(key, time.Time{}), err } return cur, s.saveUnsafe(cur) } diff --git a/x-pack/filebeat/input/o365audit/state_test.go b/x-pack/filebeat/input/o365audit/state_test.go index 71b778d16ec..4e274578e83 100644 --- a/x-pack/filebeat/input/o365audit/state_test.go +++ b/x-pack/filebeat/input/o365audit/state_test.go @@ -21,18 +21,14 @@ func TestNoopState(t *testing.T) { t.Run("new state", func(t *testing.T) { st := newStateStorage(noopPersister{}) cur, err := st.Load(myStream) - if !assert.NoError(t, err) { - t.Fatal(err) - } + assert.Equal(t, errStateNotFound, err) empty := newCursor(myStream, time.Time{}) assert.Equal(t, empty, cur) }) t.Run("update state", func(t *testing.T) { st := newStateStorage(noopPersister{}) cur, err := st.Load(myStream) - if !assert.NoError(t, err) { - t.Fatal(err) - } + assert.Equal(t, errStateNotFound, err) advanced := cur.TryAdvance(content{ Type: tn, ID: "1234", diff --git a/x-pack/filebeat/module/o365/_meta/config.yml b/x-pack/filebeat/module/o365/_meta/config.yml index 8114b404aa4..b1a30d6dbe9 100644 --- a/x-pack/filebeat/module/o365/_meta/config.yml +++ b/x-pack/filebeat/module/o365/_meta/config.yml @@ -40,6 +40,6 @@ # authentication_endpoint: "https://login.microsoftonline.us/" # resource: "https://manage.office365.us" # - # max_retention: 7d + # max_retention: 168h # max_requests_per_minute: 2000 # poll_interval: 3m diff --git a/x-pack/filebeat/module/o365/audit/config/input.yml b/x-pack/filebeat/module/o365/audit/config/input.yml index 71e9c9c59f3..bc2a6903e18 100644 --- a/x-pack/filebeat/module/o365/audit/config/input.yml +++ b/x-pack/filebeat/module/o365/audit/config/input.yml @@ -21,7 +21,7 @@ content_type: {{ if .api }} api: {{ range $k, $v := .api }} - - {{ $k }}: {{ $v -}} + {{ $k }}: {{ $v -}} {{ end }} {{ end }} @@ -49,6 +49,7 @@ processors: - 2006-01-02T15:04:05 {{ end }} - script: + when.has_fields: ['o365audit'] lang: javascript id: o365audit_script file: ${path.home}/module/o365/audit/config/pipeline.js diff --git a/x-pack/filebeat/module/o365/audit/config/pipeline.js b/x-pack/filebeat/module/o365/audit/config/pipeline.js index ae8e1a7afe6..8ff5e572fc2 100644 --- a/x-pack/filebeat/module/o365/audit/config/pipeline.js +++ b/x-pack/filebeat/module/o365/audit/config/pipeline.js @@ -729,7 +729,10 @@ function AuditProcessor(tenant_names, debug) { tokenizer: '[%{_ip}]:%{port}', field: 'client.address', target_prefix: 'client', - 'when.contains.client.address': ']:', + 'when.and': [ + {'not.has_fields': ['client._ip', 'client.port']}, + {'contains.client.address': ']:'}, + ], })); builder.Add("extractClientIPv4Port", new processor.Dissect({ tokenizer: '%{_ip}:%{port}', diff --git a/x-pack/filebeat/modules.d/o365.yml.disabled b/x-pack/filebeat/modules.d/o365.yml.disabled index b957965fa75..578ff365df7 100644 --- a/x-pack/filebeat/modules.d/o365.yml.disabled +++ b/x-pack/filebeat/modules.d/o365.yml.disabled @@ -43,6 +43,6 @@ # authentication_endpoint: "https://login.microsoftonline.us/" # resource: "https://manage.office365.us" # - # max_retention: 7d + # max_retention: 168h # max_requests_per_minute: 2000 # poll_interval: 3m