From f3a4d2edbe17565be8f0c544bc07fab53f49fab6 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Thu, 6 Aug 2020 18:20:20 -0600 Subject: [PATCH 1/9] Change add_host_metadata to not overwrite if host fields already exist --- .../add_host_metadata/add_host_metadata.go | 6 ++ .../add_host_metadata_test.go | 58 +++++++++++++++++++ libbeat/publisher/processing/default.go | 3 + 3 files changed, 67 insertions(+) diff --git a/libbeat/processors/add_host_metadata/add_host_metadata.go b/libbeat/processors/add_host_metadata/add_host_metadata.go index 1cf1bd90503a..9f0fae9b2c0e 100644 --- a/libbeat/processors/add_host_metadata/add_host_metadata.go +++ b/libbeat/processors/add_host_metadata/add_host_metadata.go @@ -81,6 +81,12 @@ func New(cfg *common.Config) (processors.Processor, error) { // Run enriches the given event with the host meta data func (p *addHostMetadata) Run(event *beat.Event) (*beat.Event, error) { + // If host fields exist in event, skip add_host_metadata. + hostFields, _ := event.Fields.GetValue("host") + if hostFields != nil && len(hostFields.(common.MapStr)) != 0 { + return event, nil + } + err := p.loadData() if err != nil { return nil, err diff --git a/libbeat/processors/add_host_metadata/add_host_metadata_test.go b/libbeat/processors/add_host_metadata/add_host_metadata_test.go index 500fc4ba9d8c..cfb692f5a712 100644 --- a/libbeat/processors/add_host_metadata/add_host_metadata_test.go +++ b/libbeat/processors/add_host_metadata/add_host_metadata_test.go @@ -196,3 +196,61 @@ func TestConfigGeoDisabled(t *testing.T) { assert.Error(t, err) assert.Equal(t, nil, eventGeoField) } + +func TestEventWithExistingHostField(t *testing.T) { + hostID := "9C7FAB7B" + event := &beat.Event{ + Fields: common.MapStr{ + "host": common.MapStr{ + "id": hostID, + }, + }, + Timestamp: time.Now(), + } + testConfig, err := common.NewConfigFrom(map[string]interface{}{}) + assert.NoError(t, err) + + p, err := New(testConfig) + switch runtime.GOOS { + case "windows", "darwin", "linux": + assert.NoError(t, err) + default: + assert.IsType(t, types.ErrNotImplemented, err) + return + } + + newEvent, err := p.Run(event) + assert.NoError(t, err) + + v, err := newEvent.GetValue("host") + assert.NoError(t, err) + + hostFields := v.(common.MapStr) + assert.Equal(t, 1, len(hostFields)) + assert.Equal(t, hostID, hostFields["id"]) +} + +func TestEventWithoutHostFields(t *testing.T) { + event := &beat.Event{ + Fields: common.MapStr{}, + Timestamp: time.Now(), + } + testConfig, err := common.NewConfigFrom(map[string]interface{}{}) + assert.NoError(t, err) + + p, err := New(testConfig) + switch runtime.GOOS { + case "windows", "darwin", "linux": + assert.NoError(t, err) + default: + assert.IsType(t, types.ErrNotImplemented, err) + return + } + + newEvent, err := p.Run(event) + assert.NoError(t, err) + + v, err := newEvent.GetValue("host") + assert.NoError(t, err) + assert.True(t, len(v.(common.MapStr)) > 0) +} diff --git a/libbeat/publisher/processing/default.go b/libbeat/publisher/processing/default.go index b2a65642e173..a77ea3d32ccc 100644 --- a/libbeat/publisher/processing/default.go +++ b/libbeat/publisher/processing/default.go @@ -334,6 +334,9 @@ func (b *builder) Create(cfg beat.ProcessingConfig, drop bool) (beat.Processor, // setup 6: add beats and host metadata if meta := builtin; len(meta) > 0 { + // host.name will be added by add_host_metadata processor or modules + // remove host.name from libbeat + meta.Delete("host.name") processors.add(actions.NewAddFields(meta, needsCopy, false)) } From 4f6c33eeaf2ba6d3203ae75ba54aa958fb387c47 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Fri, 7 Aug 2020 10:57:24 -0600 Subject: [PATCH 2/9] undo remove host.name from libbeat --- .../add_host_metadata/add_host_metadata.go | 4 ++-- .../add_host_metadata/add_host_metadata_test.go | 15 +++++++++++---- libbeat/publisher/processing/default.go | 3 --- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/libbeat/processors/add_host_metadata/add_host_metadata.go b/libbeat/processors/add_host_metadata/add_host_metadata.go index 9f0fae9b2c0e..0e6b66ef9b64 100644 --- a/libbeat/processors/add_host_metadata/add_host_metadata.go +++ b/libbeat/processors/add_host_metadata/add_host_metadata.go @@ -81,9 +81,9 @@ func New(cfg *common.Config) (processors.Processor, error) { // Run enriches the given event with the host meta data func (p *addHostMetadata) Run(event *beat.Event) (*beat.Event, error) { - // If host fields exist in event, skip add_host_metadata. + // If host fields exist(besides host.name added by libbeat) in event, skip add_host_metadata. hostFields, _ := event.Fields.GetValue("host") - if hostFields != nil && len(hostFields.(common.MapStr)) != 0 { + if hostFields != nil && len(hostFields.(common.MapStr)) > 1 { return event, nil } diff --git a/libbeat/processors/add_host_metadata/add_host_metadata_test.go b/libbeat/processors/add_host_metadata/add_host_metadata_test.go index cfb692f5a712..5d5f4f48ca7c 100644 --- a/libbeat/processors/add_host_metadata/add_host_metadata_test.go +++ b/libbeat/processors/add_host_metadata/add_host_metadata_test.go @@ -31,6 +31,8 @@ import ( "github.com/elastic/go-sysinfo/types" ) +var hostName = "testHost" + func TestConfigDefault(t *testing.T) { event := &beat.Event{ Fields: common.MapStr{}, @@ -202,7 +204,8 @@ func TestEventWithExistingHostField(t *testing.T) { event := &beat.Event{ Fields: common.MapStr{ "host": common.MapStr{ - "id": hostID, + "name": hostName, + "id": hostID, }, }, Timestamp: time.Now(), @@ -226,13 +229,17 @@ func TestEventWithExistingHostField(t *testing.T) { assert.NoError(t, err) hostFields := v.(common.MapStr) - assert.Equal(t, 1, len(hostFields)) + assert.Equal(t, 2, len(hostFields)) assert.Equal(t, hostID, hostFields["id"]) } func TestEventWithoutHostFields(t *testing.T) { event := &beat.Event{ - Fields: common.MapStr{}, + Fields: common.MapStr{ + "host": common.MapStr{ + "name": hostName, + }, + }, Timestamp: time.Now(), } testConfig, err := common.NewConfigFrom(map[string]interface{}{}) @@ -252,5 +259,5 @@ func TestEventWithoutHostFields(t *testing.T) { v, err := newEvent.GetValue("host") assert.NoError(t, err) - assert.True(t, len(v.(common.MapStr)) > 0) + assert.True(t, len(v.(common.MapStr)) > 1) } diff --git a/libbeat/publisher/processing/default.go b/libbeat/publisher/processing/default.go index a77ea3d32ccc..b2a65642e173 100644 --- a/libbeat/publisher/processing/default.go +++ b/libbeat/publisher/processing/default.go @@ -334,9 +334,6 @@ func (b *builder) Create(cfg beat.ProcessingConfig, drop bool) (beat.Processor, // setup 6: add beats and host metadata if meta := builtin; len(meta) > 0 { - // host.name will be added by add_host_metadata processor or modules - // remove host.name from libbeat - meta.Delete("host.name") processors.add(actions.NewAddFields(meta, needsCopy, false)) } From 1e8e6d54d2013e20fa63dd1004390ddb0efc1696 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Mon, 10 Aug 2020 08:36:58 -0600 Subject: [PATCH 3/9] add changelog --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index f818a846ac8b..49f688928598 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -19,6 +19,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Make error message about locked data path actionable. {pull}18667[18667] - Ensure dynamic template names are unique for the same field. {pull}18849[18849] - Remove the deprecated `xpack.monitoring.*` settings. Going forward only `monitoring.*` settings may be used. {issue}9424[9424] {pull}18608[18608] +- Change add_host_metadata to not overwrite if host fields already exist. {pull}20490[20490] {issue}20464[20464] *Auditbeat* From dee70c9e88178f73fb43c4628931a9c0d0ca63eb Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Mon, 10 Aug 2020 11:40:13 -0600 Subject: [PATCH 4/9] update add_host_metadata doc --- .../add_host_metadata/docs/add_host_metadata.asciidoc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/libbeat/processors/add_host_metadata/docs/add_host_metadata.asciidoc b/libbeat/processors/add_host_metadata/docs/add_host_metadata.asciidoc index 0c71f10d2005..237078e8ec29 100644 --- a/libbeat/processors/add_host_metadata/docs/add_host_metadata.asciidoc +++ b/libbeat/processors/add_host_metadata/docs/add_host_metadata.asciidoc @@ -75,3 +75,6 @@ The fields added to the event look like the following: } } ------------------------------------------------------------------------------- + +Note: `add_host_metadata` processor will not overwrite host fields if `host.*` +fields already exist in the event from Beats. From 8d8b4e7f55527ad01d5bb8b127355ebfd0775755 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Tue, 11 Aug 2020 11:07:53 -0600 Subject: [PATCH 5/9] remove host.name and check host field length --- .../add_host_metadata/add_host_metadata.go | 13 +++++++++---- .../add_host_metadata/add_host_metadata_test.go | 2 +- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/libbeat/processors/add_host_metadata/add_host_metadata.go b/libbeat/processors/add_host_metadata/add_host_metadata.go index 0e6b66ef9b64..a206143ef596 100644 --- a/libbeat/processors/add_host_metadata/add_host_metadata.go +++ b/libbeat/processors/add_host_metadata/add_host_metadata.go @@ -82,12 +82,17 @@ func New(cfg *common.Config) (processors.Processor, error) { // Run enriches the given event with the host meta data func (p *addHostMetadata) Run(event *beat.Event) (*beat.Event, error) { // If host fields exist(besides host.name added by libbeat) in event, skip add_host_metadata. - hostFields, _ := event.Fields.GetValue("host") - if hostFields != nil && len(hostFields.(common.MapStr)) > 1 { - return event, nil + hostFields, err := event.Fields.GetValue("host") + if err == nil && hostFields != nil && len(hostFields.(common.MapStr)) > 1 { + hostsClone := hostFields.(common.MapStr).Clone() + err := hostsClone.Delete("name") + // remove host.name field and then check if there are other host fields exist + if err == nil && len(hostsClone) > 0 { + return event, nil + } } - err := p.loadData() + err = p.loadData() if err != nil { return nil, err } diff --git a/libbeat/processors/add_host_metadata/add_host_metadata_test.go b/libbeat/processors/add_host_metadata/add_host_metadata_test.go index 5d5f4f48ca7c..c18a292c606c 100644 --- a/libbeat/processors/add_host_metadata/add_host_metadata_test.go +++ b/libbeat/processors/add_host_metadata/add_host_metadata_test.go @@ -233,7 +233,7 @@ func TestEventWithExistingHostField(t *testing.T) { assert.Equal(t, hostID, hostFields["id"]) } -func TestEventWithoutHostFields(t *testing.T) { +func TestEventWithOnlyHostNameField(t *testing.T) { event := &beat.Event{ Fields: common.MapStr{ "host": common.MapStr{ From 569c3496d9bd10b1c05ca6c3c9f4dd123f0d3874 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Wed, 12 Aug 2020 20:50:21 -0600 Subject: [PATCH 6/9] add replace_host_fields config param --- .../add_host_metadata/add_host_metadata.go | 26 ++- .../add_host_metadata_test.go | 162 ++++++++++++++---- .../processors/add_host_metadata/config.go | 14 +- .../docs/add_host_metadata.asciidoc | 4 +- 4 files changed, 155 insertions(+), 51 deletions(-) diff --git a/libbeat/processors/add_host_metadata/add_host_metadata.go b/libbeat/processors/add_host_metadata/add_host_metadata.go index a206143ef596..b505df758ba8 100644 --- a/libbeat/processors/add_host_metadata/add_host_metadata.go +++ b/libbeat/processors/add_host_metadata/add_host_metadata.go @@ -81,18 +81,26 @@ func New(cfg *common.Config) (processors.Processor, error) { // Run enriches the given event with the host meta data func (p *addHostMetadata) Run(event *beat.Event) (*beat.Event, error) { - // If host fields exist(besides host.name added by libbeat) in event, skip add_host_metadata. - hostFields, err := event.Fields.GetValue("host") - if err == nil && hostFields != nil && len(hostFields.(common.MapStr)) > 1 { - hostsClone := hostFields.(common.MapStr).Clone() - err := hostsClone.Delete("name") - // remove host.name field and then check if there are other host fields exist - if err == nil && len(hostsClone) > 0 { - return event, nil + // check replace_host_fields field + if !p.config.ReplaceHostFields { + // If host fields exist(besides host.name added by libbeat) in event, skip add_host_metadata. + hostFields, err := event.Fields.GetValue("host") + if err == nil && hostFields != nil && len(hostFields.(common.MapStr)) >= 1 { + if hasName, _ := hostFields.(common.MapStr).HasKey("name"); hasName { + // other host fields exist on top of host.name, skip add_host_metadata. + if len(hostFields.(common.MapStr)) > 1 { + return event, nil + } + } else { + // host.name does not exist but other host fields exist, skip add_host_metadata. + if len(hostFields.(common.MapStr)) > 0 { + return event, nil + } + } } } - err = p.loadData() + err := p.loadData() if err != nil { return nil, err } diff --git a/libbeat/processors/add_host_metadata/add_host_metadata_test.go b/libbeat/processors/add_host_metadata/add_host_metadata_test.go index c18a292c606c..364a64313be0 100644 --- a/libbeat/processors/add_host_metadata/add_host_metadata_test.go +++ b/libbeat/processors/add_host_metadata/add_host_metadata_test.go @@ -31,7 +31,10 @@ import ( "github.com/elastic/go-sysinfo/types" ) -var hostName = "testHost" +var ( + hostName = "testHost" + hostID = "9C7FAB7B" +) func TestConfigDefault(t *testing.T) { event := &beat.Event{ @@ -199,18 +202,10 @@ func TestConfigGeoDisabled(t *testing.T) { assert.Equal(t, nil, eventGeoField) } -func TestEventWithExistingHostField(t *testing.T) { - hostID := "9C7FAB7B" - event := &beat.Event{ - Fields: common.MapStr{ - "host": common.MapStr{ - "name": hostName, - "id": hostID, - }, - }, - Timestamp: time.Now(), - } - testConfig, err := common.NewConfigFrom(map[string]interface{}{}) +func TestEventWithReplaceHostFieldsFalse(t *testing.T) { + cfg := map[string]interface{}{} + cfg["replace_host_fields"] = false + testConfig, err := common.NewConfigFrom(cfg) assert.NoError(t, err) p, err := New(testConfig) @@ -222,27 +217,75 @@ func TestEventWithExistingHostField(t *testing.T) { return } - newEvent, err := p.Run(event) - assert.NoError(t, err) + cases := []struct { + title string + event beat.Event + hostLengthLargerThanOne bool + hostLengthEqualsToOne bool + expectedHostFieldLength int + }{ + { + "replace_host_fields=false with only host.name", + beat.Event{ + Fields: common.MapStr{ + "host": common.MapStr{ + "name": hostName, + }, + }, + }, + true, + false, + -1, + }, + { + "replace_host_fields=false with only host.id", + beat.Event{ + Fields: common.MapStr{ + "host": common.MapStr{ + "id": hostID, + }, + }, + }, + false, + true, + 1, + }, + { + "replace_host_fields=false with host.name and host.id", + beat.Event{ + Fields: common.MapStr{ + "host": common.MapStr{ + "name": hostName, + "id": hostID, + }, + }, + }, + true, + false, + 2, + }, + } - v, err := newEvent.GetValue("host") - assert.NoError(t, err) + for _, c := range cases { + t.Run(c.title, func(t *testing.T) { + newEvent, err := p.Run(&c.event) + assert.NoError(t, err) - hostFields := v.(common.MapStr) - assert.Equal(t, 2, len(hostFields)) - assert.Equal(t, hostID, hostFields["id"]) + v, err := newEvent.GetValue("host") + assert.NoError(t, err) + assert.Equal(t, c.hostLengthLargerThanOne, len(v.(common.MapStr)) > 1) + assert.Equal(t, c.hostLengthEqualsToOne, len(v.(common.MapStr)) == 1) + if c.expectedHostFieldLength != -1 { + assert.Equal(t, c.expectedHostFieldLength, len(v.(common.MapStr))) + } + }) + } } -func TestEventWithOnlyHostNameField(t *testing.T) { - event := &beat.Event{ - Fields: common.MapStr{ - "host": common.MapStr{ - "name": hostName, - }, - }, - Timestamp: time.Now(), - } - testConfig, err := common.NewConfigFrom(map[string]interface{}{}) +func TestEventWithReplaceHostFieldsTrue(t *testing.T) { + cfg := map[string]interface{}{} + cfg["replace_host_fields"] = true + testConfig, err := common.NewConfigFrom(cfg) assert.NoError(t, err) p, err := New(testConfig) @@ -254,10 +297,59 @@ func TestEventWithOnlyHostNameField(t *testing.T) { return } - newEvent, err := p.Run(event) - assert.NoError(t, err) + cases := []struct { + title string + event beat.Event + hostLengthLargerThanOne bool + hostLengthEqualsToOne bool + }{ + { + "replace_host_fields=true with host.name", + beat.Event{ + Fields: common.MapStr{ + "host": common.MapStr{ + "name": hostName, + }, + }, + }, + true, + false, + }, + { + "replace_host_fields=true with host.id", + beat.Event{ + Fields: common.MapStr{ + "host": common.MapStr{ + "id": hostID, + }, + }, + }, + true, + false, + }, + { + "replace_host_fields=true with host.name and host.id", + beat.Event{ + Fields: common.MapStr{ + "host": common.MapStr{ + "name": hostName, + "id": hostID, + }, + }, + }, + true, + false, + }, + } + for _, c := range cases { + t.Run(c.title, func(t *testing.T) { + newEvent, err := p.Run(&c.event) + assert.NoError(t, err) - v, err := newEvent.GetValue("host") - assert.NoError(t, err) - assert.True(t, len(v.(common.MapStr)) > 1) + v, err := newEvent.GetValue("host") + assert.NoError(t, err) + assert.Equal(t, c.hostLengthLargerThanOne, len(v.(common.MapStr)) > 1) + assert.Equal(t, c.hostLengthEqualsToOne, len(v.(common.MapStr)) == 1) + }) + } } diff --git a/libbeat/processors/add_host_metadata/config.go b/libbeat/processors/add_host_metadata/config.go index 81c0452f4d93..7f1e79500b73 100644 --- a/libbeat/processors/add_host_metadata/config.go +++ b/libbeat/processors/add_host_metadata/config.go @@ -25,15 +25,17 @@ import ( // Config for add_host_metadata processor. type Config struct { - NetInfoEnabled bool `config:"netinfo.enabled"` // Add IP and MAC to event - CacheTTL time.Duration `config:"cache.ttl"` - Geo *util.GeoConfig `config:"geo"` - Name string `config:"name"` + NetInfoEnabled bool `config:"netinfo.enabled"` // Add IP and MAC to event + CacheTTL time.Duration `config:"cache.ttl"` + Geo *util.GeoConfig `config:"geo"` + Name string `config:"name"` + ReplaceHostFields bool `config:"replace_host_fields"` // replace existing host fields with add_host_metadata } func defaultConfig() Config { return Config{ - NetInfoEnabled: true, - CacheTTL: 5 * time.Minute, + NetInfoEnabled: true, + CacheTTL: 5 * time.Minute, + ReplaceHostFields: false, } } diff --git a/libbeat/processors/add_host_metadata/docs/add_host_metadata.asciidoc b/libbeat/processors/add_host_metadata/docs/add_host_metadata.asciidoc index 237078e8ec29..1011305af9af 100644 --- a/libbeat/processors/add_host_metadata/docs/add_host_metadata.asciidoc +++ b/libbeat/processors/add_host_metadata/docs/add_host_metadata.asciidoc @@ -42,6 +42,7 @@ It has the following settings: `geo.region_iso_code`:: (Optional) ISO region code. +`replace_host_fields`:: (Optional) Default false. If set to true, original `host.*` fields from the event will be replaced by `host.*` fields from `add_host_metadata`. The `add_host_metadata` processor annotates each event with relevant metadata from the host machine. The fields added to the event look like the following: @@ -77,4 +78,5 @@ The fields added to the event look like the following: ------------------------------------------------------------------------------- Note: `add_host_metadata` processor will not overwrite host fields if `host.*` -fields already exist in the event from Beats. +fields already exist in the event from Beats. Please use `add_observer_metadata` +if the beat is being used to monitor external systems. From ba72694893c0bb55ce4425a001425f79858f8316 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Thu, 13 Aug 2020 10:26:20 -0600 Subject: [PATCH 7/9] change config option to replace_fields and default to true --- .../add_host_metadata/add_host_metadata.go | 38 ++++++---- .../add_host_metadata_test.go | 71 ++++++++++++++++--- .../processors/add_host_metadata/config.go | 16 ++--- .../docs/add_host_metadata.asciidoc | 3 +- 4 files changed, 94 insertions(+), 34 deletions(-) diff --git a/libbeat/processors/add_host_metadata/add_host_metadata.go b/libbeat/processors/add_host_metadata/add_host_metadata.go index b505df758ba8..cbdddde8b240 100644 --- a/libbeat/processors/add_host_metadata/add_host_metadata.go +++ b/libbeat/processors/add_host_metadata/add_host_metadata.go @@ -82,21 +82,9 @@ func New(cfg *common.Config) (processors.Processor, error) { // Run enriches the given event with the host meta data func (p *addHostMetadata) Run(event *beat.Event) (*beat.Event, error) { // check replace_host_fields field - if !p.config.ReplaceHostFields { - // If host fields exist(besides host.name added by libbeat) in event, skip add_host_metadata. - hostFields, err := event.Fields.GetValue("host") - if err == nil && hostFields != nil && len(hostFields.(common.MapStr)) >= 1 { - if hasName, _ := hostFields.(common.MapStr).HasKey("name"); hasName { - // other host fields exist on top of host.name, skip add_host_metadata. - if len(hostFields.(common.MapStr)) > 1 { - return event, nil - } - } else { - // host.name does not exist but other host fields exist, skip add_host_metadata. - if len(hostFields.(common.MapStr)) > 0 { - return event, nil - } - } + if !p.config.ReplaceFields { + if skipAddingHostMetadata(event) { + return event, nil } } @@ -165,3 +153,23 @@ func (p *addHostMetadata) String() string { return fmt.Sprintf("%v=[netinfo.enabled=[%v], cache.ttl=[%v]]", processorName, p.config.NetInfoEnabled, p.config.CacheTTL) } + +func skipAddingHostMetadata(event *beat.Event) bool { + // If host fields exist(besides host.name added by libbeat) in event, skip add_host_metadata. + hostFields, err := event.Fields.GetValue("host") + hostFieldsMap := hostFields.(common.MapStr) + if err == nil && hostFields != nil && len(hostFieldsMap) >= 1 { + if hasName, _ := hostFieldsMap.HasKey("name"); hasName { + // other host fields exist on top of host.name, skip add_host_metadata. + if len(hostFieldsMap) > 1 { + return true + } + } else { + // host.name does not exist but other host fields exist, skip add_host_metadata. + if len(hostFieldsMap) > 0 { + return true + } + } + } + return false +} diff --git a/libbeat/processors/add_host_metadata/add_host_metadata_test.go b/libbeat/processors/add_host_metadata/add_host_metadata_test.go index 364a64313be0..7c38f028a13e 100644 --- a/libbeat/processors/add_host_metadata/add_host_metadata_test.go +++ b/libbeat/processors/add_host_metadata/add_host_metadata_test.go @@ -202,9 +202,9 @@ func TestConfigGeoDisabled(t *testing.T) { assert.Equal(t, nil, eventGeoField) } -func TestEventWithReplaceHostFieldsFalse(t *testing.T) { +func TestEventWithReplaceFieldsFalse(t *testing.T) { cfg := map[string]interface{}{} - cfg["replace_host_fields"] = false + cfg["replace_fields"] = false testConfig, err := common.NewConfigFrom(cfg) assert.NoError(t, err) @@ -225,7 +225,7 @@ func TestEventWithReplaceHostFieldsFalse(t *testing.T) { expectedHostFieldLength int }{ { - "replace_host_fields=false with only host.name", + "replace_fields=false with only host.name", beat.Event{ Fields: common.MapStr{ "host": common.MapStr{ @@ -238,7 +238,7 @@ func TestEventWithReplaceHostFieldsFalse(t *testing.T) { -1, }, { - "replace_host_fields=false with only host.id", + "replace_fields=false with only host.id", beat.Event{ Fields: common.MapStr{ "host": common.MapStr{ @@ -251,7 +251,7 @@ func TestEventWithReplaceHostFieldsFalse(t *testing.T) { 1, }, { - "replace_host_fields=false with host.name and host.id", + "replace_fields=false with host.name and host.id", beat.Event{ Fields: common.MapStr{ "host": common.MapStr{ @@ -282,9 +282,9 @@ func TestEventWithReplaceHostFieldsFalse(t *testing.T) { } } -func TestEventWithReplaceHostFieldsTrue(t *testing.T) { +func TestEventWithReplaceFieldsTrue(t *testing.T) { cfg := map[string]interface{}{} - cfg["replace_host_fields"] = true + cfg["replace_fields"] = true testConfig, err := common.NewConfigFrom(cfg) assert.NoError(t, err) @@ -304,7 +304,7 @@ func TestEventWithReplaceHostFieldsTrue(t *testing.T) { hostLengthEqualsToOne bool }{ { - "replace_host_fields=true with host.name", + "replace_fields=true with host.name", beat.Event{ Fields: common.MapStr{ "host": common.MapStr{ @@ -316,7 +316,7 @@ func TestEventWithReplaceHostFieldsTrue(t *testing.T) { false, }, { - "replace_host_fields=true with host.id", + "replace_fields=true with host.id", beat.Event{ Fields: common.MapStr{ "host": common.MapStr{ @@ -328,7 +328,7 @@ func TestEventWithReplaceHostFieldsTrue(t *testing.T) { false, }, { - "replace_host_fields=true with host.name and host.id", + "replace_fields=true with host.name and host.id", beat.Event{ Fields: common.MapStr{ "host": common.MapStr{ @@ -341,6 +341,7 @@ func TestEventWithReplaceHostFieldsTrue(t *testing.T) { false, }, } + for _, c := range cases { t.Run(c.title, func(t *testing.T) { newEvent, err := p.Run(&c.event) @@ -353,3 +354,53 @@ func TestEventWithReplaceHostFieldsTrue(t *testing.T) { }) } } + +func TestSkipAddingHostMetadata(t *testing.T) { + cases := []struct { + title string + event beat.Event + expectedSkip bool + }{ + { + "event only with host.name", + beat.Event{ + Fields: common.MapStr{ + "host": common.MapStr{ + "name": hostName, + }, + }, + }, + false, + }, + { + "event only with host.id", + beat.Event{ + Fields: common.MapStr{ + "host": common.MapStr{ + "id": hostID, + }, + }, + }, + true, + }, + { + "event with host.name and host.id", + beat.Event{ + Fields: common.MapStr{ + "host": common.MapStr{ + "name": hostName, + "id": hostID, + }, + }, + }, + true, + }, + } + + for _, c := range cases { + t.Run(c.title, func(t *testing.T) { + skip := skipAddingHostMetadata(&c.event) + assert.Equal(t, c.expectedSkip, skip) + }) + } +} diff --git a/libbeat/processors/add_host_metadata/config.go b/libbeat/processors/add_host_metadata/config.go index 7f1e79500b73..36f4a0a13f85 100644 --- a/libbeat/processors/add_host_metadata/config.go +++ b/libbeat/processors/add_host_metadata/config.go @@ -25,17 +25,17 @@ import ( // Config for add_host_metadata processor. type Config struct { - NetInfoEnabled bool `config:"netinfo.enabled"` // Add IP and MAC to event - CacheTTL time.Duration `config:"cache.ttl"` - Geo *util.GeoConfig `config:"geo"` - Name string `config:"name"` - ReplaceHostFields bool `config:"replace_host_fields"` // replace existing host fields with add_host_metadata + NetInfoEnabled bool `config:"netinfo.enabled"` // Add IP and MAC to event + CacheTTL time.Duration `config:"cache.ttl"` + Geo *util.GeoConfig `config:"geo"` + Name string `config:"name"` + ReplaceFields bool `config:"replace_fields"` // replace existing host fields with add_host_metadata } func defaultConfig() Config { return Config{ - NetInfoEnabled: true, - CacheTTL: 5 * time.Minute, - ReplaceHostFields: false, + NetInfoEnabled: true, + CacheTTL: 5 * time.Minute, + ReplaceFields: true, } } diff --git a/libbeat/processors/add_host_metadata/docs/add_host_metadata.asciidoc b/libbeat/processors/add_host_metadata/docs/add_host_metadata.asciidoc index 1011305af9af..8c8dc3500560 100644 --- a/libbeat/processors/add_host_metadata/docs/add_host_metadata.asciidoc +++ b/libbeat/processors/add_host_metadata/docs/add_host_metadata.asciidoc @@ -42,7 +42,8 @@ It has the following settings: `geo.region_iso_code`:: (Optional) ISO region code. -`replace_host_fields`:: (Optional) Default false. If set to true, original `host.*` fields from the event will be replaced by `host.*` fields from `add_host_metadata`. +`replace_host_fields`:: (Optional) Default true. If set to false, original host +fields from the event will not be replaced by host fields from `add_host_metadata`. The `add_host_metadata` processor annotates each event with relevant metadata from the host machine. The fields added to the event look like the following: From e97143966f54aff85ebe37ab2539ed52cb55f6ff Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Thu, 13 Aug 2020 10:45:49 -0600 Subject: [PATCH 8/9] update changelog --- CHANGELOG.next.asciidoc | 3 +-- .../add_host_metadata/docs/add_host_metadata.asciidoc | 10 ++++++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 28fb0fe4beab..7bd3464ab9a2 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -19,7 +19,6 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Make error message about locked data path actionable. {pull}18667[18667] - Ensure dynamic template names are unique for the same field. {pull}18849[18849] - Remove the deprecated `xpack.monitoring.*` settings. Going forward only `monitoring.*` settings may be used. {issue}9424[9424] {pull}18608[18608] -- Change add_host_metadata to not overwrite if host fields already exist. {pull}20490[20490] {issue}20464[20464] *Auditbeat* @@ -383,7 +382,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Set index.max_docvalue_fields_search in index template to increase value to 200 fields. {issue}20215[20215] - Add leader election for Kubernetes autodiscover. {pull}20281[20281] - Add capability of enriching process metadata with contianer id also for non-privileged containers in `add_process_metadata` processor. {pull}19767[19767] - +- Add replace_fields config option in add_host_metadata for replacing host fields. {pull}20490[20490] {issue}20464[20464] *Auditbeat* diff --git a/libbeat/processors/add_host_metadata/docs/add_host_metadata.asciidoc b/libbeat/processors/add_host_metadata/docs/add_host_metadata.asciidoc index 8c8dc3500560..21d308b23c1d 100644 --- a/libbeat/processors/add_host_metadata/docs/add_host_metadata.asciidoc +++ b/libbeat/processors/add_host_metadata/docs/add_host_metadata.asciidoc @@ -42,7 +42,7 @@ It has the following settings: `geo.region_iso_code`:: (Optional) ISO region code. -`replace_host_fields`:: (Optional) Default true. If set to false, original host +`replace_fields`:: (Optional) Default true. If set to false, original host fields from the event will not be replaced by host fields from `add_host_metadata`. The `add_host_metadata` processor annotates each event with relevant metadata from the host machine. @@ -78,6 +78,8 @@ The fields added to the event look like the following: } ------------------------------------------------------------------------------- -Note: `add_host_metadata` processor will not overwrite host fields if `host.*` -fields already exist in the event from Beats. Please use `add_observer_metadata` -if the beat is being used to monitor external systems. +Note: `add_host_metadata` processor will overwrite host fields if `host.*` +fields already exist in the event from Beats by default with `replace_fields` +equals to `true`. +Please use `add_observer_metadata` if the beat is being used to monitor external +systems. From f25052ac747fd6c655fdfda2f89f78d839419cc6 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Thu, 13 Aug 2020 16:04:45 -0600 Subject: [PATCH 9/9] Simplify skipAddingHostMetadata and add unit test for no host field case --- .../add_host_metadata/add_host_metadata.go | 31 +++++++++---------- .../add_host_metadata_test.go | 7 +++++ 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/libbeat/processors/add_host_metadata/add_host_metadata.go b/libbeat/processors/add_host_metadata/add_host_metadata.go index cbdddde8b240..2c419f840b94 100644 --- a/libbeat/processors/add_host_metadata/add_host_metadata.go +++ b/libbeat/processors/add_host_metadata/add_host_metadata.go @@ -82,10 +82,8 @@ func New(cfg *common.Config) (processors.Processor, error) { // Run enriches the given event with the host meta data func (p *addHostMetadata) Run(event *beat.Event) (*beat.Event, error) { // check replace_host_fields field - if !p.config.ReplaceFields { - if skipAddingHostMetadata(event) { - return event, nil - } + if !p.config.ReplaceFields && skipAddingHostMetadata(event) { + return event, nil } err := p.loadData() @@ -157,19 +155,18 @@ func (p *addHostMetadata) String() string { func skipAddingHostMetadata(event *beat.Event) bool { // If host fields exist(besides host.name added by libbeat) in event, skip add_host_metadata. hostFields, err := event.Fields.GetValue("host") + + // Don't skip if there are no fields + if err != nil || hostFields == nil { + return false + } + hostFieldsMap := hostFields.(common.MapStr) - if err == nil && hostFields != nil && len(hostFieldsMap) >= 1 { - if hasName, _ := hostFieldsMap.HasKey("name"); hasName { - // other host fields exist on top of host.name, skip add_host_metadata. - if len(hostFieldsMap) > 1 { - return true - } - } else { - // host.name does not exist but other host fields exist, skip add_host_metadata. - if len(hostFieldsMap) > 0 { - return true - } - } + // or if "name" is the only field, don't skip + hasName, _ := hostFieldsMap.HasKey("name") + if hasName && len(hostFieldsMap) == 1 { + return false } - return false + + return true } diff --git a/libbeat/processors/add_host_metadata/add_host_metadata_test.go b/libbeat/processors/add_host_metadata/add_host_metadata_test.go index 7c38f028a13e..158cc28cfbc9 100644 --- a/libbeat/processors/add_host_metadata/add_host_metadata_test.go +++ b/libbeat/processors/add_host_metadata/add_host_metadata_test.go @@ -395,6 +395,13 @@ func TestSkipAddingHostMetadata(t *testing.T) { }, true, }, + { + "event without host field", + beat.Event{ + Fields: common.MapStr{}, + }, + false, + }, } for _, c := range cases {