diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 0b9147774c9..43627fd5f38 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -29,6 +29,8 @@ https://github.com/elastic/beats/compare/v5.0.0-beta1...master[Check the HEAD di *Affecting all Beats* +- Fix ignoring all fields from drop_fields in case the first field is unknown. {pull}2685[2685] + *Metricbeat* - Fix default configuration file on Windows to not enabled the `load` metricset. {pull}2632[2632] diff --git a/libbeat/common/mapstr.go b/libbeat/common/mapstr.go index b7add33e1cd..83366e407fb 100644 --- a/libbeat/common/mapstr.go +++ b/libbeat/common/mapstr.go @@ -63,6 +63,7 @@ func (m MapStr) Delete(key string) error { keysLen := len(keyParts) mapp := m + for i := 0; i < keysLen-1; i++ { keyPart := keyParts[i] @@ -75,8 +76,11 @@ func (m MapStr) Delete(key string) error { return fmt.Errorf("unknown key %s", keyPart) } } - delete(mapp, keyParts[keysLen-1]) - return nil + if _, ok := mapp[keyParts[keysLen-1]]; ok { + delete(mapp, keyParts[keysLen-1]) + return nil + } + return fmt.Errorf("unknown key %s", keyParts[keysLen-1]) } func (m MapStr) CopyFieldsTo(to MapStr, key string) error { diff --git a/libbeat/processors/actions/drop_fields.go b/libbeat/processors/actions/drop_fields.go index cdbbd9860c6..7d32c2435ec 100644 --- a/libbeat/processors/actions/drop_fields.go +++ b/libbeat/processors/actions/drop_fields.go @@ -42,14 +42,16 @@ func newDropFields(c common.Config) (processors.Processor, error) { } func (f dropFields) Run(event common.MapStr) (common.MapStr, error) { + errors := []string{} + for _, field := range f.Fields { err := event.Delete(field) if err != nil { - return event, fmt.Errorf("Fail to delete key %s: %s", field, err) + errors = append(errors, err.Error()) } } - return event, nil + return event, fmt.Errorf(strings.Join(errors, ", ")) } func (f dropFields) String() string { diff --git a/libbeat/processors/actions/include_fields.go b/libbeat/processors/actions/include_fields.go index 1ce2e0153ed..8551587f6a1 100644 --- a/libbeat/processors/actions/include_fields.go +++ b/libbeat/processors/actions/include_fields.go @@ -47,22 +47,23 @@ func newIncludeFields(c common.Config) (processors.Processor, error) { func (f includeFields) Run(event common.MapStr) (common.MapStr, error) { filtered := common.MapStr{} + errors := []string{} for _, field := range f.Fields { hasKey, err := event.HasKey(field) if err != nil { - return filtered, fmt.Errorf("Fail to check the key %s: %s", field, err) + errors = append(errors, err.Error()) } if hasKey { errorOnCopy := event.CopyFieldsTo(filtered, field) if errorOnCopy != nil { - return filtered, fmt.Errorf("Fail to copy key %s: %s", field, err) + errors = append(errors, err.Error()) } } } - return filtered, nil + return filtered, fmt.Errorf(strings.Join(errors, ", ")) } func (f includeFields) String() string { diff --git a/libbeat/processors/processor_test.go b/libbeat/processors/processor_test.go index 41061a8887f..048e1ab2000 100644 --- a/libbeat/processors/processor_test.go +++ b/libbeat/processors/processor_test.go @@ -569,3 +569,54 @@ func TestBadConditionConfig(t *testing.T) { assert.NotNil(t, err) } + +func TestDropMissingFields(t *testing.T) { + + yml := []map[string]interface{}{ + map[string]interface{}{ + "drop_fields": map[string]interface{}{ + "fields": []string{"foo.bar", "proc.cpu", "proc.sss", "beat", "mem"}, + }, + }, + } + + processors := GetProcessors(t, yml) + + event := common.MapStr{ + "@timestamp": "2016-01-24T18:35:19.308Z", + "beat": common.MapStr{ + "hostname": "mar", + "name": "my-shipper-1", + }, + + "proc": common.MapStr{ + "cpu": common.MapStr{ + "start_time": "Jan14", + "system": 26027, + "total": 79390, + "total_p": 0, + "user": 53363, + }, + "cmdline": "/sbin/launchd", + }, + "mem": common.MapStr{ + "rss": 11194368, + "rss_p": 0, + "share": 0, + "size": 2555572224, + }, + "type": "process", + } + + processedEvent := processors.Run(event) + + expectedEvent := common.MapStr{ + "@timestamp": "2016-01-24T18:35:19.308Z", + "proc": common.MapStr{ + "cmdline": "/sbin/launchd", + }, + "type": "process", + } + + assert.Equal(t, expectedEvent, processedEvent) +}