Merge pull request #506 from magnusbaeck/fields

Filebeat: Support array and dictionary fields

filebeat/config/config.go

@@ -7,6 +7,7 @@ import (
 	"time"
 
 	"github.com/elastic/beats/libbeat/cfgfile"
+	"github.com/elastic/beats/libbeat/common"
 	"github.com/elastic/beats/libbeat/logp"
 )
 
@@ -52,7 +53,7 @@ type ProspectorConfig struct {
 
 type HarvesterConfig struct {
 	InputType          string `yaml:"input_type"`
-	Fields             map[string]string
+	Fields             common.MapStr
 	FieldsUnderRoot    bool   `yaml:"fields_under_root"`
 	BufferSize         int    `yaml:"harvester_buffer_size"`
 	TailFiles          bool   `yaml:"tail_files"`

filebeat/config/config_test.go

@@ -33,7 +33,7 @@ func TestReadConfig(t *testing.T) {
 	assert.Equal(t, "/var/log/s*.log", prospectors[0].Paths[1])
 	assert.Equal(t, "log", prospectors[0].Input)
 	assert.Equal(t, 3, len(prospectors[0].Harvester.Fields))
-	assert.Equal(t, 1, len(prospectors[0].Harvester.Fields["review"]))
+	assert.Equal(t, "1", prospectors[0].Harvester.Fields["review"])
 	assert.Equal(t, "24h", prospectors[0].IgnoreOlder)
 	assert.Equal(t, "10s", prospectors[0].ScanFrequency)
 

filebeat/docs/configuration.asciidoc

@@ -99,7 +99,8 @@ If both `include_lines` and `exclude_lines` are defined, then include_lines is c
 ===== fields
 
 Optional fields that you can specify to add additional information to the output. For
-example, you might add fields that you can use for filtering log data. By default,
+example, you might add fields that you can use for filtering log data. Fields can be
+scalar values, arrays, dictionaries, or any nested combination of these. All scalar values will be interpreted as strings. By default,
 the fields that you specify here will be grouped under a `fields` sub-dictionary in the output document. To store the custom fields as top-level fields, set the `fields_under_root` option to true.
 
 [source,yaml]

filebeat/input/file.go

@@ -24,7 +24,7 @@ type FileEvent struct {
 	Offset       int64
 	Bytes        int
 	Text         *string
-	Fields       *map[string]string
+	Fields       *common.MapStr
 	Fileinfo     *os.FileInfo
 
 	fieldsUnderRoot bool

filebeat/input/file_test.go

@@ -6,6 +6,7 @@ import (
 	"path/filepath"
 	"testing"
 
+	"github.com/elastic/beats/libbeat/common"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -114,7 +115,7 @@ func TestFileEventToMapStr(t *testing.T) {
 
 func TestFieldsUnderRoot(t *testing.T) {
 	event := FileEvent{
-		Fields: &map[string]string{
+		Fields: &common.MapStr{
 			"hello": "world",
 		},
 	}

libbeat/common/mapstr.go

@@ -85,3 +85,52 @@ func (m MapStr) String() string {
 	}
 	return string(bytes)
 }
+
+// UnmarshalYAML helps out with the YAML unmarshalling when the target
+// variable is a MapStr. The default behavior is to unmarshal nested
+// maps to map[interface{}]interface{} values, and such values can't
+// be marshalled as JSON.
+//
+// The keys of map[interface{}]interface{} maps will be converted to
+// strings with a %v format string, as will any scalar values that
+// aren't already strings (i.e. numbers and boolean values).
+//
+// Since we want to modify the receiver it needs to be a pointer.
+func (ms *MapStr) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	var result map[interface{}]interface{}
+	err := unmarshal(&result)
+	if err != nil {
+		panic(err)
+	}
+	*ms = cleanUpInterfaceMap(result)
+	return nil
+}
+
+func cleanUpInterfaceArray(in []interface{}) []interface{} {
+	result := make([]interface{}, len(in))
+	for i, v := range in {
+		result[i] = cleanUpMapValue(v)
+	}
+	return result
+}
+
+func cleanUpInterfaceMap(in map[interface{}]interface{}) MapStr {
+	result := make(MapStr)
+	for k, v := range in {
+		result[fmt.Sprintf("%v", k)] = cleanUpMapValue(v)
+	}
+	return result
+}
+
+func cleanUpMapValue(v interface{}) interface{} {
+	switch v := v.(type) {
+	case []interface{}:
+		return cleanUpInterfaceArray(v)
+	case map[interface{}]interface{}:
+		return cleanUpInterfaceMap(v)
+	case string:
+		return v
+	default:
+		return fmt.Sprintf("%v", v)
+	}
+}

libbeat/common/mapstr_test.go

@@ -1,10 +1,12 @@
 package common
 
 import (
+	"strings"
 	"testing"
 	"time"
 
 	"github.com/stretchr/testify/assert"
+	"gopkg.in/yaml.v2"
 )
 
 func TestMapStrUpdate(t *testing.T) {
@@ -185,3 +187,84 @@ func TestString(t *testing.T) {
 		assert.Equal(t, test.Output, test.Input.String())
 	}
 }
+
+func TestUnmarshalYAML(t *testing.T) {
+	type io struct {
+		InputLines []string
+		Output     MapStr
+	}
+	tests := []io{
+		// should return nil for empty document
+		{
+			InputLines: []string{},
+			Output:     nil,
+		},
+		// should handle scalar values
+		{
+			InputLines: []string{
+				"a: b",
+				"c: true",
+				"123: 456",
+			},
+			Output: MapStr{
+				"a":   "b",
+				"c":   "true",
+				"123": "456",
+			},
+		},
+		// should handle array with scalar values
+		{
+			InputLines: []string{
+				"a:",
+				"  - b",
+				"  - true",
+				"  - 123",
+			},
+			Output: MapStr{
+				"a": []interface{}{"b", "true", "123"},
+			},
+		},
+		// should handle array with nested map
+		{
+			InputLines: []string{
+				"a:",
+				"  - b: c",
+				"    d: true",
+				"    123: 456",
+			},
+			Output: MapStr{
+				"a": []interface{}{
+					MapStr{
+						"b":   "c",
+						"d":   "true",
+						"123": "456",
+					},
+				},
+			},
+		},
+		// should handle nested map
+		{
+			InputLines: []string{
+				"a: ",
+				"  b: c",
+				"  d: true",
+				"  123: 456",
+			},
+			Output: MapStr{
+				"a": MapStr{
+					"b":   "c",
+					"d":   "true",
+					"123": "456",
+				},
+			},
+		},
+	}
+	for _, test := range tests {
+		var actual MapStr
+		if err := yaml.Unmarshal([]byte(strings.Join(test.InputLines, "\n")), &actual); err != nil {
+			assert.Fail(t, "YAML unmarshaling unexpectedly failed: %s", err)
+			continue
+		}
+		assert.Equal(t, test.Output, actual)
+	}
+}