Add support for analyzers and multifields

This PR enhances fields.yml to support `analyzer`, `search_analyzer`, `norms` and `multi_fields`. Currently `multi_fields` are only supported for `text` fields. Norms is still disabled by default but can now be overwritten. Below is an example on the usage:

```
fields:
  - name: phrase
    type: text
    multi_fields:
      - name: raw
        type: keyword
      - name: english
        type: text
        analyzer: english
        search_analyzer: englishenhanced
        norms: true
```

CHANGELOG.asciidoc

@@ -58,7 +58,8 @@ https://github.com/elastic/beats/compare/v6.0.0-alpha2...master[Check the HEAD d
 *Affecting all Beats*
 
 - New cli subcommands interface. {pull}4420[4420]
-- Allow source path matching in `add_docker_metadata` processor {pull}4495[4495]
+- Allow source path matching in `add_docker_metadata` processor. {pull}4495[4495]
+- Add support for analyzers and multifields in fields.yml. {pull}4574[4574]
 
 *Filebeat*
 

libbeat/template/field.go

@@ -11,14 +11,18 @@ var (
 )
 
 type Field struct {
-	Name          string `config:"name"`
-	Type          string `config:"type"`
-	Description   string `config:"description"`
-	Format        string `config:"format"`
-	ScalingFactor int    `config:"scaling_factor"`
-	Fields        Fields `config:"fields"`
-	ObjectType    string `config:"object_type"`
-	Enabled       *bool  `config:"enabled"`
+	Name           string `config:"name"`
+	Type           string `config:"type"`
+	Description    string `config:"description"`
+	Format         string `config:"format"`
+	ScalingFactor  int    `config:"scaling_factor"`
+	Fields         Fields `config:"fields"`
+	MultiFields    Fields `config:"multi_fields"`
+	ObjectType     string `config:"object_type"`
+	Enabled        *bool  `config:"enabled"`
+	Analyzer       string `config:"analyzer"`
+	SearchAnalyzer string `config:"search_analyzer"`
+	Norms          bool   `config:"norms"`
 
 	path      string
 	esVersion Version
@@ -94,20 +98,37 @@ func (f *Field) keyword() common.MapStr {
 }
 
 func (f *Field) text() common.MapStr {
-	property := f.getDefaultProperties()
+	properties := f.getDefaultProperties()
 
-	property["type"] = "text"
+	properties["type"] = "text"
 
 	if f.esVersion.IsMajor(2) {
-		property["type"] = "string"
-		property["index"] = "analyzed"
-		property["norms"] = common.MapStr{
-			"enabled": false,
+		properties["type"] = "string"
+		properties["index"] = "analyzed"
+		if !f.Norms {
+			properties["norms"] = common.MapStr{
+				"enabled": false,
+			}
 		}
 	} else {
-		property["norms"] = false
+		if !f.Norms {
+			properties["norms"] = false
+		}
 	}
-	return property
+
+	if f.Analyzer != "" {
+		properties["analyzer"] = f.Analyzer
+	}
+
+	if f.SearchAnalyzer != "" {
+		properties["search_analyzer"] = f.SearchAnalyzer
+	}
+
+	if len(f.MultiFields) > 0 {
+		properties["fields"] = f.MultiFields.process("", f.esVersion)
+	}
+
+	return properties
 }
 
 func (f *Field) array() common.MapStr {
@@ -152,6 +173,16 @@ func (f *Field) addDynamicTemplate(properties common.MapStr, matchType string) {
 	dynamicTemplates = append(dynamicTemplates, template)
 }
 
+func (f *Field) getDefaultProperties() common.MapStr {
+	// Currently no defaults exist
+	property := common.MapStr{}
+	if f.Enabled != nil {
+		property["enabled"] = *f.Enabled
+	}
+
+	return property
+}
+
 // Recursively generates the correct key based on the dots
 // The mapping requires "properties" between each layer. This is added here.
 func generateKey(key string) string {
@@ -161,12 +192,3 @@ func generateKey(key string) string {
 	}
 	return key
 }
-
-func (f *Field) getDefaultProperties() common.MapStr {
-	// Currently no defaults exist
-	property := common.MapStr{}
-	if f.Enabled != nil {
-		property["enabled"] = *f.Enabled
-	}
-	return property
-}

libbeat/template/field_test.go

@@ -65,6 +65,73 @@ func TestField(t *testing.T) {
 				"enabled": false,
 			},
 		},
+		{
+			field:  Field{Type: "text", Analyzer: "autocomplete"},
+			method: func(f Field) common.MapStr { return f.text() },
+			output: common.MapStr{
+				"type":     "text",
+				"analyzer": "autocomplete",
+				"norms":    false,
+			},
+		},
+		{
+			field:  Field{Type: "text", Analyzer: "autocomplete", Norms: true},
+			method: func(f Field) common.MapStr { return f.text() },
+			output: common.MapStr{
+				"type":     "text",
+				"analyzer": "autocomplete",
+			},
+		},
+		{
+			field:  Field{Type: "text", SearchAnalyzer: "standard", Norms: true},
+			method: func(f Field) common.MapStr { return f.text() },
+			output: common.MapStr{
+				"type":            "text",
+				"search_analyzer": "standard",
+			},
+		},
+		{
+			field:  Field{Type: "text", Analyzer: "autocomplete", SearchAnalyzer: "standard", Norms: true},
+			method: func(f Field) common.MapStr { return f.text() },
+			output: common.MapStr{
+				"type":            "text",
+				"analyzer":        "autocomplete",
+				"search_analyzer": "standard",
+			},
+		},
+		{
+			field:  Field{Type: "text", MultiFields: Fields{Field{Name: "raw", Type: "keyword"}}, Norms: true},
+			method: func(f Field) common.MapStr { return f.text() },
+			output: common.MapStr{
+				"type": "text",
+				"fields": common.MapStr{
+					"raw": common.MapStr{
+						"type":         "keyword",
+						"ignore_above": 1024,
+					},
+				},
+			},
+		},
+		{
+			field: Field{Type: "text", MultiFields: Fields{
+				Field{Name: "raw", Type: "keyword"},
+				Field{Name: "indexed", Type: "text"},
+			}, Norms: true},
+			method: func(f Field) common.MapStr { return f.text() },
+			output: common.MapStr{
+				"type": "text",
+				"fields": common.MapStr{
+					"raw": common.MapStr{
+						"type":         "keyword",
+						"ignore_above": 1024,
+					},
+					"indexed": common.MapStr{
+						"type":  "text",
+						"norms": false,
+					},
+				},
+			},
+		},
 	}
 
 	for _, test := range tests {