Allow multiple object types per field.

CHANGELOG-developer.next.asciidoc

@@ -23,3 +23,5 @@ The list below covers the major changes between 7.0.0-alpha2 and master only.
 ==== Bugfixes
 
 ==== Added
+
+- Allow multiple object type configurations per field. {pull}9772[9772]

libbeat/common/field.go

@@ -21,6 +21,8 @@ import (
 	"fmt"
 	"strings"
 
+	"github.com/pkg/errors"
+
 	"github.com/elastic/go-ucfg/yaml"
 )
 
@@ -34,25 +36,27 @@ import (
 type Fields []Field
 
 type Field struct {
-	Name                  string      `config:"name"`
-	Type                  string      `config:"type"`
-	Description           string      `config:"description"`
-	Format                string      `config:"format"`
-	ScalingFactor         int         `config:"scaling_factor"`
-	Fields                Fields      `config:"fields"`
-	MultiFields           Fields      `config:"multi_fields"`
-	ObjectType            string      `config:"object_type"`
-	ObjectTypeMappingType string      `config:"object_type_mapping_type"`
-	Enabled               *bool       `config:"enabled"`
-	Analyzer              string      `config:"analyzer"`
-	SearchAnalyzer        string      `config:"search_analyzer"`
-	Norms                 bool        `config:"norms"`
-	Dynamic               DynamicType `config:"dynamic"`
-	Index                 *bool       `config:"index"`
-	DocValues             *bool       `config:"doc_values"`
-	CopyTo                string      `config:"copy_to"`
-	IgnoreAbove           int         `config:"ignore_above"`
-	AliasPath             string      `config:"path"`
+	Name           string      `config:"name"`
+	Type           string      `config:"type"`
+	Description    string      `config:"description"`
+	Format         string      `config:"format"`
+	Fields         Fields      `config:"fields"`
+	MultiFields    Fields      `config:"multi_fields"`
+	Enabled        *bool       `config:"enabled"`
+	Analyzer       string      `config:"analyzer"`
+	SearchAnalyzer string      `config:"search_analyzer"`
+	Norms          bool        `config:"norms"`
+	Dynamic        DynamicType `config:"dynamic"`
+	Index          *bool       `config:"index"`
+	DocValues      *bool       `config:"doc_values"`
+	CopyTo         string      `config:"copy_to"`
+	IgnoreAbove    int         `config:"ignore_above"`
+	AliasPath      string      `config:"path"`
+
+	ObjectType            string          `config:"object_type"`
+	ObjectTypeMappingType string          `config:"object_type_mapping_type"`
+	ScalingFactor         int             `config:"scaling_factor"`
+	ObjectTypeParams      []ObjectTypeCfg `config:"object_type_params"`
 
 	// Kibana specific
 	Analyzed     *bool  `config:"analyzed"`
@@ -73,6 +77,13 @@ type Field struct {
 	Path      string
 }
 
+// ObjectTypeCfg defines type and configuration of object attributes
+type ObjectTypeCfg struct {
+	ObjectType            string `config:"object_type"`
+	ObjectTypeMappingType string `config:"object_type_mapping_type"`
+	ScalingFactor         int    `config:"scaling_factor"`
+}
+
 type VersionizedString struct {
 	MinVersion string `config:"min_version"`
 	Value      string `config:"value"`
@@ -94,6 +105,17 @@ func (d *DynamicType) Unpack(s string) error {
 	return nil
 }
 
+// Validate ensures objectTypeParams are not mixed with top level objectType configuration
+func (f *Field) Validate() error {
+	if len(f.ObjectTypeParams) == 0 {
+		return nil
+	}
+	if f.ScalingFactor != 0 || f.ObjectTypeMappingType != "" || f.ObjectType != "" {
+		return errors.New("mixing top level objectType configuration with array of object type configurations is forbidden")
+	}
+	return nil
+}
+
 func LoadFieldsYaml(path string) (Fields, error) {
 	keys := []Field{}
 

libbeat/common/field_test.go

@@ -20,6 +20,8 @@ package common
 import (
 	"testing"
 
+	"github.com/stretchr/testify/require"
+
 	"github.com/stretchr/testify/assert"
 
 	"github.com/elastic/go-ucfg/yaml"
@@ -189,3 +191,59 @@ func TestGetKeys(t *testing.T) {
 		assert.Equal(t, test.keys, test.fields.GetKeys())
 	}
 }
+
+func TestFieldValidate(t *testing.T) {
+	tests := []struct {
+		cfg   MapStr
+		field Field
+		err   bool
+		name  string
+	}{
+		{
+			cfg:   MapStr{"object_type": "scaled_float", "object_type_mapping_type": "float", "scaling_factor": 10},
+			field: Field{ObjectType: "scaled_float", ObjectTypeMappingType: "float", ScalingFactor: 10},
+			err:   false,
+			name:  "top level object type config",
+		}, {
+			cfg: MapStr{"object_type_params": []MapStr{
+				{"object_type": "scaled_float", "object_type_mapping_type": "float", "scaling_factor": 100}}},
+			field: Field{ObjectTypeParams: []ObjectTypeCfg{{ObjectType: "scaled_float", ObjectTypeMappingType: "float", ScalingFactor: 100}}},
+			err:   false,
+			name:  "multiple object type configs",
+		}, {
+			cfg: MapStr{
+				"object_type":        "scaled_float",
+				"object_type_params": []MapStr{{"object_type": "scaled_float", "object_type_mapping_type": "float"}}},
+			err:  true,
+			name: "invalid config mixing object_type and object_type_params",
+		}, {
+			cfg: MapStr{
+				"object_type_mapping_type": "float",
+				"object_type_params":       []MapStr{{"object_type": "scaled_float", "object_type_mapping_type": "float"}}},
+			err:  true,
+			name: "invalid config mixing object_type_mapping_type and object_type_params",
+		}, {
+			cfg: MapStr{
+				"scaling_factor":     100,
+				"object_type_params": []MapStr{{"object_type": "scaled_float", "object_type_mapping_type": "float"}}},
+			err:  true,
+			name: "invalid config mixing scaling_factor and object_type_params",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			cfg, err := NewConfigFrom(test.cfg)
+			require.NoError(t, err)
+			var f Field
+			err = cfg.Unpack(&f)
+			if test.err {
+				assert.Error(t, err)
+			} else {
+				assert.NoError(t, err)
+				assert.Equal(t, test.field, f)
+			}
+		})
+	}
+
+}

libbeat/template/processor.go

@@ -34,6 +34,8 @@ var (
 	defaultIgnoreAbove   = 1024
 )
 
+const scalingFactorKey = "scalingFactor"
+
 // Process recursively processes the given fields and writes the template in the given output
 func (p *Processor) Process(fields common.Fields, path string, output common.MapStr) error {
 	for _, field := range fields {
@@ -119,20 +121,28 @@ func (p *Processor) integer(f *common.Field) common.MapStr {
 	return property
 }
 
-func (p *Processor) scaledFloat(f *common.Field) common.MapStr {
+func (p *Processor) scaledFloat(f *common.Field, params ...common.MapStr) common.MapStr {
 	property := getDefaultProperties(f)
 	property["type"] = "scaled_float"
 
 	if p.EsVersion.IsMajor(2) {
 		property["type"] = "float"
-	} else {
-		scalingFactor := f.ScalingFactor
-		// Set default scaling factor
-		if scalingFactor == 0 {
-			scalingFactor = defaultScalingFactor
+		return property
+	}
+
+	// Set scaling factor
+	scalingFactor := defaultScalingFactor
+	if f.ScalingFactor != 0 && len(f.ObjectTypeParams) == 0 {
+		scalingFactor = f.ScalingFactor
+	}
+
+	if len(params) > 0 {
+		if s, ok := params[0][scalingFactorKey].(int); ok && s != 0 {
+			scalingFactor = s
 		}
-		property["scaling_factor"] = scalingFactor
 	}
+
+	property["scaling_factor"] = scalingFactor
 	return property
 }
 
@@ -256,33 +266,43 @@ func (p *Processor) alias(f *common.Field) common.MapStr {
 }
 
 func (p *Processor) object(f *common.Field) common.MapStr {
-	dynProperties := getDefaultProperties(f)
-
-	matchType := func(onlyType string) string {
-		if f.ObjectTypeMappingType != "" {
-			return f.ObjectTypeMappingType
+	matchType := func(onlyType string, mt string) string {
+		if mt != "" {
+			return mt
 		}
 		return onlyType
 	}
 
-	switch f.ObjectType {
-	case "scaled_float":
-		dynProperties = p.scaledFloat(f)
-		addDynamicTemplate(f, dynProperties, matchType("*"))
-	case "text":
-		dynProperties["type"] = "text"
+	var otParams []common.ObjectTypeCfg
+	if len(f.ObjectTypeParams) != 0 {
+		otParams = f.ObjectTypeParams
+	} else {
+		otParams = []common.ObjectTypeCfg{common.ObjectTypeCfg{
+			ObjectType: f.ObjectType, ObjectTypeMappingType: f.ObjectTypeMappingType, ScalingFactor: f.ScalingFactor}}
+	}
 
-		if p.EsVersion.IsMajor(2) {
-			dynProperties["type"] = "string"
-			dynProperties["index"] = "analyzed"
+	for _, otp := range otParams {
+		dynProperties := getDefaultProperties(f)
+
+		switch otp.ObjectType {
+		case "scaled_float":
+			dynProperties = p.scaledFloat(f, common.MapStr{scalingFactorKey: otp.ScalingFactor})
+			addDynamicTemplate(f, dynProperties, matchType("*", otp.ObjectTypeMappingType))
+		case "text":
+			dynProperties["type"] = "text"
+
+			if p.EsVersion.IsMajor(2) {
+				dynProperties["type"] = "string"
+				dynProperties["index"] = "analyzed"
+			}
+			addDynamicTemplate(f, dynProperties, matchType("string", otp.ObjectTypeMappingType))
+		case "keyword":
+			dynProperties["type"] = otp.ObjectType
+			addDynamicTemplate(f, dynProperties, matchType("string", otp.ObjectTypeMappingType))
+		case "byte", "double", "float", "long", "short", "boolean":
+			dynProperties["type"] = otp.ObjectType
+			addDynamicTemplate(f, dynProperties, matchType(otp.ObjectType, otp.ObjectTypeMappingType))
 		}
-		addDynamicTemplate(f, dynProperties, matchType("string"))
-	case "keyword":
-		dynProperties["type"] = f.ObjectType
-		addDynamicTemplate(f, dynProperties, matchType("string"))
-	case "byte", "double", "float", "long", "short":
-		dynProperties["type"] = f.ObjectType
-		addDynamicTemplate(f, dynProperties, matchType(f.ObjectType))
 	}
 
 	properties := getDefaultProperties(f)