From 9e75599caa2f85abea4f2a1c5cc1206bca7fbe4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Thu, 7 Mar 2019 11:00:42 +0100 Subject: [PATCH 1/5] move Field/Fields to asset package from common --- libbeat/{common => asset}/field.go | 2 +- libbeat/{common => asset}/field_test.go | 23 +- libbeat/idxmgmt/idxmgmt_test.go | 3 +- libbeat/kibana/fields_transformer.go | 25 +- libbeat/kibana/fields_transformer_test.go | 269 +++++++++++----------- libbeat/kibana/index_pattern_generator.go | 3 +- libbeat/template/config.go | 4 +- libbeat/template/processor.go | 33 +-- libbeat/template/processor_test.go | 137 +++++------ libbeat/template/template.go | 13 +- 10 files changed, 260 insertions(+), 252 deletions(-) rename libbeat/{common => asset}/field.go (99%) rename libbeat/{common => asset}/field_test.go (93%) diff --git a/libbeat/common/field.go b/libbeat/asset/field.go similarity index 99% rename from libbeat/common/field.go rename to libbeat/asset/field.go index e6f14a50ba5..9df7489046c 100644 --- a/libbeat/common/field.go +++ b/libbeat/asset/field.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package common +package asset import ( "fmt" diff --git a/libbeat/common/field_test.go b/libbeat/asset/field_test.go similarity index 93% rename from libbeat/common/field_test.go rename to libbeat/asset/field_test.go index 31d792cc98b..b5d266c0f12 100644 --- a/libbeat/common/field_test.go +++ b/libbeat/asset/field_test.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package common +package asset import ( "strings" @@ -25,6 +25,7 @@ import ( "github.com/stretchr/testify/assert" + "github.com/elastic/beats/libbeat/common" "github.com/elastic/go-ucfg/yaml" ) @@ -298,44 +299,44 @@ func TestGetField(t *testing.T) { func TestFieldValidate(t *testing.T) { tests := map[string]struct { - cfg MapStr + cfg common.MapStr field Field err bool }{ "top level object type config": { - cfg: MapStr{"object_type": "scaled_float", "object_type_mapping_type": "float", "scaling_factor": 10}, + cfg: common.MapStr{"object_type": "scaled_float", "object_type_mapping_type": "float", "scaling_factor": 10}, field: Field{ObjectType: "scaled_float", ObjectTypeMappingType: "float", ScalingFactor: 10}, err: false, }, "multiple object type configs": { - cfg: MapStr{"object_type_params": []MapStr{ + cfg: common.MapStr{"object_type_params": []common.MapStr{ {"object_type": "scaled_float", "object_type_mapping_type": "float", "scaling_factor": 100}}}, field: Field{ObjectTypeParams: []ObjectTypeCfg{{ObjectType: "scaled_float", ObjectTypeMappingType: "float", ScalingFactor: 100}}}, err: false, }, "invalid config mixing object_type and object_type_params": { - cfg: MapStr{ + cfg: common.MapStr{ "object_type": "scaled_float", - "object_type_params": []MapStr{{"object_type": "scaled_float", "object_type_mapping_type": "float"}}}, + "object_type_params": []common.MapStr{{"object_type": "scaled_float", "object_type_mapping_type": "float"}}}, err: true, }, "invalid config mixing object_type_mapping_type and object_type_params": { - cfg: MapStr{ + cfg: common.MapStr{ "object_type_mapping_type": "float", - "object_type_params": []MapStr{{"object_type": "scaled_float", "object_type_mapping_type": "float"}}}, + "object_type_params": []common.MapStr{{"object_type": "scaled_float", "object_type_mapping_type": "float"}}}, err: true, }, "invalid config mixing scaling_factor and object_type_params": { - cfg: MapStr{ + cfg: common.MapStr{ "scaling_factor": 100, - "object_type_params": []MapStr{{"object_type": "scaled_float", "object_type_mapping_type": "float"}}}, + "object_type_params": []common.MapStr{{"object_type": "scaled_float", "object_type_mapping_type": "float"}}}, err: true, }, } for name, test := range tests { t.Run(name, func(t *testing.T) { - cfg, err := NewConfigFrom(test.cfg) + cfg, err := common.NewConfigFrom(test.cfg) require.NoError(t, err) var f Field err = cfg.Unpack(&f) diff --git a/libbeat/idxmgmt/idxmgmt_test.go b/libbeat/idxmgmt/idxmgmt_test.go index 3d7a841b797..82ed67448ed 100644 --- a/libbeat/idxmgmt/idxmgmt_test.go +++ b/libbeat/idxmgmt/idxmgmt_test.go @@ -25,6 +25,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/elastic/beats/libbeat/asset" "github.com/elastic/beats/libbeat/beat" "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/idxmgmt/ilm" @@ -96,7 +97,7 @@ func TestDefaultSupport_TemplateConfig(t *testing.T) { cloneCfg := func(c template.TemplateConfig) template.TemplateConfig { if c.AppendFields != nil { - tmp := make(common.Fields, len(c.AppendFields)) + tmp := make(asset.Fields, len(c.AppendFields)) copy(tmp, c.AppendFields) c.AppendFields = tmp } diff --git a/libbeat/kibana/fields_transformer.go b/libbeat/kibana/fields_transformer.go index 3e385144c62..6849f9520bd 100644 --- a/libbeat/kibana/fields_transformer.go +++ b/libbeat/kibana/fields_transformer.go @@ -21,13 +21,14 @@ import ( "errors" "fmt" + "github.com/elastic/beats/libbeat/asset" "github.com/elastic/beats/libbeat/common" ) var v640 = common.MustNewVersion("6.4.0") type fieldsTransformer struct { - fields common.Fields + fields asset.Fields transformedFields []common.MapStr transformedFieldFormatMap common.MapStr version *common.Version @@ -35,7 +36,7 @@ type fieldsTransformer struct { migration bool } -func newFieldsTransformer(version *common.Version, fields common.Fields, migration bool) (*fieldsTransformer, error) { +func newFieldsTransformer(version *common.Version, fields asset.Fields, migration bool) (*fieldsTransformer, error) { if version == nil { return nil, errors.New("Version must be given") } @@ -64,10 +65,10 @@ func (t *fieldsTransformer) transform() (transformed common.MapStr, err error) { // add some meta fields truthy := true falsy := false - t.add(common.Field{Path: "_id", Type: "keyword", Index: &falsy, Analyzed: &falsy, DocValues: &falsy, Searchable: &falsy, Aggregatable: &falsy}) - t.add(common.Field{Path: "_type", Type: "keyword", Index: &falsy, Analyzed: &falsy, DocValues: &falsy, Searchable: &truthy, Aggregatable: &truthy}) - t.add(common.Field{Path: "_index", Type: "keyword", Index: &falsy, Analyzed: &falsy, DocValues: &falsy, Searchable: &falsy, Aggregatable: &falsy}) - t.add(common.Field{Path: "_score", Type: "integer", Index: &falsy, Analyzed: &falsy, DocValues: &falsy, Searchable: &falsy, Aggregatable: &falsy}) + t.add(asset.Field{Path: "_id", Type: "keyword", Index: &falsy, Analyzed: &falsy, DocValues: &falsy, Searchable: &falsy, Aggregatable: &falsy}) + t.add(asset.Field{Path: "_type", Type: "keyword", Index: &falsy, Analyzed: &falsy, DocValues: &falsy, Searchable: &truthy, Aggregatable: &truthy}) + t.add(asset.Field{Path: "_index", Type: "keyword", Index: &falsy, Analyzed: &falsy, DocValues: &falsy, Searchable: &falsy, Aggregatable: &falsy}) + t.add(asset.Field{Path: "_score", Type: "integer", Index: &falsy, Analyzed: &falsy, DocValues: &falsy, Searchable: &falsy, Aggregatable: &falsy}) transformed = common.MapStr{ "fields": t.transformedFields, @@ -76,7 +77,7 @@ func (t *fieldsTransformer) transform() (transformed common.MapStr, err error) { return } -func (t *fieldsTransformer) transformFields(commonFields common.Fields, path string) { +func (t *fieldsTransformer) transformFields(commonFields asset.Fields, path string) { for _, f := range commonFields { f.Path = f.Name if path != "" { @@ -119,7 +120,7 @@ func (t *fieldsTransformer) transformFields(commonFields common.Fields, path str } } -func (t *fieldsTransformer) update(target *common.MapStr, override common.Field) error { +func (t *fieldsTransformer) update(target *common.MapStr, override asset.Field) error { field, _ := transformField(t.version, override) if override.Type == "" || (*target)["type"] == field["type"] { target.Update(field) @@ -133,7 +134,7 @@ func (t *fieldsTransformer) update(target *common.MapStr, override common.Field) return fmt.Errorf("field <%s> is duplicated", override.Path) } -func (t *fieldsTransformer) add(f common.Field) { +func (t *fieldsTransformer) add(f asset.Field) { if idx := t.keys[f.Path]; idx > 0 { target := &t.transformedFields[idx-1] // 1-indexed if err := t.update(target, f); err != nil { @@ -150,7 +151,7 @@ func (t *fieldsTransformer) add(f common.Field) { } } -func transformField(version *common.Version, f common.Field) (common.MapStr, common.MapStr) { +func transformField(version *common.Version, f asset.Field) (common.MapStr, common.MapStr) { field := common.MapStr{ "name": f.Path, "count": f.Count, @@ -217,7 +218,7 @@ func getVal(valP *bool, def bool) bool { return def } -func addParams(format *common.MapStr, version *common.Version, f common.Field) { +func addParams(format *common.MapStr, version *common.Version, f asset.Field) { addFormatParam(format, "pattern", f.Pattern) addFormatParam(format, "inputFormat", f.InputFormat) addFormatParam(format, "outputFormat", f.OutputFormat) @@ -248,7 +249,7 @@ func addFormatParam(f *common.MapStr, key string, val interface{}) { } // takes the highest version where major version <= given version -func addVersionedFormatParam(f *common.MapStr, version *common.Version, key string, val []common.VersionizedString) { +func addVersionedFormatParam(f *common.MapStr, version *common.Version, key string, val []asset.VersionizedString) { if len(val) == 0 { return } diff --git a/libbeat/kibana/fields_transformer_test.go b/libbeat/kibana/fields_transformer_test.go index 1fb1e8b3a0b..ae260cdc447 100644 --- a/libbeat/kibana/fields_transformer_test.go +++ b/libbeat/kibana/fields_transformer_test.go @@ -24,6 +24,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/elastic/beats/libbeat/asset" "github.com/elastic/beats/libbeat/common" ) @@ -35,7 +36,7 @@ var ( ) func TestEmpty(t *testing.T) { - trans, err := newFieldsTransformer(version, common.Fields{}, true) + trans, err := newFieldsTransformer(version, asset.Fields{}, true) assert.NoError(t, err) out, err := trans.transform() assert.NoError(t, err) @@ -93,26 +94,26 @@ func TestEmpty(t *testing.T) { func TestMissingVersion(t *testing.T) { var c *common.Version - _, err := newFieldsTransformer(c, common.Fields{}, true) + _, err := newFieldsTransformer(c, asset.Fields{}, true) assert.Error(t, err) } func TestDuplicateField(t *testing.T) { testCases := []struct { - commonFields []common.Field + commonFields []asset.Field }{ // type change - {commonFields: []common.Field{ + {commonFields: []asset.Field{ {Name: "context", Path: "something"}, {Name: "context", Path: "something", Type: "date"}, }}, // missing overwrite - {commonFields: []common.Field{ + {commonFields: []asset.Field{ {Name: "context", Path: "something"}, {Name: "context", Path: "something"}, }}, // missing overwrite in source - {commonFields: []common.Field{ + {commonFields: []asset.Field{ {Name: "context", Path: "something", Overwrite: true}, {Name: "context", Path: "something"}, }}, @@ -127,9 +128,9 @@ func TestDuplicateField(t *testing.T) { } func TestValidDuplicateField(t *testing.T) { - commonFields := common.Fields{ - common.Field{Name: "context", Path: "something", Type: "keyword", Description: "original description"}, - common.Field{Name: "context", Path: "something", Overwrite: true, Description: "updated description", + commonFields := asset.Fields{ + asset.Field{Name: "context", Path: "something", Type: "keyword", Description: "original description"}, + asset.Field{Name: "context", Path: "something", Overwrite: true, Description: "updated description", Aggregatable: &falsy, Analyzed: &truthy, Count: 2, @@ -137,18 +138,18 @@ func TestValidDuplicateField(t *testing.T) { Index: &falsy, Searchable: &falsy, }, - common.Field{ + asset.Field{ Name: "context", Type: "group", - Fields: common.Fields{ - common.Field{Name: "another", Type: "date"}, + Fields: asset.Fields{ + asset.Field{Name: "another", Type: "date"}, }, }, - common.Field{ + asset.Field{ Name: "context", Type: "group", - Fields: common.Fields{ - common.Field{Name: "another", Overwrite: true}, + Fields: asset.Fields{ + asset.Field{Name: "another", Overwrite: true}, }, }, } @@ -171,11 +172,11 @@ func TestValidDuplicateField(t *testing.T) { } func TestInvalidVersion(t *testing.T) { - commonFields := common.Fields{ - common.Field{ + commonFields := asset.Fields{ + asset.Field{ Name: "versionTest", Format: "url", - UrlTemplate: []common.VersionizedString{ + UrlTemplate: []asset.VersionizedString{ {MinVersion: "3", Value: ""}, }, }, @@ -188,26 +189,26 @@ func TestInvalidVersion(t *testing.T) { func TestTransformTypes(t *testing.T) { tests := []struct { - commonField common.Field + commonField asset.Field expected interface{} }{ - {commonField: common.Field{}, expected: "string"}, - {commonField: common.Field{Type: "half_float"}, expected: "number"}, - {commonField: common.Field{Type: "scaled_float"}, expected: "number"}, - {commonField: common.Field{Type: "float"}, expected: "number"}, - {commonField: common.Field{Type: "integer"}, expected: "number"}, - {commonField: common.Field{Type: "long"}, expected: "number"}, - {commonField: common.Field{Type: "short"}, expected: "number"}, - {commonField: common.Field{Type: "byte"}, expected: "number"}, - {commonField: common.Field{Type: "keyword"}, expected: "string"}, - {commonField: common.Field{Type: "text"}, expected: "string"}, - {commonField: common.Field{Type: "string"}, expected: nil}, - {commonField: common.Field{Type: "date"}, expected: "date"}, - {commonField: common.Field{Type: "geo_point"}, expected: "geo_point"}, - {commonField: common.Field{Type: "invalid"}, expected: nil}, + {commonField: asset.Field{}, expected: "string"}, + {commonField: asset.Field{Type: "half_float"}, expected: "number"}, + {commonField: asset.Field{Type: "scaled_float"}, expected: "number"}, + {commonField: asset.Field{Type: "float"}, expected: "number"}, + {commonField: asset.Field{Type: "integer"}, expected: "number"}, + {commonField: asset.Field{Type: "long"}, expected: "number"}, + {commonField: asset.Field{Type: "short"}, expected: "number"}, + {commonField: asset.Field{Type: "byte"}, expected: "number"}, + {commonField: asset.Field{Type: "keyword"}, expected: "string"}, + {commonField: asset.Field{Type: "text"}, expected: "string"}, + {commonField: asset.Field{Type: "string"}, expected: nil}, + {commonField: asset.Field{Type: "date"}, expected: "date"}, + {commonField: asset.Field{Type: "geo_point"}, expected: "geo_point"}, + {commonField: asset.Field{Type: "invalid"}, expected: nil}, } for idx, test := range tests { - trans, _ := newFieldsTransformer(version, common.Fields{test.commonField}, true) + trans, _ := newFieldsTransformer(version, asset.Fields{test.commonField}, true) transformed, err := trans.transform() assert.NoError(t, err) out := transformed["fields"].([]common.MapStr)[0] @@ -217,32 +218,32 @@ func TestTransformTypes(t *testing.T) { func TestTransformGroup(t *testing.T) { tests := []struct { - commonFields common.Fields + commonFields asset.Fields expected []string }{ { - commonFields: common.Fields{common.Field{Name: "context", Path: "something"}}, + commonFields: asset.Fields{asset.Field{Name: "context", Path: "something"}}, expected: []string{"context"}, }, { - commonFields: common.Fields{ - common.Field{ + commonFields: asset.Fields{ + asset.Field{ Name: "context", Type: "group", - Fields: common.Fields{ - common.Field{Name: "another", Type: ""}, + Fields: asset.Fields{ + asset.Field{Name: "another", Type: ""}, }, }, - common.Field{ + asset.Field{ Name: "context", Type: "group", - Fields: common.Fields{ - common.Field{Name: "type", Type: ""}, - common.Field{ + Fields: asset.Fields{ + asset.Field{Name: "type", Type: ""}, + asset.Field{ Name: "metric", Type: "group", - Fields: common.Fields{ - common.Field{Name: "object"}, + Fields: asset.Fields{ + asset.Field{Name: "object"}, }, }, }, @@ -265,70 +266,70 @@ func TestTransformGroup(t *testing.T) { func TestTransformMisc(t *testing.T) { tests := []struct { - commonField common.Field + commonField asset.Field expected interface{} attr string }{ - {commonField: common.Field{}, expected: 0, attr: "count"}, - {commonField: common.Field{Count: 4}, expected: 4, attr: "count"}, + {commonField: asset.Field{}, expected: 0, attr: "count"}, + {commonField: asset.Field{Count: 4}, expected: 4, attr: "count"}, // searchable - {commonField: common.Field{}, expected: true, attr: "searchable"}, - {commonField: common.Field{Searchable: &truthy}, expected: true, attr: "searchable"}, - {commonField: common.Field{Searchable: &falsy}, expected: false, attr: "searchable"}, - {commonField: common.Field{Type: "binary"}, expected: false, attr: "searchable"}, - {commonField: common.Field{Searchable: &truthy, Type: "binary"}, expected: false, attr: "searchable"}, + {commonField: asset.Field{}, expected: true, attr: "searchable"}, + {commonField: asset.Field{Searchable: &truthy}, expected: true, attr: "searchable"}, + {commonField: asset.Field{Searchable: &falsy}, expected: false, attr: "searchable"}, + {commonField: asset.Field{Type: "binary"}, expected: false, attr: "searchable"}, + {commonField: asset.Field{Searchable: &truthy, Type: "binary"}, expected: false, attr: "searchable"}, // aggregatable - {commonField: common.Field{}, expected: true, attr: "aggregatable"}, - {commonField: common.Field{Aggregatable: &truthy}, expected: true, attr: "aggregatable"}, - {commonField: common.Field{Aggregatable: &falsy}, expected: false, attr: "aggregatable"}, - {commonField: common.Field{Type: "binary"}, expected: false, attr: "aggregatable"}, - {commonField: common.Field{Aggregatable: &truthy, Type: "binary"}, expected: false, attr: "aggregatable"}, - {commonField: common.Field{Type: "keyword"}, expected: true, attr: "aggregatable"}, - {commonField: common.Field{Aggregatable: &truthy, Type: "text"}, expected: false, attr: "aggregatable"}, - {commonField: common.Field{Type: "text"}, expected: false, attr: "aggregatable"}, + {commonField: asset.Field{}, expected: true, attr: "aggregatable"}, + {commonField: asset.Field{Aggregatable: &truthy}, expected: true, attr: "aggregatable"}, + {commonField: asset.Field{Aggregatable: &falsy}, expected: false, attr: "aggregatable"}, + {commonField: asset.Field{Type: "binary"}, expected: false, attr: "aggregatable"}, + {commonField: asset.Field{Aggregatable: &truthy, Type: "binary"}, expected: false, attr: "aggregatable"}, + {commonField: asset.Field{Type: "keyword"}, expected: true, attr: "aggregatable"}, + {commonField: asset.Field{Aggregatable: &truthy, Type: "text"}, expected: false, attr: "aggregatable"}, + {commonField: asset.Field{Type: "text"}, expected: false, attr: "aggregatable"}, // analyzed - {commonField: common.Field{}, expected: false, attr: "analyzed"}, - {commonField: common.Field{Analyzed: &truthy}, expected: true, attr: "analyzed"}, - {commonField: common.Field{Analyzed: &falsy}, expected: false, attr: "analyzed"}, - {commonField: common.Field{Type: "binary"}, expected: false, attr: "analyzed"}, - {commonField: common.Field{Analyzed: &truthy, Type: "binary"}, expected: false, attr: "analyzed"}, + {commonField: asset.Field{}, expected: false, attr: "analyzed"}, + {commonField: asset.Field{Analyzed: &truthy}, expected: true, attr: "analyzed"}, + {commonField: asset.Field{Analyzed: &falsy}, expected: false, attr: "analyzed"}, + {commonField: asset.Field{Type: "binary"}, expected: false, attr: "analyzed"}, + {commonField: asset.Field{Analyzed: &truthy, Type: "binary"}, expected: false, attr: "analyzed"}, // doc_values always set to true except for meta fields - {commonField: common.Field{}, expected: true, attr: "doc_values"}, - {commonField: common.Field{DocValues: &truthy}, expected: true, attr: "doc_values"}, - {commonField: common.Field{DocValues: &falsy}, expected: false, attr: "doc_values"}, - {commonField: common.Field{Script: "doc[]"}, expected: false, attr: "doc_values"}, - {commonField: common.Field{DocValues: &truthy, Script: "doc[]"}, expected: false, attr: "doc_values"}, - {commonField: common.Field{Type: "binary"}, expected: false, attr: "doc_values"}, - {commonField: common.Field{DocValues: &truthy, Type: "binary"}, expected: true, attr: "doc_values"}, + {commonField: asset.Field{}, expected: true, attr: "doc_values"}, + {commonField: asset.Field{DocValues: &truthy}, expected: true, attr: "doc_values"}, + {commonField: asset.Field{DocValues: &falsy}, expected: false, attr: "doc_values"}, + {commonField: asset.Field{Script: "doc[]"}, expected: false, attr: "doc_values"}, + {commonField: asset.Field{DocValues: &truthy, Script: "doc[]"}, expected: false, attr: "doc_values"}, + {commonField: asset.Field{Type: "binary"}, expected: false, attr: "doc_values"}, + {commonField: asset.Field{DocValues: &truthy, Type: "binary"}, expected: true, attr: "doc_values"}, // enabled - only applies to objects (and only if set) - {commonField: common.Field{Type: "binary", Enabled: &falsy}, expected: nil, attr: "enabled"}, - {commonField: common.Field{Type: "binary", Enabled: &truthy}, expected: nil, attr: "enabled"}, - {commonField: common.Field{Type: "object", Enabled: &truthy}, expected: true, attr: "enabled"}, - {commonField: common.Field{Type: "object", Enabled: &falsy}, expected: false, attr: "enabled"}, - {commonField: common.Field{Type: "object", Enabled: &falsy}, expected: false, attr: "doc_values"}, + {commonField: asset.Field{Type: "binary", Enabled: &falsy}, expected: nil, attr: "enabled"}, + {commonField: asset.Field{Type: "binary", Enabled: &truthy}, expected: nil, attr: "enabled"}, + {commonField: asset.Field{Type: "object", Enabled: &truthy}, expected: true, attr: "enabled"}, + {commonField: asset.Field{Type: "object", Enabled: &falsy}, expected: false, attr: "enabled"}, + {commonField: asset.Field{Type: "object", Enabled: &falsy}, expected: false, attr: "doc_values"}, // indexed - {commonField: common.Field{Type: "binary"}, expected: false, attr: "indexed"}, - {commonField: common.Field{Index: &truthy, Type: "binary"}, expected: false, attr: "indexed"}, + {commonField: asset.Field{Type: "binary"}, expected: false, attr: "indexed"}, + {commonField: asset.Field{Index: &truthy, Type: "binary"}, expected: false, attr: "indexed"}, // script, scripted - {commonField: common.Field{}, expected: false, attr: "scripted"}, - {commonField: common.Field{}, expected: nil, attr: "script"}, - {commonField: common.Field{Script: "doc[]"}, expected: true, attr: "scripted"}, - {commonField: common.Field{Script: "doc[]"}, expected: "doc[]", attr: "script"}, - {commonField: common.Field{Type: "binary"}, expected: false, attr: "scripted"}, + {commonField: asset.Field{}, expected: false, attr: "scripted"}, + {commonField: asset.Field{}, expected: nil, attr: "script"}, + {commonField: asset.Field{Script: "doc[]"}, expected: true, attr: "scripted"}, + {commonField: asset.Field{Script: "doc[]"}, expected: "doc[]", attr: "script"}, + {commonField: asset.Field{Type: "binary"}, expected: false, attr: "scripted"}, // language - {commonField: common.Field{}, expected: nil, attr: "lang"}, - {commonField: common.Field{Script: "doc[]"}, expected: "painless", attr: "lang"}, + {commonField: asset.Field{}, expected: nil, attr: "lang"}, + {commonField: asset.Field{Script: "doc[]"}, expected: "painless", attr: "lang"}, } for idx, test := range tests { - trans, _ := newFieldsTransformer(version, common.Fields{test.commonField}, true) + trans, _ := newFieldsTransformer(version, asset.Fields{test.commonField}, true) transformed, err := trans.transform() assert.NoError(t, err) out := transformed["fields"].([]common.MapStr)[0] @@ -344,27 +345,27 @@ func TestTransformFieldFormatMap(t *testing.T) { falsy := false tests := []struct { - commonField common.Field + commonField asset.Field version *common.Version expected common.MapStr }{ { - commonField: common.Field{Name: "c"}, + commonField: asset.Field{Name: "c"}, expected: common.MapStr{}, version: version, }, { - commonField: common.Field{Name: "c", Format: "url"}, + commonField: asset.Field{Name: "c", Format: "url"}, expected: common.MapStr{"c": common.MapStr{"id": "url"}}, version: version, }, { - commonField: common.Field{Name: "c", Pattern: "p"}, + commonField: asset.Field{Name: "c", Pattern: "p"}, expected: common.MapStr{"c": common.MapStr{"params": common.MapStr{"pattern": "p"}}}, version: version, }, { - commonField: common.Field{ + commonField: asset.Field{ Name: "c", Format: "url", Pattern: "p", @@ -378,7 +379,7 @@ func TestTransformFieldFormatMap(t *testing.T) { version: version, }, { - commonField: common.Field{ + commonField: asset.Field{ Name: "c", Format: "url", InputFormat: "string", @@ -394,7 +395,7 @@ func TestTransformFieldFormatMap(t *testing.T) { version: version, }, { - commonField: common.Field{ + commonField: asset.Field{ Name: "c", Format: "url", Pattern: "[^-]", @@ -414,7 +415,7 @@ func TestTransformFieldFormatMap(t *testing.T) { version: version, }, { - commonField: common.Field{ + commonField: asset.Field{ Name: "c", InputFormat: "string", }, @@ -423,7 +424,7 @@ func TestTransformFieldFormatMap(t *testing.T) { }, { version: version620, - commonField: common.Field{ + commonField: asset.Field{ Name: "c", Format: "url", Pattern: "[^-]", @@ -432,7 +433,7 @@ func TestTransformFieldFormatMap(t *testing.T) { OutputFormat: "float", OutputPrecision: &precision, LabelTemplate: "lblT", - UrlTemplate: []common.VersionizedString{ + UrlTemplate: []asset.VersionizedString{ {MinVersion: "5.0.0", Value: "5x.urlTemplate"}, {MinVersion: "6.0.0", Value: "6x.urlTemplate"}, }, @@ -454,10 +455,10 @@ func TestTransformFieldFormatMap(t *testing.T) { }, { version: version620, - commonField: common.Field{ + commonField: asset.Field{ Name: "c", Format: "url", - UrlTemplate: []common.VersionizedString{ + UrlTemplate: []asset.VersionizedString{ {MinVersion: "6.4.0", Value: "6x.urlTemplate"}, }, }, @@ -467,10 +468,10 @@ func TestTransformFieldFormatMap(t *testing.T) { }, { version: version620, - commonField: common.Field{ + commonField: asset.Field{ Name: "c", Format: "url", - UrlTemplate: []common.VersionizedString{ + UrlTemplate: []asset.VersionizedString{ {MinVersion: "4.7.2", Value: "4x.urlTemplate"}, {MinVersion: "6.5.1", Value: "6x.urlTemplate"}, }, @@ -486,10 +487,10 @@ func TestTransformFieldFormatMap(t *testing.T) { }, { version: version620, - commonField: common.Field{ + commonField: asset.Field{ Name: "c", Format: "url", - UrlTemplate: []common.VersionizedString{ + UrlTemplate: []asset.VersionizedString{ {MinVersion: "6.2.0", Value: "6.2.0.urlTemplate"}, {MinVersion: "6.2.0-alpha", Value: "6.2.0-alpha.urlTemplate"}, {MinVersion: "6.2.7", Value: "6.2.7.urlTemplate"}, @@ -506,10 +507,10 @@ func TestTransformFieldFormatMap(t *testing.T) { }, { version: version620, - commonField: common.Field{ + commonField: asset.Field{ Name: "c", Format: "url", - UrlTemplate: []common.VersionizedString{ + UrlTemplate: []asset.VersionizedString{ {MinVersion: "4.1.0", Value: "4x.urlTemplate"}, {MinVersion: "5.2.0-rc2", Value: "5.2.0-rc2.urlTemplate"}, {MinVersion: "5.2.0-rc3", Value: "5.2.0-rc3.urlTemplate"}, @@ -527,7 +528,7 @@ func TestTransformFieldFormatMap(t *testing.T) { }, } for idx, test := range tests { - trans, _ := newFieldsTransformer(test.version, common.Fields{test.commonField}, true) + trans, _ := newFieldsTransformer(test.version, asset.Fields{test.commonField}, true) transformed, err := trans.transform() assert.NoError(t, err) out := transformed["fieldFormatMap"] @@ -537,25 +538,25 @@ func TestTransformFieldFormatMap(t *testing.T) { func TestTransformGroupAndEnabled(t *testing.T) { tests := []struct { - commonFields common.Fields + commonFields asset.Fields expected []string }{ { - commonFields: common.Fields{common.Field{Name: "context", Path: "something"}}, + commonFields: asset.Fields{asset.Field{Name: "context", Path: "something"}}, expected: []string{"context"}, }, { - commonFields: common.Fields{ - common.Field{ + commonFields: asset.Fields{ + asset.Field{ Name: "context", Type: "group", - Fields: common.Fields{ - common.Field{Name: "type", Type: ""}, - common.Field{ + Fields: asset.Fields{ + asset.Field{Name: "type", Type: ""}, + asset.Field{ Name: "metric", Type: "group", - Fields: common.Fields{ - common.Field{Name: "object"}, + Fields: asset.Fields{ + asset.Field{Name: "object"}, }, }, }, @@ -564,28 +565,28 @@ func TestTransformGroupAndEnabled(t *testing.T) { expected: []string{"context.type", "context.metric.object"}, }, { - commonFields: common.Fields{ - common.Field{Name: "enabledField"}, - common.Field{Name: "disabledField", Enabled: &falsy}, //enabled is ignored for Type!=group - common.Field{ + commonFields: asset.Fields{ + asset.Field{Name: "enabledField"}, + asset.Field{Name: "disabledField", Enabled: &falsy}, //enabled is ignored for Type!=group + asset.Field{ Name: "enabledGroup", Type: "group", Enabled: &truthy, - Fields: common.Fields{ - common.Field{Name: "type", Type: ""}, + Fields: asset.Fields{ + asset.Field{Name: "type", Type: ""}, }, }, - common.Field{ + asset.Field{ Name: "context", Type: "group", Enabled: &falsy, - Fields: common.Fields{ - common.Field{Name: "type", Type: ""}, - common.Field{ + Fields: asset.Fields{ + asset.Field{Name: "type", Type: ""}, + asset.Field{ Name: "metric", Type: "group", - Fields: common.Fields{ - common.Field{Name: "object"}, + Fields: asset.Fields{ + asset.Field{Name: "object"}, }, }, }, @@ -607,15 +608,15 @@ func TestTransformGroupAndEnabled(t *testing.T) { } func TestTransformMultiField(t *testing.T) { - f := common.Field{ + f := asset.Field{ Name: "context", Type: "", - MultiFields: common.Fields{ - common.Field{Name: "keyword", Type: "keyword"}, - common.Field{Name: "text", Type: "text"}, + MultiFields: asset.Fields{ + asset.Field{Name: "keyword", Type: "keyword"}, + asset.Field{Name: "text", Type: "text"}, }, } - trans, _ := newFieldsTransformer(version, common.Fields{f}, true) + trans, _ := newFieldsTransformer(version, asset.Fields{f}, true) transformed, err := trans.transform() assert.NoError(t, err) out := transformed["fields"].([]common.MapStr) diff --git a/libbeat/kibana/index_pattern_generator.go b/libbeat/kibana/index_pattern_generator.go index fe426d0a033..1222881ddc3 100644 --- a/libbeat/kibana/index_pattern_generator.go +++ b/libbeat/kibana/index_pattern_generator.go @@ -24,6 +24,7 @@ import ( "path/filepath" "regexp" + "github.com/elastic/beats/libbeat/asset" "github.com/elastic/beats/libbeat/common" ) @@ -110,7 +111,7 @@ func (i *IndexPatternGenerator) addGeneral(indexPattern *common.MapStr) error { } func (i *IndexPatternGenerator) addFieldsSpecific(indexPattern *common.MapStr) error { - fields, err := common.LoadFields(i.fields) + fields, err := asset.LoadFields(i.fields) if err != nil { return err } diff --git a/libbeat/template/config.go b/libbeat/template/config.go index ed939fcc9a0..6143e2ec31f 100644 --- a/libbeat/template/config.go +++ b/libbeat/template/config.go @@ -17,7 +17,7 @@ package template -import "github.com/elastic/beats/libbeat/common" +import "github.com/elastic/beats/libbeat/asset" type TemplateConfig struct { Enabled bool `config:"enabled"` @@ -29,7 +29,7 @@ type TemplateConfig struct { Path string `config:"path"` Name string `config:"name"` } `config:"json"` - AppendFields common.Fields `config:"append_fields"` + AppendFields asset.Fields `config:"append_fields"` Overwrite bool `config:"overwrite"` Settings TemplateSettings `config:"settings"` } diff --git a/libbeat/template/processor.go b/libbeat/template/processor.go index 4e25bcec03f..8333ed73647 100644 --- a/libbeat/template/processor.go +++ b/libbeat/template/processor.go @@ -21,6 +21,7 @@ import ( "errors" "strings" + "github.com/elastic/beats/libbeat/asset" "github.com/elastic/beats/libbeat/common" ) @@ -38,7 +39,7 @@ var ( const scalingFactorKey = "scalingFactor" // Process recursively processes the given fields and writes the template in the given output -func (p *Processor) Process(fields common.Fields, path string, output common.MapStr) error { +func (p *Processor) Process(fields asset.Fields, path string, output common.MapStr) error { for _, field := range fields { if field.Name == "" { @@ -81,7 +82,7 @@ func (p *Processor) Process(fields common.Fields, path string, output common.Map // Combine properties with previous field definitions (if any) properties := common.MapStr{} - key := common.GenerateKey(field.Name) + ".properties" + key := asset.GenerateKey(field.Name) + ".properties" currentProperties, err := output.GetValue(key) if err == nil { var ok bool @@ -106,7 +107,7 @@ func (p *Processor) Process(fields common.Fields, path string, output common.Map } if len(mapping) > 0 { - output.Put(common.GenerateKey(field.Name), mapping) + output.Put(asset.GenerateKey(field.Name), mapping) } } return nil @@ -132,13 +133,13 @@ func (p *Processor) other(f *common.Field) common.MapStr { return property } -func (p *Processor) integer(f *common.Field) common.MapStr { +func (p *Processor) integer(f *asset.Field) common.MapStr { property := getDefaultProperties(f) property["type"] = "long" return property } -func (p *Processor) scaledFloat(f *common.Field, params ...common.MapStr) common.MapStr { +func (p *Processor) scaledFloat(f *asset.Field, params ...common.MapStr) common.MapStr { property := getDefaultProperties(f) property["type"] = "scaled_float" @@ -163,7 +164,7 @@ func (p *Processor) scaledFloat(f *common.Field, params ...common.MapStr) common return property } -func (p *Processor) halfFloat(f *common.Field) common.MapStr { +func (p *Processor) halfFloat(f *asset.Field) common.MapStr { property := getDefaultProperties(f) property["type"] = "half_float" @@ -173,7 +174,7 @@ func (p *Processor) halfFloat(f *common.Field) common.MapStr { return property } -func (p *Processor) ip(f *common.Field) common.MapStr { +func (p *Processor) ip(f *asset.Field) common.MapStr { property := getDefaultProperties(f) property["type"] = "ip" @@ -186,7 +187,7 @@ func (p *Processor) ip(f *common.Field) common.MapStr { return property } -func (p *Processor) keyword(f *common.Field) common.MapStr { +func (p *Processor) keyword(f *asset.Field) common.MapStr { property := getDefaultProperties(f) property["type"] = "keyword" @@ -213,7 +214,7 @@ func (p *Processor) keyword(f *common.Field) common.MapStr { return property } -func (p *Processor) text(f *common.Field) common.MapStr { +func (p *Processor) text(f *asset.Field) common.MapStr { properties := getDefaultProperties(f) properties["type"] = "text" @@ -249,7 +250,7 @@ func (p *Processor) text(f *common.Field) common.MapStr { return properties } -func (p *Processor) array(f *common.Field) common.MapStr { +func (p *Processor) array(f *asset.Field) common.MapStr { properties := getDefaultProperties(f) if f.ObjectType != "" { properties["type"] = f.ObjectType @@ -257,7 +258,7 @@ func (p *Processor) array(f *common.Field) common.MapStr { return properties } -func (p *Processor) alias(f *common.Field) common.MapStr { +func (p *Processor) alias(f *asset.Field) common.MapStr { // Aliases were introduced in Elasticsearch 6.4, ignore if unsupported if p.EsVersion.LessThan(common.MustNewVersion("6.4.0")) { return nil @@ -274,7 +275,7 @@ func (p *Processor) alias(f *common.Field) common.MapStr { return properties } -func (p *Processor) object(f *common.Field) common.MapStr { +func (p *Processor) object(f *asset.Field) common.MapStr { matchType := func(onlyType string, mt string) string { if mt != "" { return mt @@ -282,11 +283,11 @@ func (p *Processor) object(f *common.Field) common.MapStr { return onlyType } - var otParams []common.ObjectTypeCfg + var otParams []asset.ObjectTypeCfg if len(f.ObjectTypeParams) != 0 { otParams = f.ObjectTypeParams } else { - otParams = []common.ObjectTypeCfg{common.ObjectTypeCfg{ + otParams = []asset.ObjectTypeCfg{asset.ObjectTypeCfg{ ObjectType: f.ObjectType, ObjectTypeMappingType: f.ObjectTypeMappingType, ScalingFactor: f.ScalingFactor}} } @@ -327,7 +328,7 @@ func (p *Processor) object(f *common.Field) common.MapStr { return properties } -func addDynamicTemplate(f *common.Field, properties common.MapStr, matchType string) { +func addDynamicTemplate(f *asset.Field, properties common.MapStr, matchType string) { path := "" if len(f.Path) > 0 { path = f.Path + "." @@ -348,7 +349,7 @@ func addDynamicTemplate(f *common.Field, properties common.MapStr, matchType str dynamicTemplates = append(dynamicTemplates, template) } -func getDefaultProperties(f *common.Field) common.MapStr { +func getDefaultProperties(f *asset.Field) common.MapStr { // Currently no defaults exist properties := common.MapStr{} diff --git a/libbeat/template/processor_test.go b/libbeat/template/processor_test.go index ff8688107aa..b71064b007a 100644 --- a/libbeat/template/processor_test.go +++ b/libbeat/template/processor_test.go @@ -22,6 +22,7 @@ import ( "github.com/stretchr/testify/assert" + "github.com/elastic/beats/libbeat/asset" "github.com/elastic/beats/libbeat/common" ) @@ -39,92 +40,92 @@ func TestProcessor(t *testing.T) { expected common.MapStr }{ { - output: p.other(&common.Field{Type: "long"}), + output: p.other(&asset.Field{Type: "long"}), expected: common.MapStr{"type": "long"}, }, { - output: p.scaledFloat(&common.Field{Type: "scaled_float"}), + output: p.scaledFloat(&asset.Field{Type: "scaled_float"}), expected: common.MapStr{ "type": "scaled_float", "scaling_factor": 1000, }, }, { - output: p.scaledFloat(&common.Field{Type: "scaled_float", ScalingFactor: 100}), + output: p.scaledFloat(&asset.Field{Type: "scaled_float", ScalingFactor: 100}), expected: common.MapStr{ "type": "scaled_float", "scaling_factor": 100, }, }, { - output: p.scaledFloat(&common.Field{Type: "scaled_float"}, common.MapStr{scalingFactorKey: 0}), + output: p.scaledFloat(&asset.Field{Type: "scaled_float"}, common.MapStr{scalingFactorKey: 0}), expected: common.MapStr{ "type": "scaled_float", "scaling_factor": 1000, }, }, { - output: p.scaledFloat(&common.Field{Type: "scaled_float"}, common.MapStr{"someKey": 10}), + output: p.scaledFloat(&asset.Field{Type: "scaled_float"}, common.MapStr{"someKey": 10}), expected: common.MapStr{ "type": "scaled_float", "scaling_factor": 1000, }, }, { - output: p.scaledFloat(&common.Field{Type: "scaled_float"}, common.MapStr{scalingFactorKey: 10}), + output: p.scaledFloat(&asset.Field{Type: "scaled_float"}, common.MapStr{scalingFactorKey: 10}), expected: common.MapStr{ "type": "scaled_float", "scaling_factor": 10, }, }, { - output: pEsVersion2.scaledFloat(&common.Field{Type: "scaled_float"}), + output: pEsVersion2.scaledFloat(&asset.Field{Type: "scaled_float"}), expected: common.MapStr{"type": "float"}, }, { - output: p.object(&common.Field{Type: "object", Enabled: &falseVar}), + output: p.object(&asset.Field{Type: "object", Enabled: &falseVar}), expected: common.MapStr{ "type": "object", "enabled": false, }, }, { - output: p.integer(&common.Field{Type: "long", CopyTo: "hello.world"}), + output: p.integer(&asset.Field{Type: "long", CopyTo: "hello.world"}), expected: common.MapStr{ "type": "long", "copy_to": "hello.world", }, }, { - output: p.array(&common.Field{Type: "array"}), + output: p.array(&asset.Field{Type: "array"}), expected: common.MapStr{}, }, { - output: p.array(&common.Field{Type: "array", ObjectType: "text"}), + output: p.array(&asset.Field{Type: "array", ObjectType: "text"}), expected: common.MapStr{"type": "text"}, }, { - output: p.array(&common.Field{Type: "array", Index: &falseVar, ObjectType: "keyword"}), + output: p.array(&asset.Field{Type: "array", Index: &falseVar, ObjectType: "keyword"}), expected: common.MapStr{"index": false, "type": "keyword"}, }, { - output: pEsVersion64.alias(&common.Field{Type: "alias", AliasPath: "a.b"}), + output: pEsVersion64.alias(&asset.Field{Type: "alias", AliasPath: "a.b"}), expected: common.MapStr{"path": "a.b", "type": "alias"}, }, { // alias unsupported in ES < 6.4 - output: pEsVersion63.alias(&common.Field{Type: "alias", AliasPath: "a.b"}), + output: pEsVersion63.alias(&asset.Field{Type: "alias", AliasPath: "a.b"}), expected: nil, }, { - output: p.object(&common.Field{Type: "object", Enabled: &falseVar}), + output: p.object(&asset.Field{Type: "object", Enabled: &falseVar}), expected: common.MapStr{ "type": "object", "enabled": false, }, }, { - output: p.text(&common.Field{Type: "text", Analyzer: "autocomplete"}), + output: p.text(&asset.Field{Type: "text", Analyzer: "autocomplete"}), expected: common.MapStr{ "type": "text", "analyzer": "autocomplete", @@ -132,21 +133,21 @@ func TestProcessor(t *testing.T) { }, }, { - output: p.text(&common.Field{Type: "text", Analyzer: "autocomplete", Norms: true}), + output: p.text(&asset.Field{Type: "text", Analyzer: "autocomplete", Norms: true}), expected: common.MapStr{ "type": "text", "analyzer": "autocomplete", }, }, { - output: p.text(&common.Field{Type: "text", SearchAnalyzer: "standard", Norms: true}), + output: p.text(&asset.Field{Type: "text", SearchAnalyzer: "standard", Norms: true}), expected: common.MapStr{ "type": "text", "search_analyzer": "standard", }, }, { - output: p.text(&common.Field{Type: "text", Analyzer: "autocomplete", SearchAnalyzer: "standard", Norms: true}), + output: p.text(&asset.Field{Type: "text", Analyzer: "autocomplete", SearchAnalyzer: "standard", Norms: true}), expected: common.MapStr{ "type": "text", "analyzer": "autocomplete", @@ -154,7 +155,7 @@ func TestProcessor(t *testing.T) { }, }, { - output: p.text(&common.Field{Type: "text", MultiFields: common.Fields{common.Field{Name: "raw", Type: "keyword"}}, Norms: true}), + output: p.text(&asset.Field{Type: "text", MultiFields: asset.Fields{asset.Field{Name: "raw", Type: "keyword"}}, Norms: true}), expected: common.MapStr{ "type": "text", "fields": common.MapStr{ @@ -166,7 +167,7 @@ func TestProcessor(t *testing.T) { }, }, { - output: p.keyword(&common.Field{Type: "keyword", MultiFields: common.Fields{common.Field{Name: "analyzed", Type: "text", Norms: true}}}), + output: p.keyword(&asset.Field{Type: "keyword", MultiFields: asset.Fields{asset.Field{Name: "analyzed", Type: "text", Norms: true}}}), expected: common.MapStr{ "type": "keyword", "ignore_above": 1024, @@ -178,29 +179,29 @@ func TestProcessor(t *testing.T) { }, }, { - output: p.keyword(&common.Field{Type: "keyword", IgnoreAbove: 256}), + output: p.keyword(&asset.Field{Type: "keyword", IgnoreAbove: 256}), expected: common.MapStr{ "type": "keyword", "ignore_above": 256, }, }, { - output: p.keyword(&common.Field{Type: "keyword", IgnoreAbove: -1}), + output: p.keyword(&asset.Field{Type: "keyword", IgnoreAbove: -1}), expected: common.MapStr{ "type": "keyword", }, }, { - output: p.keyword(&common.Field{Type: "keyword"}), + output: p.keyword(&asset.Field{Type: "keyword"}), expected: common.MapStr{ "type": "keyword", "ignore_above": 1024, }, }, { - output: p.text(&common.Field{Type: "text", MultiFields: common.Fields{ - common.Field{Name: "raw", Type: "keyword"}, - common.Field{Name: "indexed", Type: "text"}, + output: p.text(&asset.Field{Type: "text", MultiFields: asset.Fields{ + asset.Field{Name: "raw", Type: "keyword"}, + asset.Field{Name: "indexed", Type: "text"}, }, Norms: true}), expected: common.MapStr{ "type": "text", @@ -217,9 +218,9 @@ func TestProcessor(t *testing.T) { }, }, { - output: p.text(&common.Field{Type: "text", MultiFields: common.Fields{ - common.Field{Name: "raw", Type: "keyword"}, - common.Field{Name: "indexed", Type: "text"}, + output: p.text(&asset.Field{Type: "text", MultiFields: asset.Fields{ + asset.Field{Name: "raw", Type: "keyword"}, + asset.Field{Name: "indexed", Type: "text"}, }, Norms: true}), expected: common.MapStr{ "type": "text", @@ -236,67 +237,67 @@ func TestProcessor(t *testing.T) { }, }, { - output: p.object(&common.Field{Dynamic: common.DynamicType{Value: false}}), + output: p.object(&asset.Field{Dynamic: asset.DynamicType{Value: false}}), expected: common.MapStr{ "dynamic": false, "type": "object", }, }, { - output: p.object(&common.Field{Dynamic: common.DynamicType{Value: true}}), + output: p.object(&asset.Field{Dynamic: asset.DynamicType{Value: true}}), expected: common.MapStr{ "dynamic": true, "type": "object", }, }, { - output: p.object(&common.Field{Dynamic: common.DynamicType{Value: "strict"}}), + output: p.object(&asset.Field{Dynamic: asset.DynamicType{Value: "strict"}}), expected: common.MapStr{ "dynamic": "strict", "type": "object", }, }, { - output: p.other(&common.Field{Type: "long", Index: &falseVar}), + output: p.other(&asset.Field{Type: "long", Index: &falseVar}), expected: common.MapStr{ "type": "long", "index": false, }, }, { - output: p.other(&common.Field{Type: "text", Index: &trueVar}), + output: p.other(&asset.Field{Type: "text", Index: &trueVar}), expected: common.MapStr{ "type": "text", "index": true, }, }, { - output: p.other(&common.Field{Type: "long", DocValues: &falseVar}), + output: p.other(&asset.Field{Type: "long", DocValues: &falseVar}), expected: common.MapStr{ "type": "long", "doc_values": false, }, }, { - output: p.other(&common.Field{Type: "double", DocValues: &falseVar}), + output: p.other(&asset.Field{Type: "double", DocValues: &falseVar}), expected: common.MapStr{ "type": "double", "doc_values": false, }, }, { - output: p.other(&common.Field{Type: "text", DocValues: &trueVar}), + output: p.other(&asset.Field{Type: "text", DocValues: &trueVar}), expected: common.MapStr{ "type": "text", "doc_values": true, }, }, { - output: p.alias(&common.Field{Type: "alias", AliasPath: "a.c", MigrationAlias: false}), + output: p.alias(&asset.Field{Type: "alias", AliasPath: "a.c", MigrationAlias: false}), expected: common.MapStr{"path": "a.c", "type": "alias"}, }, { - output: p.alias(&common.Field{Type: "alias", AliasPath: "a.d", MigrationAlias: true}), + output: p.alias(&asset.Field{Type: "alias", AliasPath: "a.d", MigrationAlias: true}), expected: nil, }, { - output: migrationP.alias(&common.Field{Type: "alias", AliasPath: "a.e", MigrationAlias: false}), + output: migrationP.alias(&asset.Field{Type: "alias", AliasPath: "a.e", MigrationAlias: false}), expected: common.MapStr{"path": "a.e", "type": "alias"}, }, { - output: migrationP.alias(&common.Field{Type: "alias", AliasPath: "a.f", MigrationAlias: true}), + output: migrationP.alias(&asset.Field{Type: "alias", AliasPath: "a.f", MigrationAlias: true}), expected: common.MapStr{"path": "a.f", "type": "alias"}, }, } @@ -309,11 +310,11 @@ func TestProcessor(t *testing.T) { func TestDynamicTemplates(t *testing.T) { p := &Processor{} tests := []struct { - field common.Field + field asset.Field expected []common.MapStr }{ { - field: common.Field{ + field: asset.Field{ Type: "object", ObjectType: "keyword", Name: "context", }, @@ -328,7 +329,7 @@ func TestDynamicTemplates(t *testing.T) { }, }, { - field: common.Field{ + field: asset.Field{ Type: "object", ObjectType: "long", ObjectTypeMappingType: "futuretype", Path: "language", Name: "english", }, @@ -343,7 +344,7 @@ func TestDynamicTemplates(t *testing.T) { }, }, { - field: common.Field{ + field: asset.Field{ Type: "object", ObjectType: "long", ObjectTypeMappingType: "*", Path: "language", Name: "english", }, @@ -358,7 +359,7 @@ func TestDynamicTemplates(t *testing.T) { }, }, { - field: common.Field{ + field: asset.Field{ Type: "object", ObjectType: "long", Path: "language", Name: "english", }, @@ -373,7 +374,7 @@ func TestDynamicTemplates(t *testing.T) { }, }, { - field: common.Field{ + field: asset.Field{ Type: "object", ObjectType: "text", Path: "language", Name: "english", }, @@ -388,7 +389,7 @@ func TestDynamicTemplates(t *testing.T) { }, }, { - field: common.Field{ + field: asset.Field{ Type: "object", ObjectType: "scaled_float", Name: "core.*.pct", }, @@ -406,7 +407,7 @@ func TestDynamicTemplates(t *testing.T) { }, }, { - field: common.Field{ + field: asset.Field{ Type: "object", ObjectType: "scaled_float", Name: "core.*.pct", ScalingFactor: 100, ObjectTypeMappingType: "float", }, @@ -424,8 +425,8 @@ func TestDynamicTemplates(t *testing.T) { }, }, { - field: common.Field{ - Type: "object", ObjectTypeParams: []common.ObjectTypeCfg{ + field: asset.Field{ + Type: "object", ObjectTypeParams: []asset.ObjectTypeCfg{ {ObjectType: "float", ObjectTypeMappingType: "float"}, {ObjectType: "boolean"}, {ObjectType: "scaled_float", ScalingFactor: 10000}, @@ -460,10 +461,10 @@ func TestDynamicTemplates(t *testing.T) { for _, numericType := range []string{"byte", "double", "float", "long", "short", "boolean"} { gen := struct { - field common.Field + field asset.Field expected []common.MapStr }{ - field: common.Field{ + field: asset.Field{ Type: "object", ObjectType: numericType, Name: "somefield", ObjectTypeMappingType: "long", }, @@ -491,22 +492,22 @@ func TestDynamicTemplates(t *testing.T) { func TestPropertiesCombine(t *testing.T) { // Test common fields are combined even if they come from different objects - fields := common.Fields{ - common.Field{ + fields := asset.Fields{ + asset.Field{ Name: "test", Type: "group", - Fields: common.Fields{ - common.Field{ + Fields: asset.Fields{ + asset.Field{ Name: "one", Type: "text", }, }, }, - common.Field{ + asset.Field{ Name: "test", Type: "group", - Fields: common.Fields{ - common.Field{ + Fields: asset.Fields{ + asset.Field{ Name: "two", Type: "text", }, @@ -541,20 +542,20 @@ func TestPropertiesCombine(t *testing.T) { func TestProcessNoName(t *testing.T) { // Test common fields are combined even if they come from different objects - fields := common.Fields{ - common.Field{ - Fields: common.Fields{ - common.Field{ + fields := asset.Fields{ + asset.Field{ + Fields: asset.Fields{ + asset.Field{ Name: "one", Type: "text", }, }, }, - common.Field{ + asset.Field{ Name: "test", Type: "group", - Fields: common.Fields{ - common.Field{ + Fields: asset.Fields{ + asset.Field{ Name: "two", Type: "text", }, diff --git a/libbeat/template/template.go b/libbeat/template/template.go index 1e829d196b1..2ad48be2249 100644 --- a/libbeat/template/template.go +++ b/libbeat/template/template.go @@ -22,6 +22,7 @@ import ( "sync" "time" + "github.com/elastic/beats/libbeat/asset" "github.com/elastic/beats/libbeat/beat" "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/common/cfgwarn" @@ -129,7 +130,7 @@ func New( }, nil } -func (t *Template) load(fields common.Fields) (common.MapStr, error) { +func (t *Template) load(fields asset.Fields) (common.MapStr, error) { // Locking to make sure dynamicTemplates and defaultFields is not accessed in parallel t.Lock() @@ -141,7 +142,7 @@ func (t *Template) load(fields common.Fields) (common.MapStr, error) { var err error if len(t.config.AppendFields) > 0 { cfgwarn.Experimental("append_fields is used.") - fields, err = common.ConcatFields(fields, t.config.AppendFields) + fields, err = asset.ConcatFields(fields, t.config.AppendFields) if err != nil { return nil, err } @@ -160,7 +161,7 @@ func (t *Template) load(fields common.Fields) (common.MapStr, error) { // LoadFile loads the the template from the given file path func (t *Template) LoadFile(file string) (common.MapStr, error) { - fields, err := common.LoadFieldsYaml(file) + fields, err := asset.LoadFieldsYaml(file) if err != nil { return nil, err } @@ -311,19 +312,19 @@ func buildIdxSettings(ver common.Version, userSettings common.MapStr) common.Map return indexSettings } -func loadYamlByte(data []byte) (common.Fields, error) { +func loadYamlByte(data []byte) (asset.Fields, error) { cfg, err := yaml.NewConfig(data) if err != nil { return nil, err } - var keys []common.Field + var keys []asset.Field err = cfg.Unpack(&keys) if err != nil { return nil, err } - fields := common.Fields{} + fields := asset.Fields{} for _, key := range keys { fields = append(fields, key.Fields...) } From 68b7ee25a12b43687f2483c1845669ef5dadafd5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Tue, 12 Mar 2019 14:02:11 +0100 Subject: [PATCH 2/5] more moving --- libbeat/template/processor.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libbeat/template/processor.go b/libbeat/template/processor.go index 8333ed73647..ac09d2c0652 100644 --- a/libbeat/template/processor.go +++ b/libbeat/template/processor.go @@ -113,7 +113,7 @@ func (p *Processor) Process(fields asset.Fields, path string, output common.MapS return nil } -func addToDefaultFields(f *common.Field) { +func addToDefaultFields(f *asset.Field) { fullName := f.Name if f.Path != "" { fullName = f.Path + "." + f.Name @@ -124,7 +124,7 @@ func addToDefaultFields(f *common.Field) { } } -func (p *Processor) other(f *common.Field) common.MapStr { +func (p *Processor) other(f *asset.Field) common.MapStr { property := getDefaultProperties(f) if f.Type != "" { property["type"] = f.Type From 91b7b553065a3c079a80a9b016f104e309018684 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Tue, 12 Mar 2019 14:26:50 +0100 Subject: [PATCH 3/5] move more functions --- auditbeat/module/auditd/audit_linux_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/auditbeat/module/auditd/audit_linux_test.go b/auditbeat/module/auditd/audit_linux_test.go index 5c5cc10f979..cb5ef1cca07 100644 --- a/auditbeat/module/auditd/audit_linux_test.go +++ b/auditbeat/module/auditd/audit_linux_test.go @@ -33,7 +33,7 @@ import ( "github.com/prometheus/procfs" "github.com/elastic/beats/auditbeat/core" - "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/asset" "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/metricbeat/mb" mbtest "github.com/elastic/beats/metricbeat/mb/testing" @@ -102,7 +102,7 @@ func TestData(t *testing.T) { // assertFieldsAreDocumented mimics assert_fields_are_documented in Python system tests. func assertFieldsAreDocumented(t *testing.T, events []mb.Event) { - fieldsYml, err := common.LoadFieldsYaml("../../fields.yml") + fieldsYml, err := asset.LoadFieldsYaml("../../fields.yml") if err != nil { t.Fatal(err) } From 3ecc1b4473288ac20eadec6e725fc56e2e48fbe6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Wed, 13 Mar 2019 10:26:38 +0100 Subject: [PATCH 4/5] move Field/Fields to libbeat/mapping --- auditbeat/module/auditd/audit_linux_test.go | 4 +- libbeat/idxmgmt/idxmgmt_test.go | 4 +- libbeat/kibana/fields_transformer.go | 26 +- libbeat/kibana/fields_transformer_test.go | 270 ++++++++++---------- libbeat/kibana/index_pattern_generator.go | 4 +- libbeat/{asset => mapping}/field.go | 2 +- libbeat/{asset => mapping}/field_test.go | 2 +- libbeat/template/config.go | 4 +- libbeat/template/processor.go | 68 ++--- libbeat/template/processor_test.go | 138 +++++----- libbeat/template/template.go | 14 +- 11 files changed, 268 insertions(+), 268 deletions(-) rename libbeat/{asset => mapping}/field.go (99%) rename libbeat/{asset => mapping}/field_test.go (99%) diff --git a/auditbeat/module/auditd/audit_linux_test.go b/auditbeat/module/auditd/audit_linux_test.go index cb5ef1cca07..1d69d375ea3 100644 --- a/auditbeat/module/auditd/audit_linux_test.go +++ b/auditbeat/module/auditd/audit_linux_test.go @@ -33,8 +33,8 @@ import ( "github.com/prometheus/procfs" "github.com/elastic/beats/auditbeat/core" - "github.com/elastic/beats/libbeat/asset" "github.com/elastic/beats/libbeat/logp" + "github.com/elastic/beats/libbeat/mapping" "github.com/elastic/beats/metricbeat/mb" mbtest "github.com/elastic/beats/metricbeat/mb/testing" "github.com/elastic/go-libaudit" @@ -102,7 +102,7 @@ func TestData(t *testing.T) { // assertFieldsAreDocumented mimics assert_fields_are_documented in Python system tests. func assertFieldsAreDocumented(t *testing.T, events []mb.Event) { - fieldsYml, err := asset.LoadFieldsYaml("../../fields.yml") + fieldsYml, err := mapping.LoadFieldsYaml("../../fields.yml") if err != nil { t.Fatal(err) } diff --git a/libbeat/idxmgmt/idxmgmt_test.go b/libbeat/idxmgmt/idxmgmt_test.go index 82ed67448ed..339e8510be9 100644 --- a/libbeat/idxmgmt/idxmgmt_test.go +++ b/libbeat/idxmgmt/idxmgmt_test.go @@ -25,10 +25,10 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/elastic/beats/libbeat/asset" "github.com/elastic/beats/libbeat/beat" "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/idxmgmt/ilm" + "github.com/elastic/beats/libbeat/mapping" "github.com/elastic/beats/libbeat/template" ) @@ -97,7 +97,7 @@ func TestDefaultSupport_TemplateConfig(t *testing.T) { cloneCfg := func(c template.TemplateConfig) template.TemplateConfig { if c.AppendFields != nil { - tmp := make(asset.Fields, len(c.AppendFields)) + tmp := make(mapping.Fields, len(c.AppendFields)) copy(tmp, c.AppendFields) c.AppendFields = tmp } diff --git a/libbeat/kibana/fields_transformer.go b/libbeat/kibana/fields_transformer.go index 6849f9520bd..b5ffd82fa04 100644 --- a/libbeat/kibana/fields_transformer.go +++ b/libbeat/kibana/fields_transformer.go @@ -21,14 +21,14 @@ import ( "errors" "fmt" - "github.com/elastic/beats/libbeat/asset" "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/mapping" ) var v640 = common.MustNewVersion("6.4.0") type fieldsTransformer struct { - fields asset.Fields + fields mapping.Fields transformedFields []common.MapStr transformedFieldFormatMap common.MapStr version *common.Version @@ -36,7 +36,7 @@ type fieldsTransformer struct { migration bool } -func newFieldsTransformer(version *common.Version, fields asset.Fields, migration bool) (*fieldsTransformer, error) { +func newFieldsTransformer(version *common.Version, fields mapping.Fields, migration bool) (*fieldsTransformer, error) { if version == nil { return nil, errors.New("Version must be given") } @@ -65,10 +65,10 @@ func (t *fieldsTransformer) transform() (transformed common.MapStr, err error) { // add some meta fields truthy := true falsy := false - t.add(asset.Field{Path: "_id", Type: "keyword", Index: &falsy, Analyzed: &falsy, DocValues: &falsy, Searchable: &falsy, Aggregatable: &falsy}) - t.add(asset.Field{Path: "_type", Type: "keyword", Index: &falsy, Analyzed: &falsy, DocValues: &falsy, Searchable: &truthy, Aggregatable: &truthy}) - t.add(asset.Field{Path: "_index", Type: "keyword", Index: &falsy, Analyzed: &falsy, DocValues: &falsy, Searchable: &falsy, Aggregatable: &falsy}) - t.add(asset.Field{Path: "_score", Type: "integer", Index: &falsy, Analyzed: &falsy, DocValues: &falsy, Searchable: &falsy, Aggregatable: &falsy}) + t.add(mapping.Field{Path: "_id", Type: "keyword", Index: &falsy, Analyzed: &falsy, DocValues: &falsy, Searchable: &falsy, Aggregatable: &falsy}) + t.add(mapping.Field{Path: "_type", Type: "keyword", Index: &falsy, Analyzed: &falsy, DocValues: &falsy, Searchable: &truthy, Aggregatable: &truthy}) + t.add(mapping.Field{Path: "_index", Type: "keyword", Index: &falsy, Analyzed: &falsy, DocValues: &falsy, Searchable: &falsy, Aggregatable: &falsy}) + t.add(mapping.Field{Path: "_score", Type: "integer", Index: &falsy, Analyzed: &falsy, DocValues: &falsy, Searchable: &falsy, Aggregatable: &falsy}) transformed = common.MapStr{ "fields": t.transformedFields, @@ -77,7 +77,7 @@ func (t *fieldsTransformer) transform() (transformed common.MapStr, err error) { return } -func (t *fieldsTransformer) transformFields(commonFields asset.Fields, path string) { +func (t *fieldsTransformer) transformFields(commonFields mapping.Fields, path string) { for _, f := range commonFields { f.Path = f.Name if path != "" { @@ -120,7 +120,7 @@ func (t *fieldsTransformer) transformFields(commonFields asset.Fields, path stri } } -func (t *fieldsTransformer) update(target *common.MapStr, override asset.Field) error { +func (t *fieldsTransformer) update(target *common.MapStr, override mapping.Field) error { field, _ := transformField(t.version, override) if override.Type == "" || (*target)["type"] == field["type"] { target.Update(field) @@ -134,7 +134,7 @@ func (t *fieldsTransformer) update(target *common.MapStr, override asset.Field) return fmt.Errorf("field <%s> is duplicated", override.Path) } -func (t *fieldsTransformer) add(f asset.Field) { +func (t *fieldsTransformer) add(f mapping.Field) { if idx := t.keys[f.Path]; idx > 0 { target := &t.transformedFields[idx-1] // 1-indexed if err := t.update(target, f); err != nil { @@ -151,7 +151,7 @@ func (t *fieldsTransformer) add(f asset.Field) { } } -func transformField(version *common.Version, f asset.Field) (common.MapStr, common.MapStr) { +func transformField(version *common.Version, f mapping.Field) (common.MapStr, common.MapStr) { field := common.MapStr{ "name": f.Path, "count": f.Count, @@ -218,7 +218,7 @@ func getVal(valP *bool, def bool) bool { return def } -func addParams(format *common.MapStr, version *common.Version, f asset.Field) { +func addParams(format *common.MapStr, version *common.Version, f mapping.Field) { addFormatParam(format, "pattern", f.Pattern) addFormatParam(format, "inputFormat", f.InputFormat) addFormatParam(format, "outputFormat", f.OutputFormat) @@ -249,7 +249,7 @@ func addFormatParam(f *common.MapStr, key string, val interface{}) { } // takes the highest version where major version <= given version -func addVersionedFormatParam(f *common.MapStr, version *common.Version, key string, val []asset.VersionizedString) { +func addVersionedFormatParam(f *common.MapStr, version *common.Version, key string, val []mapping.VersionizedString) { if len(val) == 0 { return } diff --git a/libbeat/kibana/fields_transformer_test.go b/libbeat/kibana/fields_transformer_test.go index ae260cdc447..be2fd6864fc 100644 --- a/libbeat/kibana/fields_transformer_test.go +++ b/libbeat/kibana/fields_transformer_test.go @@ -24,8 +24,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/elastic/beats/libbeat/asset" "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/mapping" ) var ( @@ -36,7 +36,7 @@ var ( ) func TestEmpty(t *testing.T) { - trans, err := newFieldsTransformer(version, asset.Fields{}, true) + trans, err := newFieldsTransformer(version, mapping.Fields{}, true) assert.NoError(t, err) out, err := trans.transform() assert.NoError(t, err) @@ -94,26 +94,26 @@ func TestEmpty(t *testing.T) { func TestMissingVersion(t *testing.T) { var c *common.Version - _, err := newFieldsTransformer(c, asset.Fields{}, true) + _, err := newFieldsTransformer(c, mapping.Fields{}, true) assert.Error(t, err) } func TestDuplicateField(t *testing.T) { testCases := []struct { - commonFields []asset.Field + commonFields []mapping.Field }{ // type change - {commonFields: []asset.Field{ + {commonFields: []mapping.Field{ {Name: "context", Path: "something"}, {Name: "context", Path: "something", Type: "date"}, }}, // missing overwrite - {commonFields: []asset.Field{ + {commonFields: []mapping.Field{ {Name: "context", Path: "something"}, {Name: "context", Path: "something"}, }}, // missing overwrite in source - {commonFields: []asset.Field{ + {commonFields: []mapping.Field{ {Name: "context", Path: "something", Overwrite: true}, {Name: "context", Path: "something"}, }}, @@ -128,9 +128,9 @@ func TestDuplicateField(t *testing.T) { } func TestValidDuplicateField(t *testing.T) { - commonFields := asset.Fields{ - asset.Field{Name: "context", Path: "something", Type: "keyword", Description: "original description"}, - asset.Field{Name: "context", Path: "something", Overwrite: true, Description: "updated description", + commonFields := mapping.Fields{ + mapping.Field{Name: "context", Path: "something", Type: "keyword", Description: "original description"}, + mapping.Field{Name: "context", Path: "something", Overwrite: true, Description: "updated description", Aggregatable: &falsy, Analyzed: &truthy, Count: 2, @@ -138,18 +138,18 @@ func TestValidDuplicateField(t *testing.T) { Index: &falsy, Searchable: &falsy, }, - asset.Field{ + mapping.Field{ Name: "context", Type: "group", - Fields: asset.Fields{ - asset.Field{Name: "another", Type: "date"}, + Fields: mapping.Fields{ + mapping.Field{Name: "another", Type: "date"}, }, }, - asset.Field{ + mapping.Field{ Name: "context", Type: "group", - Fields: asset.Fields{ - asset.Field{Name: "another", Overwrite: true}, + Fields: mapping.Fields{ + mapping.Field{Name: "another", Overwrite: true}, }, }, } @@ -172,11 +172,11 @@ func TestValidDuplicateField(t *testing.T) { } func TestInvalidVersion(t *testing.T) { - commonFields := asset.Fields{ - asset.Field{ + commonFields := mapping.Fields{ + mapping.Field{ Name: "versionTest", Format: "url", - UrlTemplate: []asset.VersionizedString{ + UrlTemplate: []mapping.VersionizedString{ {MinVersion: "3", Value: ""}, }, }, @@ -189,26 +189,26 @@ func TestInvalidVersion(t *testing.T) { func TestTransformTypes(t *testing.T) { tests := []struct { - commonField asset.Field + commonField mapping.Field expected interface{} }{ - {commonField: asset.Field{}, expected: "string"}, - {commonField: asset.Field{Type: "half_float"}, expected: "number"}, - {commonField: asset.Field{Type: "scaled_float"}, expected: "number"}, - {commonField: asset.Field{Type: "float"}, expected: "number"}, - {commonField: asset.Field{Type: "integer"}, expected: "number"}, - {commonField: asset.Field{Type: "long"}, expected: "number"}, - {commonField: asset.Field{Type: "short"}, expected: "number"}, - {commonField: asset.Field{Type: "byte"}, expected: "number"}, - {commonField: asset.Field{Type: "keyword"}, expected: "string"}, - {commonField: asset.Field{Type: "text"}, expected: "string"}, - {commonField: asset.Field{Type: "string"}, expected: nil}, - {commonField: asset.Field{Type: "date"}, expected: "date"}, - {commonField: asset.Field{Type: "geo_point"}, expected: "geo_point"}, - {commonField: asset.Field{Type: "invalid"}, expected: nil}, + {commonField: mapping.Field{}, expected: "string"}, + {commonField: mapping.Field{Type: "half_float"}, expected: "number"}, + {commonField: mapping.Field{Type: "scaled_float"}, expected: "number"}, + {commonField: mapping.Field{Type: "float"}, expected: "number"}, + {commonField: mapping.Field{Type: "integer"}, expected: "number"}, + {commonField: mapping.Field{Type: "long"}, expected: "number"}, + {commonField: mapping.Field{Type: "short"}, expected: "number"}, + {commonField: mapping.Field{Type: "byte"}, expected: "number"}, + {commonField: mapping.Field{Type: "keyword"}, expected: "string"}, + {commonField: mapping.Field{Type: "text"}, expected: "string"}, + {commonField: mapping.Field{Type: "string"}, expected: nil}, + {commonField: mapping.Field{Type: "date"}, expected: "date"}, + {commonField: mapping.Field{Type: "geo_point"}, expected: "geo_point"}, + {commonField: mapping.Field{Type: "invalid"}, expected: nil}, } for idx, test := range tests { - trans, _ := newFieldsTransformer(version, asset.Fields{test.commonField}, true) + trans, _ := newFieldsTransformer(version, mapping.Fields{test.commonField}, true) transformed, err := trans.transform() assert.NoError(t, err) out := transformed["fields"].([]common.MapStr)[0] @@ -218,32 +218,32 @@ func TestTransformTypes(t *testing.T) { func TestTransformGroup(t *testing.T) { tests := []struct { - commonFields asset.Fields + commonFields mapping.Fields expected []string }{ { - commonFields: asset.Fields{asset.Field{Name: "context", Path: "something"}}, + commonFields: mapping.Fields{mapping.Field{Name: "context", Path: "something"}}, expected: []string{"context"}, }, { - commonFields: asset.Fields{ - asset.Field{ + commonFields: mapping.Fields{ + mapping.Field{ Name: "context", Type: "group", - Fields: asset.Fields{ - asset.Field{Name: "another", Type: ""}, + Fields: mapping.Fields{ + mapping.Field{Name: "another", Type: ""}, }, }, - asset.Field{ + mapping.Field{ Name: "context", Type: "group", - Fields: asset.Fields{ - asset.Field{Name: "type", Type: ""}, - asset.Field{ + Fields: mapping.Fields{ + mapping.Field{Name: "type", Type: ""}, + mapping.Field{ Name: "metric", Type: "group", - Fields: asset.Fields{ - asset.Field{Name: "object"}, + Fields: mapping.Fields{ + mapping.Field{Name: "object"}, }, }, }, @@ -266,70 +266,70 @@ func TestTransformGroup(t *testing.T) { func TestTransformMisc(t *testing.T) { tests := []struct { - commonField asset.Field + commonField mapping.Field expected interface{} attr string }{ - {commonField: asset.Field{}, expected: 0, attr: "count"}, - {commonField: asset.Field{Count: 4}, expected: 4, attr: "count"}, + {commonField: mapping.Field{}, expected: 0, attr: "count"}, + {commonField: mapping.Field{Count: 4}, expected: 4, attr: "count"}, // searchable - {commonField: asset.Field{}, expected: true, attr: "searchable"}, - {commonField: asset.Field{Searchable: &truthy}, expected: true, attr: "searchable"}, - {commonField: asset.Field{Searchable: &falsy}, expected: false, attr: "searchable"}, - {commonField: asset.Field{Type: "binary"}, expected: false, attr: "searchable"}, - {commonField: asset.Field{Searchable: &truthy, Type: "binary"}, expected: false, attr: "searchable"}, + {commonField: mapping.Field{}, expected: true, attr: "searchable"}, + {commonField: mapping.Field{Searchable: &truthy}, expected: true, attr: "searchable"}, + {commonField: mapping.Field{Searchable: &falsy}, expected: false, attr: "searchable"}, + {commonField: mapping.Field{Type: "binary"}, expected: false, attr: "searchable"}, + {commonField: mapping.Field{Searchable: &truthy, Type: "binary"}, expected: false, attr: "searchable"}, // aggregatable - {commonField: asset.Field{}, expected: true, attr: "aggregatable"}, - {commonField: asset.Field{Aggregatable: &truthy}, expected: true, attr: "aggregatable"}, - {commonField: asset.Field{Aggregatable: &falsy}, expected: false, attr: "aggregatable"}, - {commonField: asset.Field{Type: "binary"}, expected: false, attr: "aggregatable"}, - {commonField: asset.Field{Aggregatable: &truthy, Type: "binary"}, expected: false, attr: "aggregatable"}, - {commonField: asset.Field{Type: "keyword"}, expected: true, attr: "aggregatable"}, - {commonField: asset.Field{Aggregatable: &truthy, Type: "text"}, expected: false, attr: "aggregatable"}, - {commonField: asset.Field{Type: "text"}, expected: false, attr: "aggregatable"}, + {commonField: mapping.Field{}, expected: true, attr: "aggregatable"}, + {commonField: mapping.Field{Aggregatable: &truthy}, expected: true, attr: "aggregatable"}, + {commonField: mapping.Field{Aggregatable: &falsy}, expected: false, attr: "aggregatable"}, + {commonField: mapping.Field{Type: "binary"}, expected: false, attr: "aggregatable"}, + {commonField: mapping.Field{Aggregatable: &truthy, Type: "binary"}, expected: false, attr: "aggregatable"}, + {commonField: mapping.Field{Type: "keyword"}, expected: true, attr: "aggregatable"}, + {commonField: mapping.Field{Aggregatable: &truthy, Type: "text"}, expected: false, attr: "aggregatable"}, + {commonField: mapping.Field{Type: "text"}, expected: false, attr: "aggregatable"}, // analyzed - {commonField: asset.Field{}, expected: false, attr: "analyzed"}, - {commonField: asset.Field{Analyzed: &truthy}, expected: true, attr: "analyzed"}, - {commonField: asset.Field{Analyzed: &falsy}, expected: false, attr: "analyzed"}, - {commonField: asset.Field{Type: "binary"}, expected: false, attr: "analyzed"}, - {commonField: asset.Field{Analyzed: &truthy, Type: "binary"}, expected: false, attr: "analyzed"}, + {commonField: mapping.Field{}, expected: false, attr: "analyzed"}, + {commonField: mapping.Field{Analyzed: &truthy}, expected: true, attr: "analyzed"}, + {commonField: mapping.Field{Analyzed: &falsy}, expected: false, attr: "analyzed"}, + {commonField: mapping.Field{Type: "binary"}, expected: false, attr: "analyzed"}, + {commonField: mapping.Field{Analyzed: &truthy, Type: "binary"}, expected: false, attr: "analyzed"}, // doc_values always set to true except for meta fields - {commonField: asset.Field{}, expected: true, attr: "doc_values"}, - {commonField: asset.Field{DocValues: &truthy}, expected: true, attr: "doc_values"}, - {commonField: asset.Field{DocValues: &falsy}, expected: false, attr: "doc_values"}, - {commonField: asset.Field{Script: "doc[]"}, expected: false, attr: "doc_values"}, - {commonField: asset.Field{DocValues: &truthy, Script: "doc[]"}, expected: false, attr: "doc_values"}, - {commonField: asset.Field{Type: "binary"}, expected: false, attr: "doc_values"}, - {commonField: asset.Field{DocValues: &truthy, Type: "binary"}, expected: true, attr: "doc_values"}, + {commonField: mapping.Field{}, expected: true, attr: "doc_values"}, + {commonField: mapping.Field{DocValues: &truthy}, expected: true, attr: "doc_values"}, + {commonField: mapping.Field{DocValues: &falsy}, expected: false, attr: "doc_values"}, + {commonField: mapping.Field{Script: "doc[]"}, expected: false, attr: "doc_values"}, + {commonField: mapping.Field{DocValues: &truthy, Script: "doc[]"}, expected: false, attr: "doc_values"}, + {commonField: mapping.Field{Type: "binary"}, expected: false, attr: "doc_values"}, + {commonField: mapping.Field{DocValues: &truthy, Type: "binary"}, expected: true, attr: "doc_values"}, // enabled - only applies to objects (and only if set) - {commonField: asset.Field{Type: "binary", Enabled: &falsy}, expected: nil, attr: "enabled"}, - {commonField: asset.Field{Type: "binary", Enabled: &truthy}, expected: nil, attr: "enabled"}, - {commonField: asset.Field{Type: "object", Enabled: &truthy}, expected: true, attr: "enabled"}, - {commonField: asset.Field{Type: "object", Enabled: &falsy}, expected: false, attr: "enabled"}, - {commonField: asset.Field{Type: "object", Enabled: &falsy}, expected: false, attr: "doc_values"}, + {commonField: mapping.Field{Type: "binary", Enabled: &falsy}, expected: nil, attr: "enabled"}, + {commonField: mapping.Field{Type: "binary", Enabled: &truthy}, expected: nil, attr: "enabled"}, + {commonField: mapping.Field{Type: "object", Enabled: &truthy}, expected: true, attr: "enabled"}, + {commonField: mapping.Field{Type: "object", Enabled: &falsy}, expected: false, attr: "enabled"}, + {commonField: mapping.Field{Type: "object", Enabled: &falsy}, expected: false, attr: "doc_values"}, // indexed - {commonField: asset.Field{Type: "binary"}, expected: false, attr: "indexed"}, - {commonField: asset.Field{Index: &truthy, Type: "binary"}, expected: false, attr: "indexed"}, + {commonField: mapping.Field{Type: "binary"}, expected: false, attr: "indexed"}, + {commonField: mapping.Field{Index: &truthy, Type: "binary"}, expected: false, attr: "indexed"}, // script, scripted - {commonField: asset.Field{}, expected: false, attr: "scripted"}, - {commonField: asset.Field{}, expected: nil, attr: "script"}, - {commonField: asset.Field{Script: "doc[]"}, expected: true, attr: "scripted"}, - {commonField: asset.Field{Script: "doc[]"}, expected: "doc[]", attr: "script"}, - {commonField: asset.Field{Type: "binary"}, expected: false, attr: "scripted"}, + {commonField: mapping.Field{}, expected: false, attr: "scripted"}, + {commonField: mapping.Field{}, expected: nil, attr: "script"}, + {commonField: mapping.Field{Script: "doc[]"}, expected: true, attr: "scripted"}, + {commonField: mapping.Field{Script: "doc[]"}, expected: "doc[]", attr: "script"}, + {commonField: mapping.Field{Type: "binary"}, expected: false, attr: "scripted"}, // language - {commonField: asset.Field{}, expected: nil, attr: "lang"}, - {commonField: asset.Field{Script: "doc[]"}, expected: "painless", attr: "lang"}, + {commonField: mapping.Field{}, expected: nil, attr: "lang"}, + {commonField: mapping.Field{Script: "doc[]"}, expected: "painless", attr: "lang"}, } for idx, test := range tests { - trans, _ := newFieldsTransformer(version, asset.Fields{test.commonField}, true) + trans, _ := newFieldsTransformer(version, mapping.Fields{test.commonField}, true) transformed, err := trans.transform() assert.NoError(t, err) out := transformed["fields"].([]common.MapStr)[0] @@ -345,27 +345,27 @@ func TestTransformFieldFormatMap(t *testing.T) { falsy := false tests := []struct { - commonField asset.Field + commonField mapping.Field version *common.Version expected common.MapStr }{ { - commonField: asset.Field{Name: "c"}, + commonField: mapping.Field{Name: "c"}, expected: common.MapStr{}, version: version, }, { - commonField: asset.Field{Name: "c", Format: "url"}, + commonField: mapping.Field{Name: "c", Format: "url"}, expected: common.MapStr{"c": common.MapStr{"id": "url"}}, version: version, }, { - commonField: asset.Field{Name: "c", Pattern: "p"}, + commonField: mapping.Field{Name: "c", Pattern: "p"}, expected: common.MapStr{"c": common.MapStr{"params": common.MapStr{"pattern": "p"}}}, version: version, }, { - commonField: asset.Field{ + commonField: mapping.Field{ Name: "c", Format: "url", Pattern: "p", @@ -379,7 +379,7 @@ func TestTransformFieldFormatMap(t *testing.T) { version: version, }, { - commonField: asset.Field{ + commonField: mapping.Field{ Name: "c", Format: "url", InputFormat: "string", @@ -395,7 +395,7 @@ func TestTransformFieldFormatMap(t *testing.T) { version: version, }, { - commonField: asset.Field{ + commonField: mapping.Field{ Name: "c", Format: "url", Pattern: "[^-]", @@ -415,7 +415,7 @@ func TestTransformFieldFormatMap(t *testing.T) { version: version, }, { - commonField: asset.Field{ + commonField: mapping.Field{ Name: "c", InputFormat: "string", }, @@ -424,7 +424,7 @@ func TestTransformFieldFormatMap(t *testing.T) { }, { version: version620, - commonField: asset.Field{ + commonField: mapping.Field{ Name: "c", Format: "url", Pattern: "[^-]", @@ -433,7 +433,7 @@ func TestTransformFieldFormatMap(t *testing.T) { OutputFormat: "float", OutputPrecision: &precision, LabelTemplate: "lblT", - UrlTemplate: []asset.VersionizedString{ + UrlTemplate: []mapping.VersionizedString{ {MinVersion: "5.0.0", Value: "5x.urlTemplate"}, {MinVersion: "6.0.0", Value: "6x.urlTemplate"}, }, @@ -455,10 +455,10 @@ func TestTransformFieldFormatMap(t *testing.T) { }, { version: version620, - commonField: asset.Field{ + commonField: mapping.Field{ Name: "c", Format: "url", - UrlTemplate: []asset.VersionizedString{ + UrlTemplate: []mapping.VersionizedString{ {MinVersion: "6.4.0", Value: "6x.urlTemplate"}, }, }, @@ -468,10 +468,10 @@ func TestTransformFieldFormatMap(t *testing.T) { }, { version: version620, - commonField: asset.Field{ + commonField: mapping.Field{ Name: "c", Format: "url", - UrlTemplate: []asset.VersionizedString{ + UrlTemplate: []mapping.VersionizedString{ {MinVersion: "4.7.2", Value: "4x.urlTemplate"}, {MinVersion: "6.5.1", Value: "6x.urlTemplate"}, }, @@ -487,10 +487,10 @@ func TestTransformFieldFormatMap(t *testing.T) { }, { version: version620, - commonField: asset.Field{ + commonField: mapping.Field{ Name: "c", Format: "url", - UrlTemplate: []asset.VersionizedString{ + UrlTemplate: []mapping.VersionizedString{ {MinVersion: "6.2.0", Value: "6.2.0.urlTemplate"}, {MinVersion: "6.2.0-alpha", Value: "6.2.0-alpha.urlTemplate"}, {MinVersion: "6.2.7", Value: "6.2.7.urlTemplate"}, @@ -507,10 +507,10 @@ func TestTransformFieldFormatMap(t *testing.T) { }, { version: version620, - commonField: asset.Field{ + commonField: mapping.Field{ Name: "c", Format: "url", - UrlTemplate: []asset.VersionizedString{ + UrlTemplate: []mapping.VersionizedString{ {MinVersion: "4.1.0", Value: "4x.urlTemplate"}, {MinVersion: "5.2.0-rc2", Value: "5.2.0-rc2.urlTemplate"}, {MinVersion: "5.2.0-rc3", Value: "5.2.0-rc3.urlTemplate"}, @@ -528,7 +528,7 @@ func TestTransformFieldFormatMap(t *testing.T) { }, } for idx, test := range tests { - trans, _ := newFieldsTransformer(test.version, asset.Fields{test.commonField}, true) + trans, _ := newFieldsTransformer(test.version, mapping.Fields{test.commonField}, true) transformed, err := trans.transform() assert.NoError(t, err) out := transformed["fieldFormatMap"] @@ -538,25 +538,25 @@ func TestTransformFieldFormatMap(t *testing.T) { func TestTransformGroupAndEnabled(t *testing.T) { tests := []struct { - commonFields asset.Fields + commonFields mapping.Fields expected []string }{ { - commonFields: asset.Fields{asset.Field{Name: "context", Path: "something"}}, + commonFields: mapping.Fields{mapping.Field{Name: "context", Path: "something"}}, expected: []string{"context"}, }, { - commonFields: asset.Fields{ - asset.Field{ + commonFields: mapping.Fields{ + mapping.Field{ Name: "context", Type: "group", - Fields: asset.Fields{ - asset.Field{Name: "type", Type: ""}, - asset.Field{ + Fields: mapping.Fields{ + mapping.Field{Name: "type", Type: ""}, + mapping.Field{ Name: "metric", Type: "group", - Fields: asset.Fields{ - asset.Field{Name: "object"}, + Fields: mapping.Fields{ + mapping.Field{Name: "object"}, }, }, }, @@ -565,28 +565,28 @@ func TestTransformGroupAndEnabled(t *testing.T) { expected: []string{"context.type", "context.metric.object"}, }, { - commonFields: asset.Fields{ - asset.Field{Name: "enabledField"}, - asset.Field{Name: "disabledField", Enabled: &falsy}, //enabled is ignored for Type!=group - asset.Field{ + commonFields: mapping.Fields{ + mapping.Field{Name: "enabledField"}, + mapping.Field{Name: "disabledField", Enabled: &falsy}, //enabled is ignored for Type!=group + mapping.Field{ Name: "enabledGroup", Type: "group", Enabled: &truthy, - Fields: asset.Fields{ - asset.Field{Name: "type", Type: ""}, + Fields: mapping.Fields{ + mapping.Field{Name: "type", Type: ""}, }, }, - asset.Field{ + mapping.Field{ Name: "context", Type: "group", Enabled: &falsy, - Fields: asset.Fields{ - asset.Field{Name: "type", Type: ""}, - asset.Field{ + Fields: mapping.Fields{ + mapping.Field{Name: "type", Type: ""}, + mapping.Field{ Name: "metric", Type: "group", - Fields: asset.Fields{ - asset.Field{Name: "object"}, + Fields: mapping.Fields{ + mapping.Field{Name: "object"}, }, }, }, @@ -608,15 +608,15 @@ func TestTransformGroupAndEnabled(t *testing.T) { } func TestTransformMultiField(t *testing.T) { - f := asset.Field{ + f := mapping.Field{ Name: "context", Type: "", - MultiFields: asset.Fields{ - asset.Field{Name: "keyword", Type: "keyword"}, - asset.Field{Name: "text", Type: "text"}, + MultiFields: mapping.Fields{ + mapping.Field{Name: "keyword", Type: "keyword"}, + mapping.Field{Name: "text", Type: "text"}, }, } - trans, _ := newFieldsTransformer(version, asset.Fields{f}, true) + trans, _ := newFieldsTransformer(version, mapping.Fields{f}, true) transformed, err := trans.transform() assert.NoError(t, err) out := transformed["fields"].([]common.MapStr) diff --git a/libbeat/kibana/index_pattern_generator.go b/libbeat/kibana/index_pattern_generator.go index 1222881ddc3..93b5406c2de 100644 --- a/libbeat/kibana/index_pattern_generator.go +++ b/libbeat/kibana/index_pattern_generator.go @@ -24,8 +24,8 @@ import ( "path/filepath" "regexp" - "github.com/elastic/beats/libbeat/asset" "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/mapping" ) type IndexPatternGenerator struct { @@ -111,7 +111,7 @@ func (i *IndexPatternGenerator) addGeneral(indexPattern *common.MapStr) error { } func (i *IndexPatternGenerator) addFieldsSpecific(indexPattern *common.MapStr) error { - fields, err := asset.LoadFields(i.fields) + fields, err := mapping.LoadFields(i.fields) if err != nil { return err } diff --git a/libbeat/asset/field.go b/libbeat/mapping/field.go similarity index 99% rename from libbeat/asset/field.go rename to libbeat/mapping/field.go index 9df7489046c..fc3a7d3073f 100644 --- a/libbeat/asset/field.go +++ b/libbeat/mapping/field.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package asset +package mapping import ( "fmt" diff --git a/libbeat/asset/field_test.go b/libbeat/mapping/field_test.go similarity index 99% rename from libbeat/asset/field_test.go rename to libbeat/mapping/field_test.go index b5d266c0f12..6d2bfe49551 100644 --- a/libbeat/asset/field_test.go +++ b/libbeat/mapping/field_test.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package asset +package mapping import ( "strings" diff --git a/libbeat/template/config.go b/libbeat/template/config.go index 6143e2ec31f..53d9ac5eacb 100644 --- a/libbeat/template/config.go +++ b/libbeat/template/config.go @@ -17,7 +17,7 @@ package template -import "github.com/elastic/beats/libbeat/asset" +import "github.com/elastic/beats/libbeat/mapping" type TemplateConfig struct { Enabled bool `config:"enabled"` @@ -29,7 +29,7 @@ type TemplateConfig struct { Path string `config:"path"` Name string `config:"name"` } `config:"json"` - AppendFields asset.Fields `config:"append_fields"` + AppendFields mapping.Fields `config:"append_fields"` Overwrite bool `config:"overwrite"` Settings TemplateSettings `config:"settings"` } diff --git a/libbeat/template/processor.go b/libbeat/template/processor.go index ac09d2c0652..4d316c52892 100644 --- a/libbeat/template/processor.go +++ b/libbeat/template/processor.go @@ -21,8 +21,8 @@ import ( "errors" "strings" - "github.com/elastic/beats/libbeat/asset" "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/mapping" ) // Processor struct to process fields to template @@ -39,7 +39,7 @@ var ( const scalingFactorKey = "scalingFactor" // Process recursively processes the given fields and writes the template in the given output -func (p *Processor) Process(fields asset.Fields, path string, output common.MapStr) error { +func (p *Processor) Process(fields mapping.Fields, path string, output common.MapStr) error { for _, field := range fields { if field.Name == "" { @@ -47,27 +47,27 @@ func (p *Processor) Process(fields asset.Fields, path string, output common.MapS } field.Path = path - var mapping common.MapStr + var indexMapping common.MapStr switch field.Type { case "ip": - mapping = p.ip(&field) + indexMapping = p.ip(&field) case "scaled_float": - mapping = p.scaledFloat(&field) + indexMapping = p.scaledFloat(&field) case "half_float": - mapping = p.halfFloat(&field) + indexMapping = p.halfFloat(&field) case "integer": - mapping = p.integer(&field) + indexMapping = p.integer(&field) case "text": - mapping = p.text(&field) + indexMapping = p.text(&field) case "", "keyword": - mapping = p.keyword(&field) + indexMapping = p.keyword(&field) case "object": - mapping = p.object(&field) + indexMapping = p.object(&field) case "array": - mapping = p.array(&field) + indexMapping = p.array(&field) case "alias": - mapping = p.alias(&field) + indexMapping = p.alias(&field) case "group": var newPath string if path == "" { @@ -75,14 +75,14 @@ func (p *Processor) Process(fields asset.Fields, path string, output common.MapS } else { newPath = path + "." + field.Name } - mapping = common.MapStr{} + indexMapping = common.MapStr{} if field.Dynamic.Value != nil { - mapping["dynamic"] = field.Dynamic.Value + indexMapping["dynamic"] = field.Dynamic.Value } // Combine properties with previous field definitions (if any) properties := common.MapStr{} - key := asset.GenerateKey(field.Name) + ".properties" + key := mapping.GenerateKey(field.Name) + ".properties" currentProperties, err := output.GetValue(key) if err == nil { var ok bool @@ -96,9 +96,9 @@ func (p *Processor) Process(fields asset.Fields, path string, output common.MapS if err := p.Process(field.Fields, newPath, properties); err != nil { return err } - mapping["properties"] = properties + indexMapping["properties"] = properties default: - mapping = p.other(&field) + indexMapping = p.other(&field) } switch field.Type { @@ -106,14 +106,14 @@ func (p *Processor) Process(fields asset.Fields, path string, output common.MapS addToDefaultFields(&field) } - if len(mapping) > 0 { - output.Put(asset.GenerateKey(field.Name), mapping) + if len(indexMapping) > 0 { + output.Put(mapping.GenerateKey(field.Name), indexMapping) } } return nil } -func addToDefaultFields(f *asset.Field) { +func addToDefaultFields(f *mapping.Field) { fullName := f.Name if f.Path != "" { fullName = f.Path + "." + f.Name @@ -124,7 +124,7 @@ func addToDefaultFields(f *asset.Field) { } } -func (p *Processor) other(f *asset.Field) common.MapStr { +func (p *Processor) other(f *mapping.Field) common.MapStr { property := getDefaultProperties(f) if f.Type != "" { property["type"] = f.Type @@ -133,13 +133,13 @@ func (p *Processor) other(f *asset.Field) common.MapStr { return property } -func (p *Processor) integer(f *asset.Field) common.MapStr { +func (p *Processor) integer(f *mapping.Field) common.MapStr { property := getDefaultProperties(f) property["type"] = "long" return property } -func (p *Processor) scaledFloat(f *asset.Field, params ...common.MapStr) common.MapStr { +func (p *Processor) scaledFloat(f *mapping.Field, params ...common.MapStr) common.MapStr { property := getDefaultProperties(f) property["type"] = "scaled_float" @@ -164,7 +164,7 @@ func (p *Processor) scaledFloat(f *asset.Field, params ...common.MapStr) common. return property } -func (p *Processor) halfFloat(f *asset.Field) common.MapStr { +func (p *Processor) halfFloat(f *mapping.Field) common.MapStr { property := getDefaultProperties(f) property["type"] = "half_float" @@ -174,7 +174,7 @@ func (p *Processor) halfFloat(f *asset.Field) common.MapStr { return property } -func (p *Processor) ip(f *asset.Field) common.MapStr { +func (p *Processor) ip(f *mapping.Field) common.MapStr { property := getDefaultProperties(f) property["type"] = "ip" @@ -187,7 +187,7 @@ func (p *Processor) ip(f *asset.Field) common.MapStr { return property } -func (p *Processor) keyword(f *asset.Field) common.MapStr { +func (p *Processor) keyword(f *mapping.Field) common.MapStr { property := getDefaultProperties(f) property["type"] = "keyword" @@ -214,7 +214,7 @@ func (p *Processor) keyword(f *asset.Field) common.MapStr { return property } -func (p *Processor) text(f *asset.Field) common.MapStr { +func (p *Processor) text(f *mapping.Field) common.MapStr { properties := getDefaultProperties(f) properties["type"] = "text" @@ -250,7 +250,7 @@ func (p *Processor) text(f *asset.Field) common.MapStr { return properties } -func (p *Processor) array(f *asset.Field) common.MapStr { +func (p *Processor) array(f *mapping.Field) common.MapStr { properties := getDefaultProperties(f) if f.ObjectType != "" { properties["type"] = f.ObjectType @@ -258,7 +258,7 @@ func (p *Processor) array(f *asset.Field) common.MapStr { return properties } -func (p *Processor) alias(f *asset.Field) common.MapStr { +func (p *Processor) alias(f *mapping.Field) common.MapStr { // Aliases were introduced in Elasticsearch 6.4, ignore if unsupported if p.EsVersion.LessThan(common.MustNewVersion("6.4.0")) { return nil @@ -275,7 +275,7 @@ func (p *Processor) alias(f *asset.Field) common.MapStr { return properties } -func (p *Processor) object(f *asset.Field) common.MapStr { +func (p *Processor) object(f *mapping.Field) common.MapStr { matchType := func(onlyType string, mt string) string { if mt != "" { return mt @@ -283,11 +283,11 @@ func (p *Processor) object(f *asset.Field) common.MapStr { return onlyType } - var otParams []asset.ObjectTypeCfg + var otParams []mapping.ObjectTypeCfg if len(f.ObjectTypeParams) != 0 { otParams = f.ObjectTypeParams } else { - otParams = []asset.ObjectTypeCfg{asset.ObjectTypeCfg{ + otParams = []mapping.ObjectTypeCfg{mapping.ObjectTypeCfg{ ObjectType: f.ObjectType, ObjectTypeMappingType: f.ObjectTypeMappingType, ScalingFactor: f.ScalingFactor}} } @@ -328,7 +328,7 @@ func (p *Processor) object(f *asset.Field) common.MapStr { return properties } -func addDynamicTemplate(f *asset.Field, properties common.MapStr, matchType string) { +func addDynamicTemplate(f *mapping.Field, properties common.MapStr, matchType string) { path := "" if len(f.Path) > 0 { path = f.Path + "." @@ -349,7 +349,7 @@ func addDynamicTemplate(f *asset.Field, properties common.MapStr, matchType stri dynamicTemplates = append(dynamicTemplates, template) } -func getDefaultProperties(f *asset.Field) common.MapStr { +func getDefaultProperties(f *mapping.Field) common.MapStr { // Currently no defaults exist properties := common.MapStr{} diff --git a/libbeat/template/processor_test.go b/libbeat/template/processor_test.go index b71064b007a..e97bb35219e 100644 --- a/libbeat/template/processor_test.go +++ b/libbeat/template/processor_test.go @@ -22,8 +22,8 @@ import ( "github.com/stretchr/testify/assert" - "github.com/elastic/beats/libbeat/asset" "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/mapping" ) func TestProcessor(t *testing.T) { @@ -40,92 +40,92 @@ func TestProcessor(t *testing.T) { expected common.MapStr }{ { - output: p.other(&asset.Field{Type: "long"}), + output: p.other(&mapping.Field{Type: "long"}), expected: common.MapStr{"type": "long"}, }, { - output: p.scaledFloat(&asset.Field{Type: "scaled_float"}), + output: p.scaledFloat(&mapping.Field{Type: "scaled_float"}), expected: common.MapStr{ "type": "scaled_float", "scaling_factor": 1000, }, }, { - output: p.scaledFloat(&asset.Field{Type: "scaled_float", ScalingFactor: 100}), + output: p.scaledFloat(&mapping.Field{Type: "scaled_float", ScalingFactor: 100}), expected: common.MapStr{ "type": "scaled_float", "scaling_factor": 100, }, }, { - output: p.scaledFloat(&asset.Field{Type: "scaled_float"}, common.MapStr{scalingFactorKey: 0}), + output: p.scaledFloat(&mapping.Field{Type: "scaled_float"}, common.MapStr{scalingFactorKey: 0}), expected: common.MapStr{ "type": "scaled_float", "scaling_factor": 1000, }, }, { - output: p.scaledFloat(&asset.Field{Type: "scaled_float"}, common.MapStr{"someKey": 10}), + output: p.scaledFloat(&mapping.Field{Type: "scaled_float"}, common.MapStr{"someKey": 10}), expected: common.MapStr{ "type": "scaled_float", "scaling_factor": 1000, }, }, { - output: p.scaledFloat(&asset.Field{Type: "scaled_float"}, common.MapStr{scalingFactorKey: 10}), + output: p.scaledFloat(&mapping.Field{Type: "scaled_float"}, common.MapStr{scalingFactorKey: 10}), expected: common.MapStr{ "type": "scaled_float", "scaling_factor": 10, }, }, { - output: pEsVersion2.scaledFloat(&asset.Field{Type: "scaled_float"}), + output: pEsVersion2.scaledFloat(&mapping.Field{Type: "scaled_float"}), expected: common.MapStr{"type": "float"}, }, { - output: p.object(&asset.Field{Type: "object", Enabled: &falseVar}), + output: p.object(&mapping.Field{Type: "object", Enabled: &falseVar}), expected: common.MapStr{ "type": "object", "enabled": false, }, }, { - output: p.integer(&asset.Field{Type: "long", CopyTo: "hello.world"}), + output: p.integer(&mapping.Field{Type: "long", CopyTo: "hello.world"}), expected: common.MapStr{ "type": "long", "copy_to": "hello.world", }, }, { - output: p.array(&asset.Field{Type: "array"}), + output: p.array(&mapping.Field{Type: "array"}), expected: common.MapStr{}, }, { - output: p.array(&asset.Field{Type: "array", ObjectType: "text"}), + output: p.array(&mapping.Field{Type: "array", ObjectType: "text"}), expected: common.MapStr{"type": "text"}, }, { - output: p.array(&asset.Field{Type: "array", Index: &falseVar, ObjectType: "keyword"}), + output: p.array(&mapping.Field{Type: "array", Index: &falseVar, ObjectType: "keyword"}), expected: common.MapStr{"index": false, "type": "keyword"}, }, { - output: pEsVersion64.alias(&asset.Field{Type: "alias", AliasPath: "a.b"}), + output: pEsVersion64.alias(&mapping.Field{Type: "alias", AliasPath: "a.b"}), expected: common.MapStr{"path": "a.b", "type": "alias"}, }, { // alias unsupported in ES < 6.4 - output: pEsVersion63.alias(&asset.Field{Type: "alias", AliasPath: "a.b"}), + output: pEsVersion63.alias(&mapping.Field{Type: "alias", AliasPath: "a.b"}), expected: nil, }, { - output: p.object(&asset.Field{Type: "object", Enabled: &falseVar}), + output: p.object(&mapping.Field{Type: "object", Enabled: &falseVar}), expected: common.MapStr{ "type": "object", "enabled": false, }, }, { - output: p.text(&asset.Field{Type: "text", Analyzer: "autocomplete"}), + output: p.text(&mapping.Field{Type: "text", Analyzer: "autocomplete"}), expected: common.MapStr{ "type": "text", "analyzer": "autocomplete", @@ -133,21 +133,21 @@ func TestProcessor(t *testing.T) { }, }, { - output: p.text(&asset.Field{Type: "text", Analyzer: "autocomplete", Norms: true}), + output: p.text(&mapping.Field{Type: "text", Analyzer: "autocomplete", Norms: true}), expected: common.MapStr{ "type": "text", "analyzer": "autocomplete", }, }, { - output: p.text(&asset.Field{Type: "text", SearchAnalyzer: "standard", Norms: true}), + output: p.text(&mapping.Field{Type: "text", SearchAnalyzer: "standard", Norms: true}), expected: common.MapStr{ "type": "text", "search_analyzer": "standard", }, }, { - output: p.text(&asset.Field{Type: "text", Analyzer: "autocomplete", SearchAnalyzer: "standard", Norms: true}), + output: p.text(&mapping.Field{Type: "text", Analyzer: "autocomplete", SearchAnalyzer: "standard", Norms: true}), expected: common.MapStr{ "type": "text", "analyzer": "autocomplete", @@ -155,7 +155,7 @@ func TestProcessor(t *testing.T) { }, }, { - output: p.text(&asset.Field{Type: "text", MultiFields: asset.Fields{asset.Field{Name: "raw", Type: "keyword"}}, Norms: true}), + output: p.text(&mapping.Field{Type: "text", MultiFields: mapping.Fields{mapping.Field{Name: "raw", Type: "keyword"}}, Norms: true}), expected: common.MapStr{ "type": "text", "fields": common.MapStr{ @@ -167,7 +167,7 @@ func TestProcessor(t *testing.T) { }, }, { - output: p.keyword(&asset.Field{Type: "keyword", MultiFields: asset.Fields{asset.Field{Name: "analyzed", Type: "text", Norms: true}}}), + output: p.keyword(&mapping.Field{Type: "keyword", MultiFields: mapping.Fields{mapping.Field{Name: "analyzed", Type: "text", Norms: true}}}), expected: common.MapStr{ "type": "keyword", "ignore_above": 1024, @@ -179,29 +179,29 @@ func TestProcessor(t *testing.T) { }, }, { - output: p.keyword(&asset.Field{Type: "keyword", IgnoreAbove: 256}), + output: p.keyword(&mapping.Field{Type: "keyword", IgnoreAbove: 256}), expected: common.MapStr{ "type": "keyword", "ignore_above": 256, }, }, { - output: p.keyword(&asset.Field{Type: "keyword", IgnoreAbove: -1}), + output: p.keyword(&mapping.Field{Type: "keyword", IgnoreAbove: -1}), expected: common.MapStr{ "type": "keyword", }, }, { - output: p.keyword(&asset.Field{Type: "keyword"}), + output: p.keyword(&mapping.Field{Type: "keyword"}), expected: common.MapStr{ "type": "keyword", "ignore_above": 1024, }, }, { - output: p.text(&asset.Field{Type: "text", MultiFields: asset.Fields{ - asset.Field{Name: "raw", Type: "keyword"}, - asset.Field{Name: "indexed", Type: "text"}, + output: p.text(&mapping.Field{Type: "text", MultiFields: mapping.Fields{ + mapping.Field{Name: "raw", Type: "keyword"}, + mapping.Field{Name: "indexed", Type: "text"}, }, Norms: true}), expected: common.MapStr{ "type": "text", @@ -218,9 +218,9 @@ func TestProcessor(t *testing.T) { }, }, { - output: p.text(&asset.Field{Type: "text", MultiFields: asset.Fields{ - asset.Field{Name: "raw", Type: "keyword"}, - asset.Field{Name: "indexed", Type: "text"}, + output: p.text(&mapping.Field{Type: "text", MultiFields: mapping.Fields{ + mapping.Field{Name: "raw", Type: "keyword"}, + mapping.Field{Name: "indexed", Type: "text"}, }, Norms: true}), expected: common.MapStr{ "type": "text", @@ -237,67 +237,67 @@ func TestProcessor(t *testing.T) { }, }, { - output: p.object(&asset.Field{Dynamic: asset.DynamicType{Value: false}}), + output: p.object(&mapping.Field{Dynamic: mapping.DynamicType{Value: false}}), expected: common.MapStr{ "dynamic": false, "type": "object", }, }, { - output: p.object(&asset.Field{Dynamic: asset.DynamicType{Value: true}}), + output: p.object(&mapping.Field{Dynamic: mapping.DynamicType{Value: true}}), expected: common.MapStr{ "dynamic": true, "type": "object", }, }, { - output: p.object(&asset.Field{Dynamic: asset.DynamicType{Value: "strict"}}), + output: p.object(&mapping.Field{Dynamic: mapping.DynamicType{Value: "strict"}}), expected: common.MapStr{ "dynamic": "strict", "type": "object", }, }, { - output: p.other(&asset.Field{Type: "long", Index: &falseVar}), + output: p.other(&mapping.Field{Type: "long", Index: &falseVar}), expected: common.MapStr{ "type": "long", "index": false, }, }, { - output: p.other(&asset.Field{Type: "text", Index: &trueVar}), + output: p.other(&mapping.Field{Type: "text", Index: &trueVar}), expected: common.MapStr{ "type": "text", "index": true, }, }, { - output: p.other(&asset.Field{Type: "long", DocValues: &falseVar}), + output: p.other(&mapping.Field{Type: "long", DocValues: &falseVar}), expected: common.MapStr{ "type": "long", "doc_values": false, }, }, { - output: p.other(&asset.Field{Type: "double", DocValues: &falseVar}), + output: p.other(&mapping.Field{Type: "double", DocValues: &falseVar}), expected: common.MapStr{ "type": "double", "doc_values": false, }, }, { - output: p.other(&asset.Field{Type: "text", DocValues: &trueVar}), + output: p.other(&mapping.Field{Type: "text", DocValues: &trueVar}), expected: common.MapStr{ "type": "text", "doc_values": true, }, }, { - output: p.alias(&asset.Field{Type: "alias", AliasPath: "a.c", MigrationAlias: false}), + output: p.alias(&mapping.Field{Type: "alias", AliasPath: "a.c", MigrationAlias: false}), expected: common.MapStr{"path": "a.c", "type": "alias"}, }, { - output: p.alias(&asset.Field{Type: "alias", AliasPath: "a.d", MigrationAlias: true}), + output: p.alias(&mapping.Field{Type: "alias", AliasPath: "a.d", MigrationAlias: true}), expected: nil, }, { - output: migrationP.alias(&asset.Field{Type: "alias", AliasPath: "a.e", MigrationAlias: false}), + output: migrationP.alias(&mapping.Field{Type: "alias", AliasPath: "a.e", MigrationAlias: false}), expected: common.MapStr{"path": "a.e", "type": "alias"}, }, { - output: migrationP.alias(&asset.Field{Type: "alias", AliasPath: "a.f", MigrationAlias: true}), + output: migrationP.alias(&mapping.Field{Type: "alias", AliasPath: "a.f", MigrationAlias: true}), expected: common.MapStr{"path": "a.f", "type": "alias"}, }, } @@ -310,11 +310,11 @@ func TestProcessor(t *testing.T) { func TestDynamicTemplates(t *testing.T) { p := &Processor{} tests := []struct { - field asset.Field + field mapping.Field expected []common.MapStr }{ { - field: asset.Field{ + field: mapping.Field{ Type: "object", ObjectType: "keyword", Name: "context", }, @@ -329,7 +329,7 @@ func TestDynamicTemplates(t *testing.T) { }, }, { - field: asset.Field{ + field: mapping.Field{ Type: "object", ObjectType: "long", ObjectTypeMappingType: "futuretype", Path: "language", Name: "english", }, @@ -344,7 +344,7 @@ func TestDynamicTemplates(t *testing.T) { }, }, { - field: asset.Field{ + field: mapping.Field{ Type: "object", ObjectType: "long", ObjectTypeMappingType: "*", Path: "language", Name: "english", }, @@ -359,7 +359,7 @@ func TestDynamicTemplates(t *testing.T) { }, }, { - field: asset.Field{ + field: mapping.Field{ Type: "object", ObjectType: "long", Path: "language", Name: "english", }, @@ -374,7 +374,7 @@ func TestDynamicTemplates(t *testing.T) { }, }, { - field: asset.Field{ + field: mapping.Field{ Type: "object", ObjectType: "text", Path: "language", Name: "english", }, @@ -389,7 +389,7 @@ func TestDynamicTemplates(t *testing.T) { }, }, { - field: asset.Field{ + field: mapping.Field{ Type: "object", ObjectType: "scaled_float", Name: "core.*.pct", }, @@ -407,7 +407,7 @@ func TestDynamicTemplates(t *testing.T) { }, }, { - field: asset.Field{ + field: mapping.Field{ Type: "object", ObjectType: "scaled_float", Name: "core.*.pct", ScalingFactor: 100, ObjectTypeMappingType: "float", }, @@ -425,8 +425,8 @@ func TestDynamicTemplates(t *testing.T) { }, }, { - field: asset.Field{ - Type: "object", ObjectTypeParams: []asset.ObjectTypeCfg{ + field: mapping.Field{ + Type: "object", ObjectTypeParams: []mapping.ObjectTypeCfg{ {ObjectType: "float", ObjectTypeMappingType: "float"}, {ObjectType: "boolean"}, {ObjectType: "scaled_float", ScalingFactor: 10000}, @@ -461,10 +461,10 @@ func TestDynamicTemplates(t *testing.T) { for _, numericType := range []string{"byte", "double", "float", "long", "short", "boolean"} { gen := struct { - field asset.Field + field mapping.Field expected []common.MapStr }{ - field: asset.Field{ + field: mapping.Field{ Type: "object", ObjectType: numericType, Name: "somefield", ObjectTypeMappingType: "long", }, @@ -492,22 +492,22 @@ func TestDynamicTemplates(t *testing.T) { func TestPropertiesCombine(t *testing.T) { // Test common fields are combined even if they come from different objects - fields := asset.Fields{ - asset.Field{ + fields := mapping.Fields{ + mapping.Field{ Name: "test", Type: "group", - Fields: asset.Fields{ - asset.Field{ + Fields: mapping.Fields{ + mapping.Field{ Name: "one", Type: "text", }, }, }, - asset.Field{ + mapping.Field{ Name: "test", Type: "group", - Fields: asset.Fields{ - asset.Field{ + Fields: mapping.Fields{ + mapping.Field{ Name: "two", Type: "text", }, @@ -542,20 +542,20 @@ func TestPropertiesCombine(t *testing.T) { func TestProcessNoName(t *testing.T) { // Test common fields are combined even if they come from different objects - fields := asset.Fields{ - asset.Field{ - Fields: asset.Fields{ - asset.Field{ + fields := mapping.Fields{ + mapping.Field{ + Fields: mapping.Fields{ + mapping.Field{ Name: "one", Type: "text", }, }, }, - asset.Field{ + mapping.Field{ Name: "test", Type: "group", - Fields: asset.Fields{ - asset.Field{ + Fields: mapping.Fields{ + mapping.Field{ Name: "two", Type: "text", }, diff --git a/libbeat/template/template.go b/libbeat/template/template.go index 2ad48be2249..d0b7d7d3d12 100644 --- a/libbeat/template/template.go +++ b/libbeat/template/template.go @@ -22,11 +22,11 @@ import ( "sync" "time" - "github.com/elastic/beats/libbeat/asset" "github.com/elastic/beats/libbeat/beat" "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/common/cfgwarn" "github.com/elastic/beats/libbeat/common/fmtstr" + "github.com/elastic/beats/libbeat/mapping" "github.com/elastic/go-ucfg/yaml" ) @@ -130,7 +130,7 @@ func New( }, nil } -func (t *Template) load(fields asset.Fields) (common.MapStr, error) { +func (t *Template) load(fields mapping.Fields) (common.MapStr, error) { // Locking to make sure dynamicTemplates and defaultFields is not accessed in parallel t.Lock() @@ -142,7 +142,7 @@ func (t *Template) load(fields asset.Fields) (common.MapStr, error) { var err error if len(t.config.AppendFields) > 0 { cfgwarn.Experimental("append_fields is used.") - fields, err = asset.ConcatFields(fields, t.config.AppendFields) + fields, err = mapping.ConcatFields(fields, t.config.AppendFields) if err != nil { return nil, err } @@ -161,7 +161,7 @@ func (t *Template) load(fields asset.Fields) (common.MapStr, error) { // LoadFile loads the the template from the given file path func (t *Template) LoadFile(file string) (common.MapStr, error) { - fields, err := asset.LoadFieldsYaml(file) + fields, err := mapping.LoadFieldsYaml(file) if err != nil { return nil, err } @@ -312,19 +312,19 @@ func buildIdxSettings(ver common.Version, userSettings common.MapStr) common.Map return indexSettings } -func loadYamlByte(data []byte) (asset.Fields, error) { +func loadYamlByte(data []byte) (mapping.Fields, error) { cfg, err := yaml.NewConfig(data) if err != nil { return nil, err } - var keys []asset.Field + var keys []mapping.Field err = cfg.Unpack(&keys) if err != nil { return nil, err } - fields := asset.Fields{} + fields := mapping.Fields{} for _, key := range keys { fields = append(fields, key.Fields...) } From 117160d1f6f76899f85e46eadc6d19c5307f7da7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Wed, 13 Mar 2019 10:30:49 +0100 Subject: [PATCH 5/5] add changelog entry to dev log --- CHANGELOG-developer.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG-developer.next.asciidoc b/CHANGELOG-developer.next.asciidoc index e8ab8e7f82d..c3e92e3558a 100644 --- a/CHANGELOG-developer.next.asciidoc +++ b/CHANGELOG-developer.next.asciidoc @@ -22,6 +22,7 @@ The list below covers the major changes between 7.0.0-beta1 and master only. - Remove support for deprecated `GenRootCmd` methods. {pull}10721[10721] - Remove SkipNormalization, SkipAgentMetadata, SkipAddHostName. {pull}10801[10801] {pull}10769[10769] +- Move Fields from package libbeat/common to libbeat/mapping. {pull}11198[11198] ==== Bugfixes