From 8c79674e9739b3d7d371d28440782e539d00da2b Mon Sep 17 00:00:00 2001 From: simitt Date: Wed, 8 May 2019 12:42:59 +0200 Subject: [PATCH 1/4] Add minimal ES template functionality. --- libbeat/template/load.go | 12 + libbeat/template/load_integration_test.go | 317 ++++++++++++------- libbeat/template/load_test.go | 69 ++-- libbeat/template/template.go | 19 ++ libbeat/template/testdata/default_fields.yml | 7 + libbeat/template/testdata/fields.json | 16 + 6 files changed, 303 insertions(+), 137 deletions(-) create mode 100644 libbeat/template/testdata/default_fields.yml create mode 100644 libbeat/template/testdata/fields.json diff --git a/libbeat/template/load.go b/libbeat/template/load.go index d9fe54897a6..89cd29a6f9d 100644 --- a/libbeat/template/load.go +++ b/libbeat/template/load.go @@ -176,6 +176,9 @@ func buildBody(tmpl *Template, config TemplateConfig, fields []byte) (common.Map if config.Fields != "" { return buildBodyFromFile(tmpl, config) } + if fields == nil { + return buildMinimalTemplate(tmpl) + } return buildBodyFromFields(tmpl, fields) } @@ -217,6 +220,15 @@ func buildBodyFromFields(tmpl *Template, fields []byte) (common.MapStr, error) { return body, nil } +func buildMinimalTemplate(tmpl *Template) (common.MapStr, error) { + logp.Debug("template", "Load minimal template") + body, err := tmpl.LoadMinimal() + if err != nil { + return nil, fmt.Errorf("error creating mimimal template: %v", err) + } + return body, nil +} + func esVersionParams(ver common.Version) map[string]string { if ver.Major == 6 && ver.Minor == 7 { return map[string]string{ diff --git a/libbeat/template/load_integration_test.go b/libbeat/template/load_integration_test.go index e33f8aae294..a0a67d33c2a 100644 --- a/libbeat/template/load_integration_test.go +++ b/libbeat/template/load_integration_test.go @@ -22,6 +22,7 @@ package template import ( "encoding/json" "fmt" + "io/ioutil" "path/filepath" "strconv" "testing" @@ -51,7 +52,7 @@ var ( templateName = "testbeatidx-" + version.GetDefaultVersion() ) -func defaultESLoader(t *testing.T) *ESLoader { +func testEsLoader(t *testing.T) *ESLoader { client := estest.GetTestingElasticsearch(t) if err := client.Connect(); err != nil { t.Fatal(err) @@ -60,43 +61,216 @@ func defaultESLoader(t *testing.T) *ESLoader { return NewESLoader(client) } -func TestCheckTemplate(t *testing.T) { - loader := defaultESLoader(t) +func testFields(t *testing.T, name string) []byte { + file, err := filepath.Abs(filepath.Join("testdata", "default_fields.yml")) + require.NoError(t, err) + fields, err := ioutil.ReadFile(file) + require.NoError(t, err) + return fields +} + +func TestESLoader_Load(t *testing.T) { + // Setup ES + loader := testEsLoader(t) + client := loader.client + + name := "testbeat" + fields := testFields(t, name) + + pathTestdata, err := filepath.Abs(filepath.Join("testdata")) + require.NoError(t, err) + fieldsYamlPath := filepath.Join(pathTestdata, "fields.yml") + fieldsJsonPath := filepath.Join(pathTestdata, "fields.json") + + t.Run("load template failure", func(t *testing.T) { + for run, data := range map[string]struct { + cfg TemplateConfig + info beat.Info + fields []byte + err string + }{ + "loading disabled": { + cfg: TemplateConfig{Enabled: false, Name: name}, + info: beatInfo, + }, + "invalid version": { + cfg: TemplateConfig{Enabled: true, Name: name}, + info: beat.Info{Version: "invalid", Beat: "testbeat", IndexPrefix: "testbeatidx"}, + err: "version", + }, + } { + t.Run(run, func(t *testing.T) { + // Ensure template is not loaded + client.Request("DELETE", "/_template/"+name, "", nil, nil) + assert.False(t, loader.templateExists(name)) + + err := loader.Load(data.cfg, data.info, data.fields, false) + if data.err != "" { + if assert.Error(t, err) { + assert.Contains(t, err.Error(), data.err) + } + } else { + assert.NoError(t, err) + } + + // Make sure template was not loaded + assert.False(t, loader.templateExists(name)) + }) + } + }) + + t.Run("template already exists", func(t *testing.T) { + cfg := TemplateConfig{Enabled: true, Name: name} + + // Ensure template is not loaded + client.Request("DELETE", "/_template/"+name, "", nil, nil) + assert.False(t, loader.templateExists(name)) - // Check for non existent template - assert.False(t, loader.templateExists("libbeat-notexists")) + // Load Template and check it exists + err := loader.Load(cfg, beatInfo, nil, false) + assert.NoError(t, err) + assert.True(t, loader.templateExists(name)) + // check that source is enabled + tmpl := getTemplate(t, client, name) + assert.Equal(t, true, tmpl.SourceEnabled()) + + // Load template again, this time with custom settings + cfg.Settings = TemplateSettings{ + Source: map[string]interface{}{ + "enabled": false, + }, + } + // Do not overwrite template as overwrite is not set to true + err = loader.Load(cfg, beatInfo, nil, false) + assert.NoError(t, err) + // check that template was not overwritten, by checking source is enabled + tmpl = getTemplate(t, client, name) + assert.Equal(t, true, tmpl.SourceEnabled()) + + // Do overwrite template + cfg.Overwrite = true + err = loader.Load(cfg, beatInfo, nil, false) + assert.NoError(t, err) + // check that template was not overwritten, by checking source is enabled + tmpl = getTemplate(t, client, name) + assert.Equal(t, false, tmpl.SourceEnabled()) + }) + + t.Run("template exists for `name` but not for `json.name`", func(t *testing.T) { + // Load Template with name `name` + cfg := TemplateConfig{Enabled: true, Name: name} + err := loader.Load(cfg, beatInfo, nil, false) + assert.NoError(t, err) + assert.True(t, loader.templateExists(name)) + + // Load Template with same name, but different JSON.name and ensure it is loaded + nameJson := "json-test" + cfgJson := TemplateConfig{Enabled: true, Name: name, JSON: struct { + Enabled bool `config:"enabled"` + Path string `config:"path"` + Name string `config:"name"` + }{Enabled: true, Path: fieldsJsonPath, Name: nameJson}} + err = loader.Load(cfgJson, beatInfo, nil, false) + assert.NoError(t, err) + assert.True(t, loader.templateExists(name)) + assert.True(t, loader.templateExists(nameJson)) + }) + + t.Run("load template successful", func(t *testing.T) { + + for run, data := range map[string]struct { + cfg TemplateConfig + fields []byte + name string + properties []string + }{ + "default config with fields": { + cfg: TemplateConfig{Enabled: true, Name: name}, + fields: fields, + name: name, + properties: []string{"foo", "bar"}, + }, + "minimal template": { + cfg: TemplateConfig{Enabled: true, Name: name}, + fields: nil, + name: name, + }, + "fields from file": { + cfg: TemplateConfig{Enabled: true, Name: name, Fields: fieldsYamlPath}, + fields: fields, + name: name, + properties: []string{"object", "keyword", "alias", "migration_alias_false", "object_disabled"}, + }, + "fields from json": { + cfg: TemplateConfig{Enabled: true, Name: name, JSON: struct { + Enabled bool `config:"enabled"` + Path string `config:"path"` + Name string `config:"name"` + }{Enabled: true, Path: fieldsJsonPath, Name: "json-template"}}, + fields: fields, + name: "json-template", + properties: []string{"host_name"}, + }, + } { + t.Run(run, func(t *testing.T) { + // Ensure template is not loaded + client.Request("DELETE", "/_template/"+data.name, "", nil, nil) + assert.False(t, loader.templateExists(data.name)) + + // Load Template according to config and given fields + err := loader.Load(data.cfg, beatInfo, data.fields, false) + assert.NoError(t, err) + + // Make sure template was loaded + assert.True(t, loader.templateExists(data.name)) + + // Fetch properties + tmpl := getTemplate(t, client, data.name) + val, err := tmpl.GetValue("mappings.properties") + if data.properties != nil { + require.NoError(t, err) + p, ok := val.(map[string]interface{}) + require.True(t, ok) + var properties []string + for k, _ := range p { + properties = append(properties, k) + } + fmt.Println(properties) + assert.ElementsMatch(t, properties, data.properties) + } else { + assert.Error(t, err) + } + }) + } + }) } -func TestLoadTemplate(t *testing.T) { +func TestTemplate_LoadFile(t *testing.T) { // Setup ES - loader := defaultESLoader(t) + loader := testEsLoader(t) client := loader.client - // Load template absPath, err := filepath.Abs("../") assert.NotNil(t, absPath) assert.Nil(t, err) - fieldsPath := absPath + "/fields.yml" index := "testbeat" tmpl, err := New(version.GetDefaultVersion(), index, client.GetVersion(), TemplateConfig{}, false) require.NoError(t, err) - content, err := tmpl.LoadFile(fieldsPath) - require.NoError(t, err) + + // Ensure clean state + client.Request("DELETE", "/_template/"+tmpl.GetName(), "", nil, nil) + assert.False(t, loader.templateExists(tmpl.GetName())) // Load template + content, err := tmpl.LoadFile(fieldsPath) + require.NoError(t, err) err = loader.loadTemplate(tmpl.GetName(), content) require.NoError(t, err) // Make sure template was loaded assert.True(t, loader.templateExists(tmpl.GetName())) - - // Delete template again to clean up - client.Request("DELETE", "/_template/"+tmpl.GetName(), "", nil, nil) - - // Make sure it was removed - assert.False(t, loader.templateExists(tmpl.GetName())) } func TestLoadInvalidTemplate(t *testing.T) { @@ -104,11 +278,10 @@ func TestLoadInvalidTemplate(t *testing.T) { template := map[string]interface{}{ "json": "invalid", } + templateName := "invalidtemplate" // Setup ES - loader := defaultESLoader(t) - - templateName := "invalidtemplate" + loader := testEsLoader(t) // Try to load invalid template err := loader.loadTemplate(templateName, template) @@ -119,7 +292,7 @@ func TestLoadInvalidTemplate(t *testing.T) { } // Tests loading the templates for each beat -func TestLoadBeatsTemplate(t *testing.T) { +func TestLoadBeatsTemplate_fromFile(t *testing.T) { beats := []string{ "libbeat", } @@ -131,7 +304,7 @@ func TestLoadBeatsTemplate(t *testing.T) { assert.Nil(t, err) // Setup ES - loader := defaultESLoader(t) + loader := testEsLoader(t) client := loader.client fieldsPath := absPath + "/fields.yml" @@ -142,24 +315,22 @@ func TestLoadBeatsTemplate(t *testing.T) { content, err := tmpl.LoadFile(fieldsPath) assert.NoError(t, err) + // Ensure clean state + client.Request("DELETE", "/_template/"+tmpl.GetName(), "", nil, nil) + assert.False(t, loader.templateExists(tmpl.GetName())) + // Load template err = loader.loadTemplate(tmpl.GetName(), content) assert.Nil(t, err) // Make sure template was loaded assert.True(t, loader.templateExists(tmpl.GetName())) - - // Delete template again to clean up - client.Request("DELETE", "/_template/"+tmpl.GetName(), "", nil, nil) - - // Make sure it was removed - assert.False(t, loader.templateExists(tmpl.GetName())) } } func TestTemplateSettings(t *testing.T) { // Setup ES - loader := defaultESLoader(t) + loader := testEsLoader(t) client := loader.client // Load template @@ -185,6 +356,10 @@ func TestTemplateSettings(t *testing.T) { content, err := tmpl.LoadFile(fieldsPath) assert.NoError(t, err) + // Ensure clean state + client.Request("DELETE", "/_template/"+tmpl.GetName(), "", nil, nil) + assert.False(t, loader.templateExists(tmpl.GetName())) + // Load template err = loader.loadTemplate(tmpl.GetName(), content) assert.Nil(t, err) @@ -193,74 +368,6 @@ func TestTemplateSettings(t *testing.T) { templateJSON := getTemplate(t, client, tmpl.GetName()) assert.Equal(t, 1, templateJSON.NumberOfShards()) assert.Equal(t, false, templateJSON.SourceEnabled()) - - // Delete template again to clean up - client.Request("DELETE", "/_template/"+tmpl.GetName(), "", nil, nil) - - // Make sure it was removed - assert.False(t, loader.templateExists(tmpl.GetName())) -} - -func TestOverwrite(t *testing.T) { - // Setup ES - loader := defaultESLoader(t) - client := loader.client - - templateName := "testbeatidx-" + version.GetDefaultVersion() - - absPath, err := filepath.Abs("../") - assert.NotNil(t, absPath) - assert.Nil(t, err) - - // make sure no template is already there - client.Request("DELETE", "/_template/"+templateName, "", nil, nil) - - // Load template - config := TemplateConfig{ - Enabled: true, - Fields: absPath + "/fields.yml", - } - err = loader.Load(config, beatInfo, nil, false) - assert.NoError(t, err) - - // Load template again, this time with custom settings - config = TemplateConfig{ - Enabled: true, - Fields: absPath + "/fields.yml", - Settings: TemplateSettings{ - Source: map[string]interface{}{ - "enabled": false, - }, - }, - } - - err = loader.Load(config, beatInfo, nil, false) - assert.NoError(t, err) - - // Overwrite was not enabled, so the first version should still be there - templateJSON := getTemplate(t, client, templateName) - assert.Equal(t, true, templateJSON.SourceEnabled()) - - // Load template again, this time with custom settings AND overwrite: true - config = TemplateConfig{ - Enabled: true, - Overwrite: true, - Fields: absPath + "/fields.yml", - Settings: TemplateSettings{ - Source: map[string]interface{}{ - "enabled": false, - }, - }, - } - err = loader.Load(config, beatInfo, nil, false) - assert.NoError(t, err) - - // Overwrite was enabled, so the custom setting should be there - templateJSON = getTemplate(t, client, templateName) - assert.Equal(t, false, templateJSON.SourceEnabled()) - - // Delete template again to clean up - client.Request("DELETE", "/_template/"+templateName, "", nil, nil) } var dataTests = []struct { @@ -323,6 +430,10 @@ func TestTemplateWithData(t *testing.T) { content, err := tmpl.LoadFile(fieldsPath) assert.NoError(t, err) + // Ensure clean state + client.Request("DELETE", "/_template/"+tmpl.GetName(), "", nil, nil) + assert.False(t, loader.templateExists(tmpl.GetName())) + // Load template err = loader.loadTemplate(tmpl.GetName(), content) assert.Nil(t, err) @@ -339,22 +450,16 @@ func TestTemplateWithData(t *testing.T) { assert.Nil(t, err) } } - - // Delete template again to clean up - client.Request("DELETE", "/_template/"+tmpl.GetName(), "", nil, nil) - - // Make sure it was removed - assert.False(t, loader.templateExists(tmpl.GetName())) } func getTemplate(t *testing.T, client ESClient, templateName string) testTemplate { status, body, err := client.Request("GET", "/_template/"+templateName, "", nil, nil) - assert.NoError(t, err) - assert.Equal(t, status, 200) + require.NoError(t, err) + require.Equal(t, status, 200) var response common.MapStr err = json.Unmarshal(body, &response) - assert.NoError(t, err) + require.NoError(t, err) return testTemplate{ t: t, diff --git a/libbeat/template/load_test.go b/libbeat/template/load_test.go index 22566229ab9..e3e34e9b05e 100644 --- a/libbeat/template/load_test.go +++ b/libbeat/template/load_test.go @@ -23,7 +23,6 @@ import ( "github.com/elastic/beats/libbeat/beat" "github.com/elastic/beats/libbeat/common" - "github.com/elastic/beats/libbeat/version" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -33,62 +32,70 @@ func TestFileLoader_Load(t *testing.T) { ver := "7.0.0" prefix := "mock" info := beat.Info{Version: ver, IndexPrefix: prefix} + tmplName := fmt.Sprintf("%s-%s", prefix, ver) for name, test := range map[string]struct { - cfg TemplateConfig - fields []byte - migration bool - - name string + settings TemplateSettings + body common.MapStr }{ - "default config": { - cfg: DefaultConfig(), - name: fmt.Sprintf("%s-%s", prefix, ver), - migration: false, + "load minimal config info": { + body: common.MapStr{ + "index_patterns": []string{"mock-7.0.0-*"}, + "order": 0, + "settings": common.MapStr{"index": nil}}, + }, + "load minimal config with index settings": { + settings: TemplateSettings{Index: common.MapStr{"code": "best_compression"}}, + body: common.MapStr{ + "index_patterns": []string{"mock-7.0.0-*"}, + "order": 0, + "settings": common.MapStr{"index": common.MapStr{"code": "best_compression"}}}, }, - "default config with migration": { - cfg: DefaultConfig(), - name: fmt.Sprintf("%s-%s", prefix, ver), - migration: true, + "load minimal config with source settings": { + settings: TemplateSettings{Source: common.MapStr{"enabled": false}}, + body: common.MapStr{ + "index_patterns": []string{"mock-7.0.0-*"}, + "order": 0, + "settings": common.MapStr{"index": nil}, + "mappings": common.MapStr{ + "_source": common.MapStr{"enabled": false}, + "_meta": common.MapStr{"beat": prefix, "version": ver}, + "date_detection": false, + "dynamic_templates": nil, + "properties": nil, + }}, }, } { t.Run(name, func(t *testing.T) { fc, err := newFileClient(ver) require.NoError(t, err) fl := NewFileLoader(fc) - err = fl.Load(test.cfg, info, test.fields, false) - require.NoError(t, err) - tmpl, err := New(ver, prefix, *common.MustNewVersion(ver), test.cfg, test.migration) - require.NoError(t, err) - body, err := buildBody(tmpl, test.cfg, test.fields) + cfg := DefaultConfig() + cfg.Settings = test.settings + + err = fl.Load(cfg, info, nil, false) require.NoError(t, err) - assert.Equal(t, common.MapStr{test.name: body}.StringToPrint()+"\n", fc.body) + assert.Equal(t, common.MapStr{tmplName: test.body}.StringToPrint()+"\n", fc.body) + assert.Equal(t, tmplName, fc.name) }) } } type fileClient struct { - ver common.Version - body string + name, body, ver string } func newFileClient(ver string) (*fileClient, error) { - if ver == "" { - ver = version.GetDefaultVersion() - } - v, err := common.NewVersion(ver) - if err != nil { - return nil, err - } - return &fileClient{ver: *v}, nil + return &fileClient{ver: ver}, nil } func (c *fileClient) GetVersion() common.Version { - return c.ver + return *common.MustNewVersion(c.ver) } func (c *fileClient) Write(name string, body string) error { c.body = body + c.name = name return nil } diff --git a/libbeat/template/template.go b/libbeat/template/template.go index 092280cdd6a..ecdc38d0d5f 100644 --- a/libbeat/template/template.go +++ b/libbeat/template/template.go @@ -183,6 +183,25 @@ func (t *Template) LoadBytes(data []byte) (common.MapStr, error) { return t.load(fields) } +// LoadMinimal loads the template only with the given configuration +func (t *Template) LoadMinimal() (common.MapStr, error) { + keyPattern, patterns := buildPatternSettings(t.esVersion, t.GetPattern()) + m := common.MapStr{ + keyPattern: patterns, + "order": t.order, + "settings": common.MapStr{ + "index": t.config.Settings.Index, + }, + } + if t.config.Settings.Source != nil { + m["mappings"] = buildMappings( + t.beatVersion, t.esVersion, t.beatName, + nil, nil, + common.MapStr(t.config.Settings.Source)) + } + return m, nil +} + // GetName returns the name of the template func (t *Template) GetName() string { return t.name diff --git a/libbeat/template/testdata/default_fields.yml b/libbeat/template/testdata/default_fields.yml new file mode 100644 index 00000000000..370f3199340 --- /dev/null +++ b/libbeat/template/testdata/default_fields.yml @@ -0,0 +1,7 @@ +- key: test + title: Test default fieldds + fields: + - name: foo + type: keyword + - name: bar + type: keyword diff --git a/libbeat/template/testdata/fields.json b/libbeat/template/testdata/fields.json new file mode 100644 index 00000000000..d95b7a7dabe --- /dev/null +++ b/libbeat/template/testdata/fields.json @@ -0,0 +1,16 @@ +{ + "index_patterns": ["foo"], + "settings": { + "number_of_shards": 1 + }, + "mappings": { + "_source": { + "enabled": false + }, + "properties": { + "host_name": { + "type": "keyword" + } + } + } +} From cbbdea57f07243270e7cdce1043de1ded8240844 Mon Sep 17 00:00:00 2001 From: simitt Date: Wed, 8 May 2019 20:59:12 +0200 Subject: [PATCH 2/4] make hound happy --- libbeat/template/load_integration_test.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/libbeat/template/load_integration_test.go b/libbeat/template/load_integration_test.go index a0a67d33c2a..e77d6c549e3 100644 --- a/libbeat/template/load_integration_test.go +++ b/libbeat/template/load_integration_test.go @@ -80,7 +80,7 @@ func TestESLoader_Load(t *testing.T) { pathTestdata, err := filepath.Abs(filepath.Join("testdata")) require.NoError(t, err) fieldsYamlPath := filepath.Join(pathTestdata, "fields.yml") - fieldsJsonPath := filepath.Join(pathTestdata, "fields.json") + fieldsJSONPath := filepath.Join(pathTestdata, "fields.json") t.Run("load template failure", func(t *testing.T) { for run, data := range map[string]struct { @@ -164,16 +164,16 @@ func TestESLoader_Load(t *testing.T) { assert.True(t, loader.templateExists(name)) // Load Template with same name, but different JSON.name and ensure it is loaded - nameJson := "json-test" - cfgJson := TemplateConfig{Enabled: true, Name: name, JSON: struct { + nameJSON := "json-test" + cfgJSON := TemplateConfig{Enabled: true, Name: name, JSON: struct { Enabled bool `config:"enabled"` Path string `config:"path"` Name string `config:"name"` - }{Enabled: true, Path: fieldsJsonPath, Name: nameJson}} - err = loader.Load(cfgJson, beatInfo, nil, false) + }{Enabled: true, Path: fieldsJSONPath, Name: nameJSON}} + err = loader.Load(cfgJSON, beatInfo, nil, false) assert.NoError(t, err) assert.True(t, loader.templateExists(name)) - assert.True(t, loader.templateExists(nameJson)) + assert.True(t, loader.templateExists(nameJSON)) }) t.Run("load template successful", func(t *testing.T) { @@ -206,7 +206,7 @@ func TestESLoader_Load(t *testing.T) { Enabled bool `config:"enabled"` Path string `config:"path"` Name string `config:"name"` - }{Enabled: true, Path: fieldsJsonPath, Name: "json-template"}}, + }{Enabled: true, Path: fieldsJSONPath, Name: "json-template"}}, fields: fields, name: "json-template", properties: []string{"host_name"}, @@ -232,7 +232,7 @@ func TestESLoader_Load(t *testing.T) { p, ok := val.(map[string]interface{}) require.True(t, ok) var properties []string - for k, _ := range p { + for k := range p { properties = append(properties, k) } fmt.Println(properties) From 839809083ae5e3188c01afb217b0db8da7bb243e Mon Sep 17 00:00:00 2001 From: simitt Date: Wed, 8 May 2019 23:50:19 +0200 Subject: [PATCH 3/4] rework tests --- libbeat/template/load_integration_test.go | 384 +++++++--------------- 1 file changed, 120 insertions(+), 264 deletions(-) diff --git a/libbeat/template/load_integration_test.go b/libbeat/template/load_integration_test.go index e77d6c549e3..5c292c71a47 100644 --- a/libbeat/template/load_integration_test.go +++ b/libbeat/template/load_integration_test.go @@ -23,211 +23,169 @@ import ( "encoding/json" "fmt" "io/ioutil" + "math/rand" "path/filepath" "strconv" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/elastic/beats/libbeat/beat" "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/outputs/elasticsearch" "github.com/elastic/beats/libbeat/outputs/elasticsearch/estest" "github.com/elastic/beats/libbeat/version" ) +func init() { + rand.Seed(time.Now().UnixNano()) +} + type testTemplate struct { t *testing.T client ESClient common.MapStr } -var ( - beatInfo = beat.Info{ - Beat: "testbeat", - IndexPrefix: "testbeatidx", - Version: version.GetDefaultVersion(), - } - - templateName = "testbeatidx-" + version.GetDefaultVersion() -) +type testSetup struct { + t *testing.T + client ESClient + loader *ESLoader + config TemplateConfig +} -func testEsLoader(t *testing.T) *ESLoader { +func newTestSetup(t *testing.T, cfg TemplateConfig) *testSetup { + if cfg.Name == "" { + cfg.Name = fmt.Sprintf("load-test-%+v", rand.Int()) + } client := estest.GetTestingElasticsearch(t) if err := client.Connect(); err != nil { t.Fatal(err) } + s := testSetup{t: t, client: client, loader: NewESLoader(client), config: cfg} + client.Request("DELETE", "/_template/"+cfg.Name, "", nil, nil) + require.False(t, s.loader.templateExists(cfg.Name)) + return &s +} +func (ts *testSetup) loadFromFile(fileElems []string) error { + ts.config.Fields = path(ts.t, fileElems) + beatInfo := beat.Info{Version: version.GetDefaultVersion()} + return ts.loader.Load(ts.config, beatInfo, nil, false) +} - return NewESLoader(client) +func (ts *testSetup) load(fields []byte) error { + beatInfo := beat.Info{Version: version.GetDefaultVersion()} + return ts.loader.Load(ts.config, beatInfo, fields, false) } -func testFields(t *testing.T, name string) []byte { - file, err := filepath.Abs(filepath.Join("testdata", "default_fields.yml")) - require.NoError(t, err) - fields, err := ioutil.ReadFile(file) - require.NoError(t, err) - return fields +func (ts *testSetup) mustLoad(fields []byte) { + require.NoError(ts.t, ts.load(fields)) + require.True(ts.t, ts.loader.templateExists(ts.config.Name)) } func TestESLoader_Load(t *testing.T) { - // Setup ES - loader := testEsLoader(t) - client := loader.client - - name := "testbeat" - fields := testFields(t, name) - - pathTestdata, err := filepath.Abs(filepath.Join("testdata")) - require.NoError(t, err) - fieldsYamlPath := filepath.Join(pathTestdata, "fields.yml") - fieldsJSONPath := filepath.Join(pathTestdata, "fields.json") - - t.Run("load template failure", func(t *testing.T) { - for run, data := range map[string]struct { - cfg TemplateConfig - info beat.Info - fields []byte - err string - }{ - "loading disabled": { - cfg: TemplateConfig{Enabled: false, Name: name}, - info: beatInfo, - }, - "invalid version": { - cfg: TemplateConfig{Enabled: true, Name: name}, - info: beat.Info{Version: "invalid", Beat: "testbeat", IndexPrefix: "testbeatidx"}, - err: "version", - }, - } { - t.Run(run, func(t *testing.T) { - // Ensure template is not loaded - client.Request("DELETE", "/_template/"+name, "", nil, nil) - assert.False(t, loader.templateExists(name)) - - err := loader.Load(data.cfg, data.info, data.fields, false) - if data.err != "" { - if assert.Error(t, err) { - assert.Contains(t, err.Error(), data.err) - } - } else { - assert.NoError(t, err) - } - - // Make sure template was not loaded - assert.False(t, loader.templateExists(name)) - }) - } + t.Run("failure", func(t *testing.T) { + t.Run("loading disabled", func(t *testing.T) { + setup := newTestSetup(t, TemplateConfig{Enabled: false}) + + setup.load(nil) + assert.False(t, setup.loader.templateExists(setup.config.Name)) + }) + + t.Run("invalid version", func(t *testing.T) { + setup := newTestSetup(t, TemplateConfig{Enabled: true}) + + beatInfo := beat.Info{Version: "invalid"} + err := setup.loader.Load(setup.config, beatInfo, nil, false) + if assert.Error(t, err) { + assert.Contains(t, err.Error(), "version is not semver") + } + }) }) - t.Run("template already exists", func(t *testing.T) { - cfg := TemplateConfig{Enabled: true, Name: name} - - // Ensure template is not loaded - client.Request("DELETE", "/_template/"+name, "", nil, nil) - assert.False(t, loader.templateExists(name)) - - // Load Template and check it exists - err := loader.Load(cfg, beatInfo, nil, false) - assert.NoError(t, err) - assert.True(t, loader.templateExists(name)) - // check that source is enabled - tmpl := getTemplate(t, client, name) - assert.Equal(t, true, tmpl.SourceEnabled()) - - // Load template again, this time with custom settings - cfg.Settings = TemplateSettings{ - Source: map[string]interface{}{ - "enabled": false, - }, - } - // Do not overwrite template as overwrite is not set to true - err = loader.Load(cfg, beatInfo, nil, false) - assert.NoError(t, err) - // check that template was not overwritten, by checking source is enabled - tmpl = getTemplate(t, client, name) - assert.Equal(t, true, tmpl.SourceEnabled()) - - // Do overwrite template - cfg.Overwrite = true - err = loader.Load(cfg, beatInfo, nil, false) - assert.NoError(t, err) - // check that template was not overwritten, by checking source is enabled - tmpl = getTemplate(t, client, name) - assert.Equal(t, false, tmpl.SourceEnabled()) + t.Run("overwrite", func(t *testing.T) { + // Setup create template with source enabled + setup := newTestSetup(t, TemplateConfig{Enabled: true}) + setup.mustLoad(nil) + + // Add custom settings + setup.config.Settings = TemplateSettings{Source: map[string]interface{}{"enabled": false}} + + t.Run("disabled", func(t *testing.T) { + setup.load(nil) + tmpl := getTemplate(t, setup.client, setup.config.Name) + assert.Equal(t, true, tmpl.SourceEnabled()) + }) + + t.Run("enabled", func(t *testing.T) { + setup.config.Overwrite = true + setup.load(nil) + tmpl := getTemplate(t, setup.client, setup.config.Name) + assert.Equal(t, false, tmpl.SourceEnabled()) + }) }) - t.Run("template exists for `name` but not for `json.name`", func(t *testing.T) { - // Load Template with name `name` - cfg := TemplateConfig{Enabled: true, Name: name} - err := loader.Load(cfg, beatInfo, nil, false) - assert.NoError(t, err) - assert.True(t, loader.templateExists(name)) + t.Run("json.name", func(t *testing.T) { + nameJSON := "bar" + + setup := newTestSetup(t, TemplateConfig{Enabled: true}) + setup.mustLoad(nil) - // Load Template with same name, but different JSON.name and ensure it is loaded - nameJSON := "json-test" - cfgJSON := TemplateConfig{Enabled: true, Name: name, JSON: struct { + // Load Template with same name, but different JSON.name and ensure it is used + setup.config.JSON = struct { Enabled bool `config:"enabled"` Path string `config:"path"` Name string `config:"name"` - }{Enabled: true, Path: fieldsJSONPath, Name: nameJSON}} - err = loader.Load(cfgJSON, beatInfo, nil, false) - assert.NoError(t, err) - assert.True(t, loader.templateExists(name)) - assert.True(t, loader.templateExists(nameJSON)) + }{Enabled: true, Path: path(t, []string{"testdata", "fields.json"}), Name: nameJSON} + setup.load(nil) + assert.True(t, setup.loader.templateExists(nameJSON)) }) t.Run("load template successful", func(t *testing.T) { - + fields, err := ioutil.ReadFile(path(t, []string{"testdata", "default_fields.yml"})) + require.NoError(t, err) for run, data := range map[string]struct { cfg TemplateConfig fields []byte - name string + fieldsPath string properties []string }{ "default config with fields": { - cfg: TemplateConfig{Enabled: true, Name: name}, + cfg: TemplateConfig{Enabled: true}, fields: fields, - name: name, properties: []string{"foo", "bar"}, }, "minimal template": { - cfg: TemplateConfig{Enabled: true, Name: name}, + cfg: TemplateConfig{Enabled: true}, fields: nil, - name: name, }, "fields from file": { - cfg: TemplateConfig{Enabled: true, Name: name, Fields: fieldsYamlPath}, + cfg: TemplateConfig{Enabled: true, Fields: path(t, []string{"testdata", "fields.yml"})}, fields: fields, - name: name, properties: []string{"object", "keyword", "alias", "migration_alias_false", "object_disabled"}, }, "fields from json": { - cfg: TemplateConfig{Enabled: true, Name: name, JSON: struct { + cfg: TemplateConfig{Enabled: true, Name: "json-template", JSON: struct { Enabled bool `config:"enabled"` Path string `config:"path"` Name string `config:"name"` - }{Enabled: true, Path: fieldsJSONPath, Name: "json-template"}}, + }{Enabled: true, Path: path(t, []string{"testdata", "fields.json"}), Name: "json-template"}}, fields: fields, - name: "json-template", properties: []string{"host_name"}, }, } { t.Run(run, func(t *testing.T) { - // Ensure template is not loaded - client.Request("DELETE", "/_template/"+data.name, "", nil, nil) - assert.False(t, loader.templateExists(data.name)) - - // Load Template according to config and given fields - err := loader.Load(data.cfg, beatInfo, data.fields, false) - assert.NoError(t, err) - - // Make sure template was loaded - assert.True(t, loader.templateExists(data.name)) + setup := newTestSetup(t, data.cfg) + setup.mustLoad(data.fields) // Fetch properties - tmpl := getTemplate(t, client, data.name) + tmpl := getTemplate(t, setup.client, setup.config.Name) val, err := tmpl.GetValue("mappings.properties") - if data.properties != nil { + if data.properties == nil { + assert.Error(t, err) + } else { require.NoError(t, err) p, ok := val.(map[string]interface{}) require.True(t, ok) @@ -235,10 +193,7 @@ func TestESLoader_Load(t *testing.T) { for k := range p { properties = append(properties, k) } - fmt.Println(properties) assert.ElementsMatch(t, properties, data.properties) - } else { - assert.Error(t, err) } }) } @@ -246,49 +201,19 @@ func TestESLoader_Load(t *testing.T) { } func TestTemplate_LoadFile(t *testing.T) { - // Setup ES - loader := testEsLoader(t) - client := loader.client - - absPath, err := filepath.Abs("../") - assert.NotNil(t, absPath) - assert.Nil(t, err) - fieldsPath := absPath + "/fields.yml" - index := "testbeat" - - tmpl, err := New(version.GetDefaultVersion(), index, client.GetVersion(), TemplateConfig{}, false) - require.NoError(t, err) - - // Ensure clean state - client.Request("DELETE", "/_template/"+tmpl.GetName(), "", nil, nil) - assert.False(t, loader.templateExists(tmpl.GetName())) - - // Load template - content, err := tmpl.LoadFile(fieldsPath) - require.NoError(t, err) - err = loader.loadTemplate(tmpl.GetName(), content) - require.NoError(t, err) - - // Make sure template was loaded - assert.True(t, loader.templateExists(tmpl.GetName())) + setup := newTestSetup(t, TemplateConfig{}) + assert.NoError(t, setup.loadFromFile([]string{"..", "fields.yml"})) + assert.True(t, setup.loader.templateExists(setup.config.Name)) } func TestLoadInvalidTemplate(t *testing.T) { - // Invalid Template - template := map[string]interface{}{ - "json": "invalid", - } - templateName := "invalidtemplate" - - // Setup ES - loader := testEsLoader(t) + setup := newTestSetup(t, TemplateConfig{}) // Try to load invalid template - err := loader.loadTemplate(templateName, template) + template := map[string]interface{}{"json": "invalid"} + err := setup.loader.loadTemplate(setup.config.Name, template) assert.Error(t, err) - - // Make sure template was not loaded - assert.False(t, loader.templateExists(templateName)) + assert.False(t, setup.loader.templateExists(setup.config.Name)) } // Tests loading the templates for each beat @@ -298,74 +223,22 @@ func TestLoadBeatsTemplate_fromFile(t *testing.T) { } for _, beat := range beats { - // Load template - absPath, err := filepath.Abs("../../" + beat) - assert.NotNil(t, absPath) - assert.Nil(t, err) - - // Setup ES - loader := testEsLoader(t) - client := loader.client - - fieldsPath := absPath + "/fields.yml" - index := beat - - tmpl, err := New(version.GetDefaultVersion(), index, client.GetVersion(), TemplateConfig{}, false) - assert.NoError(t, err) - content, err := tmpl.LoadFile(fieldsPath) - assert.NoError(t, err) - - // Ensure clean state - client.Request("DELETE", "/_template/"+tmpl.GetName(), "", nil, nil) - assert.False(t, loader.templateExists(tmpl.GetName())) - - // Load template - err = loader.loadTemplate(tmpl.GetName(), content) - assert.Nil(t, err) - - // Make sure template was loaded - assert.True(t, loader.templateExists(tmpl.GetName())) + setup := newTestSetup(t, TemplateConfig{Name: beat}) + assert.NoError(t, setup.loadFromFile([]string{"..", "..", beat, "fields.yml"})) + assert.True(t, setup.loader.templateExists(setup.config.Name)) } } func TestTemplateSettings(t *testing.T) { - // Setup ES - loader := testEsLoader(t) - client := loader.client - - // Load template - absPath, err := filepath.Abs("../") - assert.NotNil(t, absPath) - assert.Nil(t, err) - - fieldsPath := absPath + "/fields.yml" - settings := TemplateSettings{ - Index: common.MapStr{ - "number_of_shards": 1, - }, - Source: common.MapStr{ - "enabled": false, - }, + Index: common.MapStr{"number_of_shards": 1}, + Source: common.MapStr{"enabled": false}, } - config := TemplateConfig{ - Settings: settings, - } - tmpl, err := New(version.GetDefaultVersion(), "testbeat", client.GetVersion(), config, false) - assert.NoError(t, err) - content, err := tmpl.LoadFile(fieldsPath) - assert.NoError(t, err) - - // Ensure clean state - client.Request("DELETE", "/_template/"+tmpl.GetName(), "", nil, nil) - assert.False(t, loader.templateExists(tmpl.GetName())) - - // Load template - err = loader.loadTemplate(tmpl.GetName(), content) - assert.Nil(t, err) + setup := newTestSetup(t, TemplateConfig{Settings: settings}) + require.NoError(t, setup.loadFromFile([]string{"..", "fields.yml"})) // Check that it contains the mapping - templateJSON := getTemplate(t, client, tmpl.GetName()) + templateJSON := getTemplate(t, setup.client, setup.config.Name) assert.Equal(t, 1, templateJSON.NumberOfShards()) assert.Equal(t, false, templateJSON.SourceEnabled()) } @@ -414,35 +287,12 @@ var dataTests = []struct { // Tests if data can be loaded into elasticsearch with right types func TestTemplateWithData(t *testing.T) { - fieldsPath, err := filepath.Abs("./testdata/fields.yml") - assert.NotNil(t, fieldsPath) - assert.Nil(t, err) - - // Setup ES - client := estest.GetTestingElasticsearch(t) - if err := client.Connect(); err != nil { - t.Fatal(err) - } - loader := NewESLoader(client) - - tmpl, err := New(version.GetDefaultVersion(), "testindex", client.GetVersion(), TemplateConfig{}, false) - assert.NoError(t, err) - content, err := tmpl.LoadFile(fieldsPath) - assert.NoError(t, err) - - // Ensure clean state - client.Request("DELETE", "/_template/"+tmpl.GetName(), "", nil, nil) - assert.False(t, loader.templateExists(tmpl.GetName())) - - // Load template - err = loader.loadTemplate(tmpl.GetName(), content) - assert.Nil(t, err) - - // Make sure template was loaded - assert.True(t, loader.templateExists(tmpl.GetName())) - + setup := newTestSetup(t, TemplateConfig{}) + require.NoError(t, setup.loadFromFile([]string{"testdata", "fields.yml"})) + require.True(t, setup.loader.templateExists(setup.config.Name)) + esClient := setup.client.(*elasticsearch.Client) for _, test := range dataTests { - _, _, err = client.Index(tmpl.GetName(), "_doc", "", nil, test.data) + _, _, err := esClient.Index(setup.config.Name, "_doc", "", nil, test.data) if test.error { assert.NotNil(t, err) @@ -494,3 +344,9 @@ func (tt *testTemplate) NumberOfShards() int { require.NoError(tt.t, err) return i } + +func path(t *testing.T, fileElems []string) string { + fieldsPath, err := filepath.Abs(filepath.Join(fileElems...)) + require.NoError(t, err) + return fieldsPath +} From f6275a2bd54fa6424fd8074b67ffaee404dd876c Mon Sep 17 00:00:00 2001 From: simitt Date: Thu, 9 May 2019 10:43:50 +0200 Subject: [PATCH 4/4] Add changelog entry. --- CHANGELOG-developer.next.asciidoc | 1 + libbeat/template/load_integration_test.go | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/CHANGELOG-developer.next.asciidoc b/CHANGELOG-developer.next.asciidoc index c0a07bcd80e..4ecf5a44c08 100644 --- a/CHANGELOG-developer.next.asciidoc +++ b/CHANGELOG-developer.next.asciidoc @@ -37,3 +37,4 @@ The list below covers the major changes between 7.0.0-rc2 and master only. - Update Jinja2 version to 2.10.1. {pull}11817[11817] - Reduce idxmgmt.Supporter interface and rework export commands to reuse logic. {pull}11777[11777], {pull}12065[12065], {pull}12067[12067] - Update urllib3 version to 1.24.2 {pull}11930[11930] +- Only Load minimal template if no fields are provided. {pull}12103[12103] diff --git a/libbeat/template/load_integration_test.go b/libbeat/template/load_integration_test.go index 5c292c71a47..ae393010f0d 100644 --- a/libbeat/template/load_integration_test.go +++ b/libbeat/template/load_integration_test.go @@ -201,7 +201,7 @@ func TestESLoader_Load(t *testing.T) { } func TestTemplate_LoadFile(t *testing.T) { - setup := newTestSetup(t, TemplateConfig{}) + setup := newTestSetup(t, TemplateConfig{Enabled: true}) assert.NoError(t, setup.loadFromFile([]string{"..", "fields.yml"})) assert.True(t, setup.loader.templateExists(setup.config.Name)) } @@ -223,7 +223,7 @@ func TestLoadBeatsTemplate_fromFile(t *testing.T) { } for _, beat := range beats { - setup := newTestSetup(t, TemplateConfig{Name: beat}) + setup := newTestSetup(t, TemplateConfig{Name: beat, Enabled: true}) assert.NoError(t, setup.loadFromFile([]string{"..", "..", beat, "fields.yml"})) assert.True(t, setup.loader.templateExists(setup.config.Name)) } @@ -234,7 +234,7 @@ func TestTemplateSettings(t *testing.T) { Index: common.MapStr{"number_of_shards": 1}, Source: common.MapStr{"enabled": false}, } - setup := newTestSetup(t, TemplateConfig{Settings: settings}) + setup := newTestSetup(t, TemplateConfig{Settings: settings, Enabled: true}) require.NoError(t, setup.loadFromFile([]string{"..", "fields.yml"})) // Check that it contains the mapping @@ -287,7 +287,7 @@ var dataTests = []struct { // Tests if data can be loaded into elasticsearch with right types func TestTemplateWithData(t *testing.T) { - setup := newTestSetup(t, TemplateConfig{}) + setup := newTestSetup(t, TemplateConfig{Enabled: true}) require.NoError(t, setup.loadFromFile([]string{"testdata", "fields.yml"})) require.True(t, setup.loader.templateExists(setup.config.Name)) esClient := setup.client.(*elasticsearch.Client)