Skip to content

Commit

Permalink
Add option to adjust _source options + improvements (#4317)
Browse files Browse the repository at this point in the history 
This adds/fixes:

* Ability to disable _source, or set other _source related options
* Moves template index settings under `settings.index`
* Fixes the overwrite logic (was using the wrong template name on the check)
* Fixes error handling
* Integration tests for overwritting

Part of #3654 and #4112.
  • Loading branch information
@tsg @ruflin
tsg authored and ruflin committed May 16, 2017
1 parent cf92660 commit b2ccdcb
Show file tree
Hide file tree
Showing 7 changed files with 169 additions and 51 deletions.
2 changes: 1 addition & 1 deletion CHANGELOG.asciidoc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ https://github.com/elastic/beats/compare/v6.0.0-alpha1...master[Check the HEAD d

*Affecting all Beats*

- Added the possibility to set Elasticsearch mapping template settings from the Beat configuration file. {pull}4284[4284]
- Added the possibility to set Elasticsearch mapping template settings from the Beat configuration file. {pull}4284[4284] {pull}4317[4317]

*Filebeat*

Expand Down
8 changes: 5 additions & 3 deletions libbeat/beat/beat.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -487,12 +487,14 @@ func (b *Beat) registerTemplateLoading() error {

loader, err := template.NewLoader(b.Config.Template, esClient, b.Info)
if err != nil {
return fmt.Errorf("Error loading elasticsearch template: %v", err)
return fmt.Errorf("Error creating Elasticsearch template: %v", err)
}

loader.Load()
err = loader.Load()
if err != nil {
return fmt.Errorf("Error loading Elasticsearch template: %v", err)
}

logp.Info("ES template successfully loaded.")
return nil
}

Expand Down
22 changes: 18 additions & 4 deletions libbeat/docs/template-config.asciidoc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,11 @@ section for details.
*`overwrite`*:: A boolean that specifies whether to overwrite the existing template. The default
is false.

*`settings`*:: A dictionary of settings to place into the `settings` dictionary of the Elasticsearch
template. For more details about the available Elasticsearch mapping options, please see the
Elasticsearch {elasticsearch}/mapping.html[mapping reference].
*`settings.index`*:: A dictionary of settings to place into the `settings.index` dictionary of the
Elasticsearch template. For more details about the available Elasticsearch mapping options, please
see the Elasticsearch {elasticsearch}/mapping.html[mapping reference].

For example:
Example:

["source","yaml",subs="attributes,callouts"]
----------------------------------------------------------------------
Expand All @@ -35,3 +35,17 @@ setup.template.settings:
index.number_of_shards: 1
index.number_of_replicas: 1
----------------------------------------------------------------------

*`settings._source`*:: A dictionary of settings for the `_source` field. For the available settings,
please see the Elasticsearch {elasticsearch}/mapping-source-field.html[reference].

Example:

["source","yaml",subs="attributes,callouts"]
----------------------------------------------------------------------
setup.template.name: "{beatname_lc}"
setup.template.fields: "fields.yml"
setup.template.overwrite: false
setup.template.settings:
_source.enabled: false
----------------------------------------------------------------------
17 changes: 11 additions & 6 deletions libbeat/template/config.go
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,17 @@
package template

type TemplateConfig struct {
Enabled bool `config:"enabled"`
Name string `config:"name"`
Fields string `config:"fields"`
Overwrite bool `config:"overwrite"`
OutputToFile string `config:"output_to_file"`
Settings map[string]interface{} `config:"settings"`
Enabled bool `config:"enabled"`
Name string `config:"name"`
Fields string `config:"fields"`
Overwrite bool `config:"overwrite"`
OutputToFile string `config:"output_to_file"`
Settings templateSettings `config:"settings"`
}

type templateSettings struct {
Index map[string]interface{} `config:"index"`
Source map[string]interface{} `config:"_source"`
}

var (
Expand Down
22 changes: 11 additions & 11 deletions libbeat/template/load.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,25 +43,25 @@ func NewLoader(cfg *common.Config, client ESClient, beatInfo common.BeatInfo) (*
// template is written to index
func (l *Loader) Load() error {

if l.config.Name == "" {
l.config.Name = l.beatInfo.Beat
}

tmpl, err := New(l.beatInfo.Version, l.client.GetVersion(), l.config.Name, l.config.Settings)
if err != nil {
return fmt.Errorf("error creating template instance: %v", err)
}

// Check if template already exist or should be overwritten
exists := l.CheckTemplate(l.config.Name)
exists := l.CheckTemplate(tmpl.GetName())
if !exists || l.config.Overwrite {

logp.Info("Loading template for elasticsearch version: %s", l.client.GetVersion())
logp.Info("Loading template for Elasticsearch version: %s", l.client.GetVersion())

if l.config.Overwrite {
logp.Info("Existing template will be overwritten, as overwrite is enabled.")
}

if l.config.Name == "" {
l.config.Name = l.beatInfo.Beat
}

tmpl, err := New(l.beatInfo.Version, l.client.GetVersion(), l.config.Name, l.config.Settings)
if err != nil {
return fmt.Errorf("error creating template instance: %v", err)
}

fieldsPath := paths.Resolve(paths.Config, l.config.Fields)

output, err := tmpl.Load(fieldsPath)
Expand Down
121 changes: 107 additions & 14 deletions libbeat/template/load_integration_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ package template

import (
"encoding/json"
"fmt"
"path/filepath"
"testing"
"time"
Expand Down Expand Up @@ -46,7 +45,7 @@ func TestLoadTemplate(t *testing.T) {
fieldsPath := absPath + "/fields.yml"
index := "testbeat"

tmpl, err := New(version.GetDefaultVersion(), client.GetVersion(), index, common.MapStr{})
tmpl, err := New(version.GetDefaultVersion(), client.GetVersion(), index, templateSettings{})
assert.NoError(t, err)
content, err := tmpl.Load(fieldsPath)
assert.NoError(t, err)
Expand Down Expand Up @@ -95,6 +94,25 @@ func TestLoadInvalidTemplate(t *testing.T) {
assert.False(t, loader.CheckTemplate(templateName))
}

func getTemplate(t *testing.T, client ESClient, templateName string) common.MapStr {

status, body, err := client.Request("GET", "/_template/"+templateName, "", nil, nil)
assert.NoError(t, err)
assert.Equal(t, status, 200)

var response common.MapStr
err = json.Unmarshal(body, &response)
assert.NoError(t, err)

return common.MapStr(response[templateName].(map[string]interface{}))
}

func newConfigFrom(t *testing.T, from interface{}) *common.Config {
cfg, err := common.NewConfigFrom(from)
assert.NoError(t, err)
return cfg
}

// Tests loading the templates for each beat
func TestLoadBeatsTemplate(t *testing.T) {

Expand All @@ -117,7 +135,7 @@ func TestLoadBeatsTemplate(t *testing.T) {
fieldsPath := absPath + "/fields.yml"
index := beat

tmpl, err := New(version.GetDefaultVersion(), client.GetVersion(), index, common.MapStr{})
tmpl, err := New(version.GetDefaultVersion(), client.GetVersion(), index, templateSettings{})
assert.NoError(t, err)
content, err := tmpl.Load(fieldsPath)
assert.NoError(t, err)
Expand Down Expand Up @@ -155,10 +173,13 @@ func TestTemplateSettings(t *testing.T) {

fieldsPath := absPath + "/fields.yml"

settings := common.MapStr{
"index": common.MapStr{
settings := templateSettings{
Index: common.MapStr{
"number_of_shards": 1,
},
Source: common.MapStr{
"enabled": false,
},
}
tmpl, err := New(version.GetDefaultVersion(), client.GetVersion(), "testbeat", settings)
assert.NoError(t, err)
Expand All @@ -174,22 +195,94 @@ func TestTemplateSettings(t *testing.T) {
assert.Nil(t, err)

// Check that it contains the mapping
status, body, err := loader.client.Request("GET", "/_template/"+tmpl.GetName(), "", nil, nil)
assert.NoError(t, err)
assert.Equal(t, status, 200)

var response common.MapStr
err = json.Unmarshal(body, &response)
templateJSON := getTemplate(t, client, tmpl.GetName())
val, err := templateJSON.GetValue("settings.index.number_of_shards")
assert.NoError(t, err)
assert.Equal(t, val.(string), "1")

templateJSON := common.MapStr(response[tmpl.GetName()].(map[string]interface{}))
val, err := templateJSON.GetValue(fmt.Sprintf("settings.index.number_of_shards"))
val, err = templateJSON.GetValue("mappings._default_._source.enabled")
assert.NoError(t, err)
assert.Equal(t, val.(string), "1")
assert.Equal(t, val.(bool), false)

// Delete template again to clean up
client.Request("DELETE", "/_template/"+tmpl.GetName(), "", nil, nil)

// Make sure it was removed
assert.False(t, loader.CheckTemplate(tmpl.GetName()))
}

func TestOverwrite(t *testing.T) {

// Setup ES
client := elasticsearch.GetTestingElasticsearch()
err := client.Connect(5 * time.Second)
assert.Nil(t, err)

beatInfo := common.BeatInfo{
Beat: "testbeat",
Version: version.GetDefaultVersion(),
}
templateName := "testbeat-" + version.GetDefaultVersion()

absPath, err := filepath.Abs("../")
assert.NotNil(t, absPath)
assert.Nil(t, err)

// make sure no template is already there
client.Request("DELETE", "/_template/"+templateName, "", nil, nil)

// Load template
config := newConfigFrom(t, TemplateConfig{
Enabled: true,
Fields: absPath + "/fields.yml",
})
loader, err := NewLoader(config, client, beatInfo)
assert.NoError(t, err)
err = loader.Load()
assert.NoError(t, err)

// Load template again, this time with custom settings
config = newConfigFrom(t, TemplateConfig{
Enabled: true,
Fields: absPath + "/fields.yml",
Settings: templateSettings{
Source: map[string]interface{}{
"enabled": false,
},
},
})
loader, err = NewLoader(config, client, beatInfo)
assert.NoError(t, err)
err = loader.Load()
assert.NoError(t, err)

// Overwrite was not enabled, so the first version should still be there
templateJSON := getTemplate(t, client, templateName)
_, err = templateJSON.GetValue("mappings._default_._source.enabled")
assert.Error(t, err)

// Load template again, this time with custom settings AND overwrite: true
config = newConfigFrom(t, TemplateConfig{
Enabled: true,
Overwrite: true,
Fields: absPath + "/fields.yml",
Settings: templateSettings{
Source: map[string]interface{}{
"enabled": false,
},
},
})
loader, err = NewLoader(config, client, beatInfo)
assert.NoError(t, err)
err = loader.Load()
assert.NoError(t, err)

// Overwrite was enabled, so the custom setting should be there
templateJSON = getTemplate(t, client, templateName)
val, err := templateJSON.GetValue("mappings._default_._source.enabled")
assert.NoError(t, err)
assert.Equal(t, val.(bool), false)

// Delete template again to clean up
client.Request("DELETE", "/_template/"+templateName, "", nil, nil)
}
28 changes: 16 additions & 12 deletions libbeat/template/template.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,11 @@ type Template struct {
index string
beatVersion Version
esVersion Version
settings common.MapStr
settings templateSettings
}

// New creates a new template instance
func New(beatVersion string, esVersion string, index string, settings common.MapStr) (*Template, error) {
func New(beatVersion string, esVersion string, index string, settings templateSettings) (*Template, error) {

bV, err := NewVersion(beatVersion)
if err != nil {
Expand Down Expand Up @@ -92,17 +92,15 @@ func (t *Template) generate(properties common.MapStr, dynamicTemplates []common.

dynamicTemplates = append(dynamicTemplates, dynamicTemplateBase)

settings := common.MapStr{
"index": common.MapStr{
"refresh_interval": "5s",
"mapping": common.MapStr{
"total_fields": common.MapStr{
"limit": defaultTotalFieldsLimit,
},
indexSettings := common.MapStr{
"refresh_interval": "5s",
"mapping": common.MapStr{
"total_fields": common.MapStr{
"limit": defaultTotalFieldsLimit,
},
},
}
settings.DeepUpdate(t.settings)
indexSettings.DeepUpdate(t.settings.Index)

// Load basic structure
basicStructure := common.MapStr{
Expand All @@ -116,8 +114,14 @@ func (t *Template) generate(properties common.MapStr, dynamicTemplates []common.
"properties": properties,
},
},
"order": 1,
"settings": settings,
"order": 1,
"settings": common.MapStr{
"index": indexSettings,
},
}

if len(t.settings.Source) > 0 {
basicStructure.Put("mappings._default_._source", t.settings.Source)
}

// ES 6 moved from template to index_patterns: https://github.com/elastic/elasticsearch/pull/21009
Expand Down

0 comments on commit b2ccdcb

Please sign in to comment.