Template setting in the configuration file (#4284)

This allows adding and overwriting index/template settings via the
configuration file.

Example config:

    setup.template.settings:
      index.number_of_shards: 1
      index.number_of_replicas: 1

Needed for #4112, part of #3654.

CHANGELOG.asciidoc

@@ -46,6 +46,8 @@ https://github.com/elastic/beats/compare/v6.0.0-alpha1...master[Check the HEAD d
 
 *Affecting all Beats*
 
+- Added the possibility to set Elasticsearch mapping template settings from the Beat configuration file. {pull}4284[4284]
+
 *Filebeat*
 
 *Heartbeat*

libbeat/docs/index.asciidoc

@@ -11,6 +11,7 @@ include::./version.asciidoc[]
 :beatsdevguide: http://www.elastic.co/guide/en/beats/devguide/{doc-branch}
 :securitydoc: https://www.elastic.co/guide/en/x-pack/5.2
 :logstashdoc: https://www.elastic.co/guide/en/logstash/{doc-branch}
+:elasticsearch: https://www.elastic.co/guide/en/elasticsearch/reference/{doc-branch}
 :beatname_lc: beatname
 :beatname_uc: a Beat
 :security: X-Pack Security

libbeat/docs/template-config.asciidoc

@@ -13,18 +13,25 @@ you must <<load-template-manually,load the template manually>>.
 
 *`name`*:: The name of the template. The default is +{beatname_lc}+.
 
-*`path`*:: The path to the template file. The default is +fields.yml+. If a relative
-path is set, it is considered relative to the config path. See the <<directory-layout>> section for
-details.
+*`fields`*:: The path to the YAML file describing the fields. The default is +fields.yml+. If a
+relative path is set, it is considered relative to the config path. See the <<directory-layout>>
+section for details.
 
 *`overwrite`*:: A boolean that specifies whether to overwrite the existing template. The default
 is false.
 
+*`settings`*:: A dictionary of settings to place into the `settings` dictionary of the Elasticsearch
+template. For more details about the available Elasticsearch mapping options, please see the
+Elasticsearch {elasticsearch}/mapping.html[mapping reference].
+
 For example:
 
 ["source","yaml",subs="attributes,callouts"]
 ----------------------------------------------------------------------
 setup.template.name: "{beatname_lc}"
 setup.template.fields: "fields.yml"
 setup.template.overwrite: false
+setup.template.settings:
+  index.number_of_shards: 1
+  index.number_of_replicas: 1
 ----------------------------------------------------------------------

libbeat/template/config.go

@@ -1,12 +1,12 @@
 package template
 
 type TemplateConfig struct {
-	Enabled      bool              `config:"enabled"`
-	Name         string            `config:"name"`
-	Fields       string            `config:"fields"`
-	Overwrite    bool              `config:"overwrite"`
-	OutputToFile string            `config:"output_to_file"`
-	Settings     map[string]string `config:"settings"`
+	Enabled      bool                   `config:"enabled"`
+	Name         string                 `config:"name"`
+	Fields       string                 `config:"fields"`
+	Overwrite    bool                   `config:"overwrite"`
+	OutputToFile string                 `config:"output_to_file"`
+	Settings     map[string]interface{} `config:"settings"`
 }
 
 var (

libbeat/template/load.go

@@ -57,7 +57,7 @@ func (l *Loader) Load() error {
 			l.config.Name = l.beatInfo.Beat
 		}
 
-		tmpl, err := New(l.beatInfo.Version, l.client.GetVersion(), l.config.Name)
+		tmpl, err := New(l.beatInfo.Version, l.client.GetVersion(), l.config.Name, l.config.Settings)
 		if err != nil {
 			return fmt.Errorf("error creating template instance: %v", err)
 		}

libbeat/template/load_integration_test.go

@@ -3,10 +3,13 @@
 package template
 
 import (
+	"encoding/json"
+	"fmt"
 	"path/filepath"
 	"testing"
 	"time"
 
+	"github.com/elastic/beats/libbeat/common"
 	"github.com/elastic/beats/libbeat/outputs/elasticsearch"
 	"github.com/elastic/beats/libbeat/version"
 
@@ -43,7 +46,7 @@ func TestLoadTemplate(t *testing.T) {
 	fieldsPath := absPath + "/fields.yml"
 	index := "testbeat"
 
-	tmpl, err := New(version.GetDefaultVersion(), client.GetVersion(), index)
+	tmpl, err := New(version.GetDefaultVersion(), client.GetVersion(), index, common.MapStr{})
 	assert.NoError(t, err)
 	content, err := tmpl.Load(fieldsPath)
 	assert.NoError(t, err)
@@ -114,7 +117,7 @@ func TestLoadBeatsTemplate(t *testing.T) {
 		fieldsPath := absPath + "/fields.yml"
 		index := beat
 
-		tmpl, err := New(version.GetDefaultVersion(), client.GetVersion(), index)
+		tmpl, err := New(version.GetDefaultVersion(), client.GetVersion(), index, common.MapStr{})
 		assert.NoError(t, err)
 		content, err := tmpl.Load(fieldsPath)
 		assert.NoError(t, err)
@@ -137,3 +140,56 @@ func TestLoadBeatsTemplate(t *testing.T) {
 		assert.False(t, loader.CheckTemplate(tmpl.GetName()))
 	}
 }
+
+func TestTemplateSettings(t *testing.T) {
+
+	// Setup ES
+	client := elasticsearch.GetTestingElasticsearch()
+	err := client.Connect(5 * time.Second)
+	assert.Nil(t, err)
+
+	// Load template
+	absPath, err := filepath.Abs("../")
+	assert.NotNil(t, absPath)
+	assert.Nil(t, err)
+
+	fieldsPath := absPath + "/fields.yml"
+
+	settings := common.MapStr{
+		"index": common.MapStr{
+			"number_of_shards": 1,
+		},
+	}
+	tmpl, err := New(version.GetDefaultVersion(), client.GetVersion(), "testbeat", settings)
+	assert.NoError(t, err)
+	content, err := tmpl.Load(fieldsPath)
+	assert.NoError(t, err)
+
+	loader := &Loader{
+		client: client,
+	}
+
+	// Load template
+	err = loader.LoadTemplate(tmpl.GetName(), content)
+	assert.Nil(t, err)
+
+	// Check that it contains the mapping
+	status, body, err := loader.client.Request("GET", "/_template/"+tmpl.GetName(), "", nil, nil)
+	assert.NoError(t, err)
+	assert.Equal(t, status, 200)
+
+	var response common.MapStr
+	err = json.Unmarshal(body, &response)
+	assert.NoError(t, err)
+
+	templateJSON := common.MapStr(response[tmpl.GetName()].(map[string]interface{}))
+	val, err := templateJSON.GetValue(fmt.Sprintf("settings.index.number_of_shards"))
+	assert.NoError(t, err)
+	assert.Equal(t, val.(string), "1")
+
+	// Delete template again to clean up
+	client.Request("DELETE", "/_template/"+tmpl.GetName(), "", nil, nil)
+
+	// Make sure it was removed
+	assert.False(t, loader.CheckTemplate(tmpl.GetName()))
+}

libbeat/template/template.go

@@ -20,10 +20,11 @@ type Template struct {
 	index       string
 	beatVersion Version
 	esVersion   Version
+	settings    common.MapStr
 }
 
 // New creates a new template instance
-func New(beatVersion string, esVersion string, index string) (*Template, error) {
+func New(beatVersion string, esVersion string, index string, settings common.MapStr) (*Template, error) {
 
 	bV, err := NewVersion(beatVersion)
 	if err != nil {
@@ -44,6 +45,7 @@ func New(beatVersion string, esVersion string, index string) (*Template, error)
 		index:       index,
 		beatVersion: *bV,
 		esVersion:   *esV,
+		settings:    settings,
 	}, nil
 
 }
@@ -90,6 +92,18 @@ func (t *Template) generate(properties common.MapStr, dynamicTemplates []common.
 
 	dynamicTemplates = append(dynamicTemplates, dynamicTemplateBase)
 
+	settings := common.MapStr{
+		"index": common.MapStr{
+			"refresh_interval": "5s",
+			"mapping": common.MapStr{
+				"total_fields": common.MapStr{
+					"limit": defaultTotalFieldsLimit,
+				},
+			},
+		},
+	}
+	settings.DeepUpdate(t.settings)
+
 	// Load basic structure
 	basicStructure := common.MapStr{
 		"mappings": common.MapStr{
@@ -102,10 +116,8 @@ func (t *Template) generate(properties common.MapStr, dynamicTemplates []common.
 				"properties":        properties,
 			},
 		},
-		"order": 1,
-		"settings": common.MapStr{
-			"index.refresh_interval": "5s",
-		},
+		"order":    1,
+		"settings": settings,
 	}
 
 	// ES 6 moved from template to index_patterns: https://github.com/elastic/elasticsearch/pull/21009
@@ -119,11 +131,6 @@ func (t *Template) generate(properties common.MapStr, dynamicTemplates []common.
 		basicStructure.Put("mappings._default_._all.norms.enabled", false)
 	}
 
-	if t.esVersion.major >= 5 {
-		// Metricbeat exceeds the default of 1000 fields
-		basicStructure.Put("settings.index.mapping.total_fields.limit", defaultTotalFieldsLimit)
-	}
-
 	return basicStructure
 }