Allow template pattern to be overwritten (elastic#4769)

With the current template loading is was not possible to overload the pattern used as name and pattern were mixed. For example in the case where someone wanted to define pattern name `a` that applies to the indice `a` this was not possible because the pattern generated was `a-*` so it didn't apply to the index `a`. This change now allows to set pattern and name separately.

To make sure index pattern and templates are in sync, the beat will not start in case the index pattern was modified but the template patterns not. We should figure out later some automated mechanisms here.

We should find a better solution for 6.x to require less config options and also include dashboard generation in this.

(cherry picked from commit 02a59a1)

CHANGELOG.asciidoc

@@ -16,6 +16,9 @@ https://github.com/elastic/beats/compare/v6.0.0-beta1...master[Check the HEAD di
 - The log directory (`path.log`) for Windows services is now set to `C:\ProgramData\[beatname]\logs`. {issue}4764[4764]
 - The _all field is disabled in 6.0. This means that searching by individual
   words only work on text fields. {issue}4901[4901]
+- Fail if removed setting output.X.flush_interval is explicitly configured.
+- Rename the `/usr/bin/beatname.sh` script (e.g. `metricbeat.sh`) to `/usr/bin/beatname`. {pull}4933[4933]
+- Beat does not start if elasticsearch index pattern was modified but not the template name and pattern. {issue}4769[4769]
 
 *Auditbeat*
 

auditbeat/auditbeat.reference.yml

@@ -209,6 +209,7 @@ output.elasticsearch:
 
   # Optional index name. The default is "auditbeat" plus date
   # and generates [auditbeat-]YYYY.MM.DD keys.
+  # In case you modify this pattern you must update setup.template.name and setup.template.pattern accordingly.
   #index: "auditbeat-%{[beat.version]}-%{+yyyy.MM.dd}"
 
   # Optional ingest node pipeline. By default no pipeline will be used.
@@ -674,10 +675,14 @@ output.elasticsearch:
 # Set to false to disable template loading.
 #setup.template.enabled: true
 
-# Template name. By default the template name is auditbeat.
-# The version of the beat will always be appended to the given name
-# so the final name is auditbeat-%{[beat.version]}.
-#setup.template.name: "auditbeat"
+# Template name. By default the template name is "auditbeat-%{[beat.version]}"
+# The template name and pattern has to be set in case the elasticsearch index pattern is modified.
+#setup.template.name: "auditbeat-%{[beat.version]}"
+
+# Template patttern. By default the template patter is "-%{[beat.version]}-*" to apply to the default index settings.
+# The first part is the version of the beat and then -* is used to match all daily indicies.
+# The template name and pattern has to be set in case the elasticsearch index pattern is modified.
+#setup.template.pattern: "auditbeat-%{[beat.version]}-*"
 
 # Path to fields.yml file to generate the template
 #setup.template.fields: "${path.config}/fields.yml"

filebeat/filebeat.reference.yml

@@ -600,6 +600,7 @@ output.elasticsearch:
 
   # Optional index name. The default is "filebeat" plus date
   # and generates [filebeat-]YYYY.MM.DD keys.
+  # In case you modify this pattern you must update setup.template.name and setup.template.pattern accordingly.
   #index: "filebeat-%{[beat.version]}-%{+yyyy.MM.dd}"
 
   # Optional ingest node pipeline. By default no pipeline will be used.
@@ -1065,10 +1066,14 @@ output.elasticsearch:
 # Set to false to disable template loading.
 #setup.template.enabled: true
 
-# Template name. By default the template name is filebeat.
-# The version of the beat will always be appended to the given name
-# so the final name is filebeat-%{[beat.version]}.
-#setup.template.name: "filebeat"
+# Template name. By default the template name is "filebeat-%{[beat.version]}"
+# The template name and pattern has to be set in case the elasticsearch index pattern is modified.
+#setup.template.name: "filebeat-%{[beat.version]}"
+
+# Template patttern. By default the template patter is "-%{[beat.version]}-*" to apply to the default index settings.
+# The first part is the version of the beat and then -* is used to match all daily indicies.
+# The template name and pattern has to be set in case the elasticsearch index pattern is modified.
+#setup.template.pattern: "filebeat-%{[beat.version]}-*"
 
 # Path to fields.yml file to generate the template
 #setup.template.fields: "${path.config}/fields.yml"

filebeat/tests/system/config/filebeat.yml.j2

@@ -133,6 +133,11 @@ geoip:
   ]
 {%- endif %}
 
+{% if setup_template_name is not none %}
+setup.template.name: setup_template_name
+setup.template.pattern: setup_template_pattern
+{%- endif %}
+
 {%- if processors %}
 
 #================================ Filters =====================================

filebeat/tests/system/config/filebeat_modules.yml.j2

@@ -2,3 +2,6 @@ filebeat.registry_file: {{ beat.working_dir + '/' }}{{ registryFile|default("reg
 
 output.elasticsearch.hosts: ["{{ elasticsearch_url }}"]
 output.elasticsearch.index: {{ index_name }}
+
+setup.template.name: {{ index_name }}
+setup.template.pattern: {{ index_name }}*

filebeat/tests/system/test_modules.py

@@ -46,7 +46,8 @@ def test_modules(self):
             template_name="filebeat_modules",
             output=cfgfile,
             index_name=self.index_name,
-            elasticsearch_url=self.elasticsearch_url)
+            elasticsearch_url=self.elasticsearch_url
+        )
 
         for module in modules:
             path = os.path.join(self.modules_path, module)
@@ -152,6 +153,8 @@ def test_prospector_pipeline_config(self):
                 pipeline="estest",
                 index=index_name),
             pipeline="test",
+            setup_template_name=index_name,
+            setup_template_pattern=index_name + "*",
         )
 
         os.mkdir(self.working_dir + "/log/")

heartbeat/heartbeat.reference.yml

@@ -358,6 +358,7 @@ output.elasticsearch:
 
   # Optional index name. The default is "heartbeat" plus date
   # and generates [heartbeat-]YYYY.MM.DD keys.
+  # In case you modify this pattern you must update setup.template.name and setup.template.pattern accordingly.
   #index: "heartbeat-%{[beat.version]}-%{+yyyy.MM.dd}"
 
   # Optional ingest node pipeline. By default no pipeline will be used.
@@ -823,10 +824,14 @@ output.elasticsearch:
 # Set to false to disable template loading.
 #setup.template.enabled: true
 
-# Template name. By default the template name is heartbeat.
-# The version of the beat will always be appended to the given name
-# so the final name is heartbeat-%{[beat.version]}.
-#setup.template.name: "heartbeat"
+# Template name. By default the template name is "heartbeat-%{[beat.version]}"
+# The template name and pattern has to be set in case the elasticsearch index pattern is modified.
+#setup.template.name: "heartbeat-%{[beat.version]}"
+
+# Template patttern. By default the template patter is "-%{[beat.version]}-*" to apply to the default index settings.
+# The first part is the version of the beat and then -* is used to match all daily indicies.
+# The template name and pattern has to be set in case the elasticsearch index pattern is modified.
+#setup.template.pattern: "heartbeat-%{[beat.version]}-*"
 
 # Path to fields.yml file to generate the template
 #setup.template.fields: "${path.config}/fields.yml"

libbeat/_meta/config.reference.yml

@@ -144,6 +144,7 @@ output.elasticsearch:
 
   # Optional index name. The default is "beatname" plus date
   # and generates [beatname-]YYYY.MM.DD keys.
+  # In case you modify this pattern you must update setup.template.name and setup.template.pattern accordingly.
   #index: "beatname-%{[beat.version]}-%{+yyyy.MM.dd}"
 
   # Optional ingest node pipeline. By default no pipeline will be used.
@@ -609,10 +610,14 @@ output.elasticsearch:
 # Set to false to disable template loading.
 #setup.template.enabled: true
 
-# Template name. By default the template name is beatname.
-# The version of the beat will always be appended to the given name
-# so the final name is beatname-%{[beat.version]}.
-#setup.template.name: "beatname"
+# Template name. By default the template name is "beatname-%{[beat.version]}"
+# The template name and pattern has to be set in case the elasticsearch index pattern is modified.
+#setup.template.name: "beatname-%{[beat.version]}"
+
+# Template patttern. By default the template patter is "-%{[beat.version]}-*" to apply to the default index settings.
+# The first part is the version of the beat and then -* is used to match all daily indicies.
+# The template name and pattern has to be set in case the elasticsearch index pattern is modified.
+#setup.template.pattern: "beatname-%{[beat.version]}-*"
 
 # Path to fields.yml file to generate the template
 #setup.template.fields: "${path.config}/fields.yml"

libbeat/beat/beat.go

@@ -592,9 +592,12 @@ func (b *Beat) loadDashboards(force bool) error {
 // the elasticsearch output. It is important the the registration happens before
 // the publisher is created.
 func (b *Beat) registerTemplateLoading() error {
+
+	var cfg template.TemplateConfig
+
 	// Check if outputting to file is enabled, and output to file if it is
-	if b.Config.Template != nil && b.Config.Template.Enabled() {
-		var cfg template.TemplateConfig
+	if b.Config.Template.Enabled() {
+
 		err := b.Config.Template.Unpack(&cfg)
 		if err != nil {
 			return fmt.Errorf("unpacking template config fails: %v", err)
@@ -603,6 +606,20 @@ func (b *Beat) registerTemplateLoading() error {
 
 	// Loads template by default if esOutput is enabled
 	if b.Config.Output.Name() == "elasticsearch" {
+
+		// Get ES Index name for comparison
+		esCfg := struct {
+			Index string `config:"index"`
+		}{}
+		err := b.Config.Output.Config().Unpack(&esCfg)
+		if err != nil {
+			return err
+		}
+
+		if esCfg.Index != "" && (cfg.Name == "" || cfg.Pattern == "") {
+			return fmt.Errorf("setup.template.name and setup.template.pattern have to be set if index name is modified.")
+		}
+
 		if b.Config.Template == nil || (b.Config.Template != nil && b.Config.Template.Enabled()) {
 			// load template through callback to make sure it is also loaded
 			// on reconnecting

libbeat/cmd/export/template.go

@@ -38,7 +38,7 @@ func GenTemplateConfigCmd(name, beatVersion string) *cobra.Command {
 				}
 			}
 
-			tmpl, err := template.New(b.Info.Version, version, index, cfg.Settings)
+			tmpl, err := template.New(b.Info.Version, index, version, cfg)
 			if err != nil {
 				fmt.Fprintf(os.Stderr, "Error generating template: %+v", err)
 				os.Exit(1)

libbeat/template/config.go

@@ -3,6 +3,7 @@ package template
 type TemplateConfig struct {
 	Enabled   bool             `config:"enabled"`
 	Name      string           `config:"name"`
+	Pattern   string           `config:"pattern"`
 	Fields    string           `config:"fields"`
 	Overwrite bool             `config:"overwrite"`
 	Settings  TemplateSettings `config:"settings"`

libbeat/template/load.go

@@ -41,11 +41,8 @@ func NewLoader(cfg *common.Config, client ESClient, beatInfo common.BeatInfo) (*
 // In case the template is not already loaded or overwriting is enabled, the
 // template is written to index
 func (l *Loader) Load() error {
-	if l.config.Name == "" {
-		l.config.Name = l.beatInfo.Beat
-	}
 
-	tmpl, err := New(l.beatInfo.Version, l.client.GetVersion(), l.config.Name, l.config.Settings)
+	tmpl, err := New(l.beatInfo.Version, l.beatInfo.Beat, l.client.GetVersion(), l.config)
 	if err != nil {
 		return fmt.Errorf("error creating template instance: %v", err)
 	}

libbeat/template/load_integration_test.go

@@ -43,7 +43,7 @@ func TestLoadTemplate(t *testing.T) {
 	fieldsPath := absPath + "/fields.yml"
 	index := "testbeat"
 
-	tmpl, err := New(version.GetDefaultVersion(), client.GetVersion(), index, TemplateSettings{})
+	tmpl, err := New(version.GetDefaultVersion(), index, client.GetVersion(), TemplateConfig{})
 	assert.NoError(t, err)
 	content, err := tmpl.Load(fieldsPath)
 	assert.NoError(t, err)
@@ -131,7 +131,7 @@ func TestLoadBeatsTemplate(t *testing.T) {
 		fieldsPath := absPath + "/fields.yml"
 		index := beat
 
-		tmpl, err := New(version.GetDefaultVersion(), client.GetVersion(), index, TemplateSettings{})
+		tmpl, err := New(version.GetDefaultVersion(), index, client.GetVersion(), TemplateConfig{})
 		assert.NoError(t, err)
 		content, err := tmpl.Load(fieldsPath)
 		assert.NoError(t, err)
@@ -177,7 +177,10 @@ func TestTemplateSettings(t *testing.T) {
 			"enabled": false,
 		},
 	}
-	tmpl, err := New(version.GetDefaultVersion(), client.GetVersion(), "testbeat", settings)
+	config := TemplateConfig{
+		Settings: settings,
+	}
+	tmpl, err := New(version.GetDefaultVersion(), "testbeat", client.GetVersion(), config)
 	assert.NoError(t, err)
 	content, err := tmpl.Load(fieldsPath)
 	assert.NoError(t, err)
@@ -334,7 +337,7 @@ func TestTemplateWithData(t *testing.T) {
 	// Setup ES
 	client := elasticsearch.GetTestingElasticsearch(t)
 
-	tmpl, err := New(version.GetDefaultVersion(), client.GetVersion(), "testindex", TemplateSettings{})
+	tmpl, err := New(version.GetDefaultVersion(), "testindex", client.GetVersion(), TemplateConfig{})
 	assert.NoError(t, err)
 	content, err := tmpl.Load(fieldsPath)
 	assert.NoError(t, err)

libbeat/template/template.go

@@ -2,8 +2,11 @@ package template
 
 import (
 	"fmt"
+	"time"
 
 	"github.com/elastic/beats/libbeat/common"
+	"github.com/elastic/beats/libbeat/common/fmtstr"
+	"github.com/elastic/beats/libbeat/publisher/beat"
 	"github.com/elastic/go-ucfg/yaml"
 )
 
@@ -17,19 +20,58 @@ var (
 )
 
 type Template struct {
-	index       string
+	name        string
+	pattern     string
 	beatVersion common.Version
 	esVersion   common.Version
 	settings    TemplateSettings
 }
 
 // New creates a new template instance
-func New(beatVersion string, esVersion string, index string, settings TemplateSettings) (*Template, error) {
+func New(beatVersion string, beatName string, esVersion string, config TemplateConfig) (*Template, error) {
 	bV, err := common.NewVersion(beatVersion)
 	if err != nil {
 		return nil, err
 	}
 
+	name := config.Name
+	if name == "" {
+		name = fmt.Sprintf("%s-%s", beatName, bV.String())
+	}
+
+	pattern := config.Pattern
+	if pattern == "" {
+		pattern = name + "-*"
+	}
+
+	event := &beat.Event{
+		Fields: common.MapStr{
+			"beat": common.MapStr{
+				"name":    beatName,
+				"version": bV.String(),
+			},
+		},
+		Timestamp: time.Now(),
+	}
+
+	nameFormatter, err := fmtstr.CompileEvent(name)
+	if err != nil {
+		return nil, err
+	}
+	name, err = nameFormatter.Run(event)
+	if err != nil {
+		return nil, err
+	}
+
+	patternFormatter, err := fmtstr.CompileEvent(pattern)
+	if err != nil {
+		return nil, err
+	}
+	pattern, err = patternFormatter.Run(event)
+	if err != nil {
+		return nil, err
+	}
+
 	// In case no esVersion is set, it is assumed the same as beat version
 	if esVersion == "" {
 		esVersion = beatVersion
@@ -41,10 +83,11 @@ func New(beatVersion string, esVersion string, index string, settings TemplateSe
 	}
 
 	return &Template{
-		index:       index,
+		pattern:     pattern,
+		name:        name,
 		beatVersion: *bV,
 		esVersion:   *esV,
-		settings:    settings,
+		settings:    config.Settings,
 	}, nil
 }
 
@@ -62,9 +105,14 @@ func (t *Template) Load(file string) (common.MapStr, error) {
 	return output, nil
 }
 
-// GetName returns the name of the template which is {index}-{version}
+// GetName returns the name of the template
 func (t *Template) GetName() string {
-	return fmt.Sprintf("%s-%s", t.index, t.beatVersion.String())
+	return t.name
+}
+
+// GetPattern returns the pattern of the template
+func (t *Template) GetPattern() string {
+	return t.pattern
 }
 
 // generate generates the full template
@@ -122,9 +170,9 @@ func (t *Template) generate(properties common.MapStr, dynamicTemplates []common.
 
 	// ES 6 moved from template to index_patterns: https://github.com/elastic/elasticsearch/pull/21009
 	if t.esVersion.Major >= 6 {
-		basicStructure.Put("index_patterns", []string{t.GetName() + "-*"})
+		basicStructure.Put("index_patterns", []string{t.GetPattern()})
 	} else {
-		basicStructure.Put("template", t.GetName()+"-*")
+		basicStructure.Put("template", t.GetPattern())
 	}
 
 	if t.esVersion.IsMajor(2) {