From 2a8f795f2613a0d168e6529a4244f24d6809f616 Mon Sep 17 00:00:00 2001 From: Nicolas Ruflin Date: Tue, 22 Aug 2017 00:53:06 +0200 Subject: [PATCH] Allow template pattern to be overwritten (#4769) With the current template loading is was not possible to overload the pattern used as name and pattern were mixed. For example in the case where someone wanted to define pattern name `a` that applies to the indice `a` this was not possible because the pattern generated was `a-*` so it didn't apply to the index `a`. This change now allows to set pattern and name separately. To make sure index pattern and templates are in sync, the beat will not start in case the index pattern was modified but the template patterns not. We should figure out later some automated mechanisms here. We should find a better solution for 6.x to require less config options and also include dashboard generation in this. --- CHANGELOG.asciidoc | 1 + auditbeat/auditbeat.reference.yml | 13 +++- filebeat/filebeat.reference.yml | 13 +++- filebeat/tests/system/config/filebeat.yml.j2 | 5 ++ .../system/config/filebeat_modules.yml.j2 | 3 + filebeat/tests/system/test_modules.py | 5 +- heartbeat/heartbeat.reference.yml | 13 +++- libbeat/_meta/config.reference.yml | 13 +++- libbeat/cmd/export/template.go | 2 +- libbeat/cmd/instance/beat.go | 21 +++++- libbeat/template/config.go | 1 + libbeat/template/load.go | 5 +- libbeat/template/load_integration_test.go | 11 ++- libbeat/template/template.go | 64 ++++++++++++++-- libbeat/tests/system/beat/beat.py | 8 +- libbeat/tests/system/config/mockbeat.yml.j2 | 3 +- libbeat/tests/system/test_template.py | 74 +++++++++++++++++++ metricbeat/metricbeat.reference.yml | 13 +++- packetbeat/packetbeat.reference.yml | 13 +++- winlogbeat/winlogbeat.reference.yml | 13 +++- 20 files changed, 244 insertions(+), 50 deletions(-) create mode 100644 libbeat/tests/system/test_template.py diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 611e2ccf8ad..f10062d1b9a 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -17,6 +17,7 @@ https://github.com/elastic/beats/compare/v6.0.0-beta1...master[Check the HEAD di - The log directory (`path.log`) for Windows services is now set to `C:\ProgramData\[beatname]\logs`. {issue}4764[4764] - Fail if removed setting output.X.flush_interval is explicitly configured. - Rename the `/usr/bin/beatname.sh` script (e.g. `metricbeat.sh`) to `/usr/bin/beatname`. {pull}4933[4933] +- Beat does not start if elasticsearch index pattern was modified but not the template name and pattern. {issue}4769[4769] *Auditbeat* diff --git a/auditbeat/auditbeat.reference.yml b/auditbeat/auditbeat.reference.yml index 049ba139cac..45c92fecc4c 100644 --- a/auditbeat/auditbeat.reference.yml +++ b/auditbeat/auditbeat.reference.yml @@ -209,6 +209,7 @@ output.elasticsearch: # Optional index name. The default is "auditbeat" plus date # and generates [auditbeat-]YYYY.MM.DD keys. + # In case you modify this pattern you must update setup.template.name and setup.template.pattern accordingly. #index: "auditbeat-%{[beat.version]}-%{+yyyy.MM.dd}" # Optional ingest node pipeline. By default no pipeline will be used. @@ -666,10 +667,14 @@ output.elasticsearch: # Set to false to disable template loading. #setup.template.enabled: true -# Template name. By default the template name is auditbeat. -# The version of the beat will always be appended to the given name -# so the final name is auditbeat-%{[beat.version]}. -#setup.template.name: "auditbeat" +# Template name. By default the template name is "auditbeat-%{[beat.version]}" +# The template name and pattern has to be set in case the elasticsearch index pattern is modified. +#setup.template.name: "auditbeat-%{[beat.version]}" + +# Template patttern. By default the template patter is "-%{[beat.version]}-*" to apply to the default index settings. +# The first part is the version of the beat and then -* is used to match all daily indicies. +# The template name and pattern has to be set in case the elasticsearch index pattern is modified. +#setup.template.pattern: "auditbeat-%{[beat.version]}-*" # Path to fields.yml file to generate the template #setup.template.fields: "${path.config}/fields.yml" diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml index 058d4d88dee..a3df4633cd1 100644 --- a/filebeat/filebeat.reference.yml +++ b/filebeat/filebeat.reference.yml @@ -629,6 +629,7 @@ output.elasticsearch: # Optional index name. The default is "filebeat" plus date # and generates [filebeat-]YYYY.MM.DD keys. + # In case you modify this pattern you must update setup.template.name and setup.template.pattern accordingly. #index: "filebeat-%{[beat.version]}-%{+yyyy.MM.dd}" # Optional ingest node pipeline. By default no pipeline will be used. @@ -1086,10 +1087,14 @@ output.elasticsearch: # Set to false to disable template loading. #setup.template.enabled: true -# Template name. By default the template name is filebeat. -# The version of the beat will always be appended to the given name -# so the final name is filebeat-%{[beat.version]}. -#setup.template.name: "filebeat" +# Template name. By default the template name is "filebeat-%{[beat.version]}" +# The template name and pattern has to be set in case the elasticsearch index pattern is modified. +#setup.template.name: "filebeat-%{[beat.version]}" + +# Template patttern. By default the template patter is "-%{[beat.version]}-*" to apply to the default index settings. +# The first part is the version of the beat and then -* is used to match all daily indicies. +# The template name and pattern has to be set in case the elasticsearch index pattern is modified. +#setup.template.pattern: "filebeat-%{[beat.version]}-*" # Path to fields.yml file to generate the template #setup.template.fields: "${path.config}/fields.yml" diff --git a/filebeat/tests/system/config/filebeat.yml.j2 b/filebeat/tests/system/config/filebeat.yml.j2 index bf61fd1c244..7ab3e3115e7 100644 --- a/filebeat/tests/system/config/filebeat.yml.j2 +++ b/filebeat/tests/system/config/filebeat.yml.j2 @@ -133,6 +133,11 @@ geoip: ] {%- endif %} +{% if setup_template_name is not none %} +setup.template.name: setup_template_name +setup.template.pattern: setup_template_pattern +{%- endif %} + {%- if processors %} #================================ Filters ===================================== diff --git a/filebeat/tests/system/config/filebeat_modules.yml.j2 b/filebeat/tests/system/config/filebeat_modules.yml.j2 index 3e8083fc037..b62c9c293c9 100644 --- a/filebeat/tests/system/config/filebeat_modules.yml.j2 +++ b/filebeat/tests/system/config/filebeat_modules.yml.j2 @@ -2,3 +2,6 @@ filebeat.registry_file: {{ beat.working_dir + '/' }}{{ registryFile|default("reg output.elasticsearch.hosts: ["{{ elasticsearch_url }}"] output.elasticsearch.index: {{ index_name }} + +setup.template.name: {{ index_name }} +setup.template.pattern: {{ index_name }}* diff --git a/filebeat/tests/system/test_modules.py b/filebeat/tests/system/test_modules.py index 4b0f79bc59d..582a80b444e 100644 --- a/filebeat/tests/system/test_modules.py +++ b/filebeat/tests/system/test_modules.py @@ -46,7 +46,8 @@ def test_modules(self): template_name="filebeat_modules", output=cfgfile, index_name=self.index_name, - elasticsearch_url=self.elasticsearch_url) + elasticsearch_url=self.elasticsearch_url + ) for module in modules: path = os.path.join(self.modules_path, module) @@ -161,6 +162,8 @@ def test_prospector_pipeline_config(self): pipeline="estest", index=index_name), pipeline="test", + setup_template_name=index_name, + setup_template_pattern=index_name + "*", ) os.mkdir(self.working_dir + "/log/") diff --git a/heartbeat/heartbeat.reference.yml b/heartbeat/heartbeat.reference.yml index c3b0ebdce58..82d975ecc47 100644 --- a/heartbeat/heartbeat.reference.yml +++ b/heartbeat/heartbeat.reference.yml @@ -358,6 +358,7 @@ output.elasticsearch: # Optional index name. The default is "heartbeat" plus date # and generates [heartbeat-]YYYY.MM.DD keys. + # In case you modify this pattern you must update setup.template.name and setup.template.pattern accordingly. #index: "heartbeat-%{[beat.version]}-%{+yyyy.MM.dd}" # Optional ingest node pipeline. By default no pipeline will be used. @@ -815,10 +816,14 @@ output.elasticsearch: # Set to false to disable template loading. #setup.template.enabled: true -# Template name. By default the template name is heartbeat. -# The version of the beat will always be appended to the given name -# so the final name is heartbeat-%{[beat.version]}. -#setup.template.name: "heartbeat" +# Template name. By default the template name is "heartbeat-%{[beat.version]}" +# The template name and pattern has to be set in case the elasticsearch index pattern is modified. +#setup.template.name: "heartbeat-%{[beat.version]}" + +# Template patttern. By default the template patter is "-%{[beat.version]}-*" to apply to the default index settings. +# The first part is the version of the beat and then -* is used to match all daily indicies. +# The template name and pattern has to be set in case the elasticsearch index pattern is modified. +#setup.template.pattern: "heartbeat-%{[beat.version]}-*" # Path to fields.yml file to generate the template #setup.template.fields: "${path.config}/fields.yml" diff --git a/libbeat/_meta/config.reference.yml b/libbeat/_meta/config.reference.yml index 478a48e97cc..6dd7c4b0c2b 100644 --- a/libbeat/_meta/config.reference.yml +++ b/libbeat/_meta/config.reference.yml @@ -144,6 +144,7 @@ output.elasticsearch: # Optional index name. The default is "beatname" plus date # and generates [beatname-]YYYY.MM.DD keys. + # In case you modify this pattern you must update setup.template.name and setup.template.pattern accordingly. #index: "beatname-%{[beat.version]}-%{+yyyy.MM.dd}" # Optional ingest node pipeline. By default no pipeline will be used. @@ -601,10 +602,14 @@ output.elasticsearch: # Set to false to disable template loading. #setup.template.enabled: true -# Template name. By default the template name is beatname. -# The version of the beat will always be appended to the given name -# so the final name is beatname-%{[beat.version]}. -#setup.template.name: "beatname" +# Template name. By default the template name is "beatname-%{[beat.version]}" +# The template name and pattern has to be set in case the elasticsearch index pattern is modified. +#setup.template.name: "beatname-%{[beat.version]}" + +# Template patttern. By default the template patter is "-%{[beat.version]}-*" to apply to the default index settings. +# The first part is the version of the beat and then -* is used to match all daily indicies. +# The template name and pattern has to be set in case the elasticsearch index pattern is modified. +#setup.template.pattern: "beatname-%{[beat.version]}-*" # Path to fields.yml file to generate the template #setup.template.fields: "${path.config}/fields.yml" diff --git a/libbeat/cmd/export/template.go b/libbeat/cmd/export/template.go index 876690b6803..1403cf8b8cf 100644 --- a/libbeat/cmd/export/template.go +++ b/libbeat/cmd/export/template.go @@ -38,7 +38,7 @@ func GenTemplateConfigCmd(name, beatVersion string) *cobra.Command { } } - tmpl, err := template.New(b.Info.Version, version, index, cfg.Settings) + tmpl, err := template.New(b.Info.Version, index, version, cfg) if err != nil { fmt.Fprintf(os.Stderr, "Error generating template: %+v", err) os.Exit(1) diff --git a/libbeat/cmd/instance/beat.go b/libbeat/cmd/instance/beat.go index 22afc618bdf..6352895aedd 100644 --- a/libbeat/cmd/instance/beat.go +++ b/libbeat/cmd/instance/beat.go @@ -542,9 +542,11 @@ func (b *Beat) loadDashboards(force bool) error { // the elasticsearch output. It is important the the registration happens before // the publisher is created. func (b *Beat) registerTemplateLoading() error { + + var cfg template.TemplateConfig + // Check if outputting to file is enabled, and output to file if it is - if b.Config.Template != nil && b.Config.Template.Enabled() { - var cfg template.TemplateConfig + if b.Config.Template.Enabled() { err := b.Config.Template.Unpack(&cfg) if err != nil { return fmt.Errorf("unpacking template config fails: %v", err) @@ -553,7 +555,22 @@ func (b *Beat) registerTemplateLoading() error { // Loads template by default if esOutput is enabled if b.Config.Output.Name() == "elasticsearch" { + + // Get ES Index name for comparison + esCfg := struct { + Index string `config:"index"` + }{} + err := b.Config.Output.Config().Unpack(&esCfg) + if err != nil { + return err + } + + if esCfg.Index != "" && (cfg.Name == "" || cfg.Pattern == "") { + return fmt.Errorf("setup.template.name and setup.template.pattern have to be set if index name is modified.") + } + if b.Config.Template == nil || (b.Config.Template != nil && b.Config.Template.Enabled()) { + // load template through callback to make sure it is also loaded // on reconnecting callback, err := b.templateLoadingCallback() diff --git a/libbeat/template/config.go b/libbeat/template/config.go index 9aacc7e997a..eec7e6dab57 100644 --- a/libbeat/template/config.go +++ b/libbeat/template/config.go @@ -3,6 +3,7 @@ package template type TemplateConfig struct { Enabled bool `config:"enabled"` Name string `config:"name"` + Pattern string `config:"pattern"` Fields string `config:"fields"` Overwrite bool `config:"overwrite"` Settings TemplateSettings `config:"settings"` diff --git a/libbeat/template/load.go b/libbeat/template/load.go index b8648f809a0..07da99f3af2 100644 --- a/libbeat/template/load.go +++ b/libbeat/template/load.go @@ -42,11 +42,8 @@ func NewLoader(cfg *common.Config, client ESClient, beatInfo beat.Info) (*Loader // In case the template is not already loaded or overwriting is enabled, the // template is written to index func (l *Loader) Load() error { - if l.config.Name == "" { - l.config.Name = l.beatInfo.Beat - } - tmpl, err := New(l.beatInfo.Version, l.client.GetVersion(), l.config.Name, l.config.Settings) + tmpl, err := New(l.beatInfo.Version, l.beatInfo.Beat, l.client.GetVersion(), l.config) if err != nil { return fmt.Errorf("error creating template instance: %v", err) } diff --git a/libbeat/template/load_integration_test.go b/libbeat/template/load_integration_test.go index 827927a95e3..9e21e30036f 100644 --- a/libbeat/template/load_integration_test.go +++ b/libbeat/template/load_integration_test.go @@ -44,7 +44,7 @@ func TestLoadTemplate(t *testing.T) { fieldsPath := absPath + "/fields.yml" index := "testbeat" - tmpl, err := New(version.GetDefaultVersion(), client.GetVersion(), index, TemplateSettings{}) + tmpl, err := New(version.GetDefaultVersion(), index, client.GetVersion(), TemplateConfig{}) assert.NoError(t, err) content, err := tmpl.Load(fieldsPath) assert.NoError(t, err) @@ -132,7 +132,7 @@ func TestLoadBeatsTemplate(t *testing.T) { fieldsPath := absPath + "/fields.yml" index := beat - tmpl, err := New(version.GetDefaultVersion(), client.GetVersion(), index, TemplateSettings{}) + tmpl, err := New(version.GetDefaultVersion(), index, client.GetVersion(), TemplateConfig{}) assert.NoError(t, err) content, err := tmpl.Load(fieldsPath) assert.NoError(t, err) @@ -178,7 +178,10 @@ func TestTemplateSettings(t *testing.T) { "enabled": false, }, } - tmpl, err := New(version.GetDefaultVersion(), client.GetVersion(), "testbeat", settings) + config := TemplateConfig{ + Settings: settings, + } + tmpl, err := New(version.GetDefaultVersion(), "testbeat", client.GetVersion(), config) assert.NoError(t, err) content, err := tmpl.Load(fieldsPath) assert.NoError(t, err) @@ -335,7 +338,7 @@ func TestTemplateWithData(t *testing.T) { // Setup ES client := elasticsearch.GetTestingElasticsearch(t) - tmpl, err := New(version.GetDefaultVersion(), client.GetVersion(), "testindex", TemplateSettings{}) + tmpl, err := New(version.GetDefaultVersion(), "testindex", client.GetVersion(), TemplateConfig{}) assert.NoError(t, err) content, err := tmpl.Load(fieldsPath) assert.NoError(t, err) diff --git a/libbeat/template/template.go b/libbeat/template/template.go index 1bdc1904e42..97995d69a8e 100644 --- a/libbeat/template/template.go +++ b/libbeat/template/template.go @@ -2,8 +2,11 @@ package template import ( "fmt" + "time" + "github.com/elastic/beats/libbeat/beat" "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/common/fmtstr" "github.com/elastic/go-ucfg/yaml" ) @@ -17,19 +20,58 @@ var ( ) type Template struct { - index string + name string + pattern string beatVersion common.Version esVersion common.Version settings TemplateSettings } // New creates a new template instance -func New(beatVersion string, esVersion string, index string, settings TemplateSettings) (*Template, error) { +func New(beatVersion string, beatName string, esVersion string, config TemplateConfig) (*Template, error) { bV, err := common.NewVersion(beatVersion) if err != nil { return nil, err } + name := config.Name + if name == "" { + name = fmt.Sprintf("%s-%s", beatName, bV.String()) + } + + pattern := config.Pattern + if pattern == "" { + pattern = name + "-*" + } + + event := &beat.Event{ + Fields: common.MapStr{ + "beat": common.MapStr{ + "name": beatName, + "version": bV.String(), + }, + }, + Timestamp: time.Now(), + } + + nameFormatter, err := fmtstr.CompileEvent(name) + if err != nil { + return nil, err + } + name, err = nameFormatter.Run(event) + if err != nil { + return nil, err + } + + patternFormatter, err := fmtstr.CompileEvent(pattern) + if err != nil { + return nil, err + } + pattern, err = patternFormatter.Run(event) + if err != nil { + return nil, err + } + // In case no esVersion is set, it is assumed the same as beat version if esVersion == "" { esVersion = beatVersion @@ -41,10 +83,11 @@ func New(beatVersion string, esVersion string, index string, settings TemplateSe } return &Template{ - index: index, + pattern: pattern, + name: name, beatVersion: *bV, esVersion: *esV, - settings: settings, + settings: config.Settings, }, nil } @@ -62,9 +105,14 @@ func (t *Template) Load(file string) (common.MapStr, error) { return output, nil } -// GetName returns the name of the template which is {index}-{version} +// GetName returns the name of the template func (t *Template) GetName() string { - return fmt.Sprintf("%s-%s", t.index, t.beatVersion.String()) + return t.name +} + +// GetPattern returns the pattern of the template +func (t *Template) GetPattern() string { + return t.pattern } // generate generates the full template @@ -130,9 +178,9 @@ func (t *Template) generate(properties common.MapStr, dynamicTemplates []common. // ES 6 moved from template to index_patterns: https://github.com/elastic/elasticsearch/pull/21009 if t.esVersion.Major >= 6 { - basicStructure.Put("index_patterns", []string{t.GetName() + "-*"}) + basicStructure.Put("index_patterns", []string{t.GetPattern()}) } else { - basicStructure.Put("template", t.GetName()+"-*") + basicStructure.Put("template", t.GetPattern()) } if t.esVersion.IsMajor(2) { diff --git a/libbeat/tests/system/beat/beat.py b/libbeat/tests/system/beat/beat.py index f1a99233bd9..988653d60ca 100644 --- a/libbeat/tests/system/beat/beat.py +++ b/libbeat/tests/system/beat/beat.py @@ -11,6 +11,7 @@ import yaml from datetime import datetime, timedelta + BEAT_REQUIRED_FIELDS = ["@timestamp", "beat.name", "beat.hostname", "beat.version"] @@ -168,7 +169,7 @@ def start_beat(self, "-test.coverprofile", os.path.join(self.working_dir, "coverage.cov"), "-path.home", os.path.normpath(self.working_dir), - "-c", os.path.join(self.working_dir, config) + "-c", os.path.join(self.working_dir, config), ] if logging_args: @@ -262,6 +263,11 @@ def setUp(self): shutil.rmtree(self.working_dir) os.makedirs(self.working_dir) + fields_yml = os.path.join(self.beat_path, "fields.yml") + # Only add it if it exists + if os.path.isfile(fields_yml): + shutil.copyfile(fields_yml, os.path.join(self.working_dir, "fields.yml")) + try: # update the last_run link if os.path.islink(self.build_path + "last_run"): diff --git a/libbeat/tests/system/config/mockbeat.yml.j2 b/libbeat/tests/system/config/mockbeat.yml.j2 index cd4081d33eb..03909dfb3ac 100644 --- a/libbeat/tests/system/config/mockbeat.yml.j2 +++ b/libbeat/tests/system/config/mockbeat.yml.j2 @@ -65,7 +65,8 @@ setup.template: settings: index.number_of_shards: 1 index.codec: best_compression - + name: {{ es_template_name }} + pattern: {{ es_template_pattern }} #================================ Logging ===================================== {% if metrics_period -%} diff --git a/libbeat/tests/system/test_template.py b/libbeat/tests/system/test_template.py new file mode 100644 index 00000000000..499f36ab3e4 --- /dev/null +++ b/libbeat/tests/system/test_template.py @@ -0,0 +1,74 @@ +from base import BaseTest + + +class Test(BaseTest): + + def test_index_modified(self): + """ + Test that beat stops in case elasticsearch index is modified and pattern not + """ + self.render_config_template( + elasticsearch={"index": "test"}, + ) + + exit_code = self.run_beat() + + assert exit_code == 1 + assert self.log_contains( + "setup.template.name and setup.template.pattern have to be set if index name is modified.") is True + + def test_index_not_modified(self): + """ + Test that beat starts running if elasticsearch output is set + """ + self.render_config_template( + elasticsearch={"hosts": "localhost:9200"}, + ) + + proc = self.start_beat() + self.wait_until(lambda: self.log_contains("mockbeat start running.")) + proc.check_kill_and_wait() + + def test_index_modified_no_pattern(self): + """ + Test that beat stops in case elasticsearch index is modified and pattern not + """ + self.render_config_template( + elasticsearch={"index": "test"}, + es_template_name="test", + ) + + exit_code = self.run_beat() + + assert exit_code == 1 + assert self.log_contains( + "setup.template.name and setup.template.pattern have to be set if index name is modified.") is True + + def test_index_modified_no_name(self): + """ + Test that beat stops in case elasticsearch index is modified and name not + """ + self.render_config_template( + elasticsearch={"index": "test"}, + es_template_pattern="test", + ) + + exit_code = self.run_beat() + + assert exit_code == 1 + assert self.log_contains( + "setup.template.name and setup.template.pattern have to be set if index name is modified.") is True + + def test_index_with_pattern_name(self): + """ + Test that beat starts running if elasticsearch output with modified index and pattern and name are set + """ + self.render_config_template( + elasticsearch={"hosts": "localhost:9200"}, + es_template_name="test", + es_template_pattern="test-*", + ) + + proc = self.start_beat() + self.wait_until(lambda: self.log_contains("mockbeat start running.")) + proc.check_kill_and_wait() diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 482c60539c0..ebde2b5b23b 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -588,6 +588,7 @@ output.elasticsearch: # Optional index name. The default is "metricbeat" plus date # and generates [metricbeat-]YYYY.MM.DD keys. + # In case you modify this pattern you must update setup.template.name and setup.template.pattern accordingly. #index: "metricbeat-%{[beat.version]}-%{+yyyy.MM.dd}" # Optional ingest node pipeline. By default no pipeline will be used. @@ -1045,10 +1046,14 @@ output.elasticsearch: # Set to false to disable template loading. #setup.template.enabled: true -# Template name. By default the template name is metricbeat. -# The version of the beat will always be appended to the given name -# so the final name is metricbeat-%{[beat.version]}. -#setup.template.name: "metricbeat" +# Template name. By default the template name is "metricbeat-%{[beat.version]}" +# The template name and pattern has to be set in case the elasticsearch index pattern is modified. +#setup.template.name: "metricbeat-%{[beat.version]}" + +# Template patttern. By default the template patter is "-%{[beat.version]}-*" to apply to the default index settings. +# The first part is the version of the beat and then -* is used to match all daily indicies. +# The template name and pattern has to be set in case the elasticsearch index pattern is modified. +#setup.template.pattern: "metricbeat-%{[beat.version]}-*" # Path to fields.yml file to generate the template #setup.template.fields: "${path.config}/fields.yml" diff --git a/packetbeat/packetbeat.reference.yml b/packetbeat/packetbeat.reference.yml index 58e72b6228c..e72d7386f8f 100644 --- a/packetbeat/packetbeat.reference.yml +++ b/packetbeat/packetbeat.reference.yml @@ -596,6 +596,7 @@ output.elasticsearch: # Optional index name. The default is "packetbeat" plus date # and generates [packetbeat-]YYYY.MM.DD keys. + # In case you modify this pattern you must update setup.template.name and setup.template.pattern accordingly. #index: "packetbeat-%{[beat.version]}-%{+yyyy.MM.dd}" # Optional ingest node pipeline. By default no pipeline will be used. @@ -1053,10 +1054,14 @@ output.elasticsearch: # Set to false to disable template loading. #setup.template.enabled: true -# Template name. By default the template name is packetbeat. -# The version of the beat will always be appended to the given name -# so the final name is packetbeat-%{[beat.version]}. -#setup.template.name: "packetbeat" +# Template name. By default the template name is "packetbeat-%{[beat.version]}" +# The template name and pattern has to be set in case the elasticsearch index pattern is modified. +#setup.template.name: "packetbeat-%{[beat.version]}" + +# Template patttern. By default the template patter is "-%{[beat.version]}-*" to apply to the default index settings. +# The first part is the version of the beat and then -* is used to match all daily indicies. +# The template name and pattern has to be set in case the elasticsearch index pattern is modified. +#setup.template.pattern: "packetbeat-%{[beat.version]}-*" # Path to fields.yml file to generate the template #setup.template.fields: "${path.config}/fields.yml" diff --git a/winlogbeat/winlogbeat.reference.yml b/winlogbeat/winlogbeat.reference.yml index 668b3deac40..f38bd235a8f 100644 --- a/winlogbeat/winlogbeat.reference.yml +++ b/winlogbeat/winlogbeat.reference.yml @@ -173,6 +173,7 @@ output.elasticsearch: # Optional index name. The default is "winlogbeat" plus date # and generates [winlogbeat-]YYYY.MM.DD keys. + # In case you modify this pattern you must update setup.template.name and setup.template.pattern accordingly. #index: "winlogbeat-%{[beat.version]}-%{+yyyy.MM.dd}" # Optional ingest node pipeline. By default no pipeline will be used. @@ -630,10 +631,14 @@ output.elasticsearch: # Set to false to disable template loading. #setup.template.enabled: true -# Template name. By default the template name is winlogbeat. -# The version of the beat will always be appended to the given name -# so the final name is winlogbeat-%{[beat.version]}. -#setup.template.name: "winlogbeat" +# Template name. By default the template name is "winlogbeat-%{[beat.version]}" +# The template name and pattern has to be set in case the elasticsearch index pattern is modified. +#setup.template.name: "winlogbeat-%{[beat.version]}" + +# Template patttern. By default the template patter is "-%{[beat.version]}-*" to apply to the default index settings. +# The first part is the version of the beat and then -* is used to match all daily indicies. +# The template name and pattern has to be set in case the elasticsearch index pattern is modified. +#setup.template.pattern: "winlogbeat-%{[beat.version]}-*" # Path to fields.yml file to generate the template #setup.template.fields: "${path.config}/fields.yml"