diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 48d186ef1c89..8aed2361ee7c 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -21,6 +21,7 @@ https://github.com/elastic/beats/compare/v5.1.1...master[Check the HEAD diff] - Change vendor manager from glide to govendor. {pull}3851[3851] - Rename `error` field to `error.message`. {pull}3987[3987] - Change `dashboards.*` config options to `setup.dashboards.*`. {pull}3921[3921] +- Change `outputs.elasticsearch.template.* to `setup.template.*` {pull}4080[4080] *Filebeat* - Always use absolute path for event and registry. This can lead to issues when relative paths were used before. {pull}3328[3328] diff --git a/filebeat/docs/reference/configuration/filebeat-options.asciidoc b/filebeat/docs/reference/configuration/filebeat-options.asciidoc index 9af1dde0e2ad..c9860a76b38b 100644 --- a/filebeat/docs/reference/configuration/filebeat-options.asciidoc +++ b/filebeat/docs/reference/configuration/filebeat-options.asciidoc @@ -596,7 +596,7 @@ include::../../../../libbeat/docs/outputconfig.asciidoc[] include::../../../../libbeat/docs/shared-path-config.asciidoc[] -include::../../../../libbeat/docs/dashboardsconfig.asciidoc[] +include::../../../../libbeat/docs/setup-config.asciidoc[] include::../../../../libbeat/docs/loggingconfig.asciidoc[] diff --git a/filebeat/filebeat.full.yml b/filebeat/filebeat.full.yml index 6027f6c75684..14105fa48132 100644 --- a/filebeat/filebeat.full.yml +++ b/filebeat/filebeat.full.yml @@ -559,24 +559,6 @@ output.elasticsearch: # requests are made. #flush_interval: 1s - # A template is used to set the mapping in Elasticsearch - # By default template loading is enabled and the template is loaded. - # These settings can be adjusted to load your own template or overwrite existing ones. - - # Set to false to disable template loading. - #template.enabled: true - - # Template name. By default the template name is filebeat. - # The version of the beat will always be appended to the given name - # so the final name is filebeat-%{[beat.version]}. - #template.name: "filebeat" - - # Path to fields.yml file to generate the template - #template.fields: "${path.config}/fields.yml" - - # Overwrite existing template - #template.overwrite: false - # Use SSL settings for HTTPS. Default is true. #ssl.enabled: true @@ -1003,6 +985,27 @@ output.elasticsearch: # dashboards and index pattern. Example: testbeat-* #setup.dashboards.index: +#============================== Template ===================================== + +# A template is used to set the mapping in Elasticsearch +# By default template loading is enabled and the template is loaded. +# These settings can be adjusted to load your own template or overwrite existing ones. + +# Set to false to disable template loading. +#setup.template.enabled: true + +# Template name. By default the template name is filebeat. +# The version of the beat will always be appended to the given name +# so the final name is filebeat-%{[beat.version]}. +#setup.template.name: "filebeat" + +# Path to fields.yml file to generate the template +#setup.template.fields: "${path.config}/fields.yml" + +# Overwrite existing template +#setup.template.overwrite: false + + #================================ HTTP Endpoint ====================================== # Each beat can expose internal data points through a http endpoint. For security # reason the endpoint is disabled by default. This feature is currently in beta. diff --git a/heartbeat/docs/reference/configuration/heartbeat-options.asciidoc b/heartbeat/docs/reference/configuration/heartbeat-options.asciidoc index f283e6dc5ec6..1613be56d927 100644 --- a/heartbeat/docs/reference/configuration/heartbeat-options.asciidoc +++ b/heartbeat/docs/reference/configuration/heartbeat-options.asciidoc @@ -5,7 +5,7 @@ The `heartbeat` section of the +heartbeat.yml+ config file specifies the list of `monitors` that Heartbeat uses to check your remote hosts to determine if they are available. Each `monitor` item begins with a dash (-) and specifies the type of monitor to use, the hosts to check, and other settings -that control Heartbeat behavior. +that control Heartbeat behavior. The following example configures three monitors, `icmp`, `tcp`, and `http`, and demonstrates how to use TCP Echo and HTTP response verification: @@ -35,7 +35,7 @@ heartbeat.scheduler: You can specify the following options in the `monitors` section of the +heartbeat.yml+ config file. These options are the same for all monitors. Each monitor type has additional configuration options that are specific to that -monitor type. +monitor type. [[monitor-type]] @@ -51,7 +51,7 @@ receiving a custom payload. See <>. expected response. See <>. The `tcp` and `http` monitor types both support SSL/TLS and some proxy -settings. +settings. [[monitor-name]] @@ -133,7 +133,7 @@ you can specify settings in the JSON file that overwrite the settings in the main config. In this way, the configuration that you specify for the monitor in the main Heartbeat config file acts like a default config that you can live-reconfigure by specifying additional configurations in the external -JSON file. +JSON file. Example configuration: @@ -148,7 +148,7 @@ heartbeat.monitors: interval: 5s ------------------------------------------------------------------------------- -*`path`*:: Specifies the path to the JSON file to check for updates. +*`path`*:: Specifies the path to the JSON file to check for updates. *`interval`*:: Specifies how often Heartbeat checks the file for changes. To reconfigure the settings specified in the example config, you could define @@ -156,13 +156,13 @@ the following JSON objects in `dynamic.json`: [source, json] ------------------------------------------------------------------------------- -{"hosts": ["myhost:1234"], "schedule": "*/15 * * * * * *"} <1> +{"hosts": ["myhost:1234"], "schedule": "*/15 * * * * * *"} <1> {"hosts": ["tls://otherhost:479"], "ssl.certificate_authorities": ["path/to/ca/file.pem"]} <2> ------------------------------------------------------------------------------- <1> Upon detecting the changes, Heartbeat stops the old monitor and then restarts it with a schedule of 15 seconds between checks. <2> Heartbeat starts a new monitor that uses a TLS-based connection with a -custom CA certificate. +custom CA certificate. [[monitor-icmp-options]] ==== ICMP Options @@ -222,7 +222,7 @@ Example configuration: [source,yaml] ------------------------------------------------------------------------------- -- type: tcp +- type: tcp schedule: '@every 5s' hosts: ["myhost"] ports: [80, 9200, 5044] @@ -242,7 +242,7 @@ Example configuration: [source,yaml] ------------------------------------------------------------------------------- -- type: tcp +- type: tcp schedule: '@every 5s' hosts: ["myhost"] ports: [7] @@ -255,7 +255,7 @@ Example configuration: ===== proxy_url The URL of the SOCKS5 proxy to use when connecting to the server. The value -must be a URL with a scheme of socks5://. +must be a URL with a scheme of socks5://. If the SOCKS5 proxy server requires client authentication, then a username and password can be embedded in the URL as shown in the example. @@ -289,7 +289,7 @@ Example configuration: [source,yaml] ------------------------------------------------------------------------------- -- type: tcp +- type: tcp schedule: '@every 5s' hosts: ["myhost"] ports: [80, 9200, 5044] @@ -315,7 +315,7 @@ Example configuration: [source,yaml] ------------------------------------------------------------------------------- -- type: http +- type: http schedule: '@every 5s' urls: ["http://myhost:80"] ------------------------------------------------------------------------------- @@ -331,11 +331,11 @@ environment variable is used. ===== username The username for authenticating with the server. The credentials are passed -with the request. This setting is optional. +with the request. This setting is optional. You need to specify credentials when your `check.response` settings require it. For example, you can check for a 403 response (`check.response.status: 403`) -without setting credentials. +without setting credentials. [[monitor-http-password]] ===== password @@ -346,32 +346,32 @@ The password for authenticating with the server. This setting is optional. ===== ssl The TLS/SSL connection settings for use with the HTTPS endpoint. If you don't -specify settings, the system defaults are used. +specify settings, the system defaults are used. Example configuration: [source,yaml] ------------------------------------------------------------------------------- -- type: http +- type: http schedule: '@every 5s' urls: ["https://myhost:80"] ssl: certificate_authorities: ['/etc/ca.crt'] - supported_protocols: ["TLSv1.0", "TLSv1.1", "TLSv1.2"] + supported_protocols: ["TLSv1.0", "TLSv1.1", "TLSv1.2"] ------------------------------------------------------------------------------- [[monitor-http-check]] ===== check -An optional `request` to send to the remote host and the expected `response`. +An optional `request` to send to the remote host and the expected `response`. Example configuration: [source,yaml] ------------------------------------------------------------------------------- -- type: http +- type: http schedule: '@every 5s' urls: ["http://myhost:80"] check.request.method: HEAD @@ -389,8 +389,8 @@ Under `check.request`, specify these options: Under `check.response`, specify these options: *`status`*:: The expected status code. If this setting is not configured or -it's set to 0, any status code other than 404 is accepted. -*`headers`*:: The required response headers. +it's set to 0, any status code other than 404 is accepted. +*`headers`*:: The required response headers. *`body`*:: The required response body content. The following configuration shows how to check the response when the body @@ -398,7 +398,7 @@ contains JSON: [source,yaml] ------------------------------------------------------------------------------- -- type: http +- type: http schedule: '@every 5s' urls: ["https://myhost:80"] check.request: @@ -417,7 +417,7 @@ check.response: You specify options under `scheduler` to control the behavior of the task scheduler. -Example configuration: +Example configuration: [source,yaml] ------------------------------------------------------------------------------- @@ -452,7 +452,7 @@ include::../../../../libbeat/docs/outputconfig.asciidoc[] include::../../../../libbeat/docs/shared-path-config.asciidoc[] -include::../../../../libbeat/docs/dashboardsconfig.asciidoc[] +include::../../../../libbeat/docs/setup-config.asciidoc[] include::../../../../libbeat/docs/loggingconfig.asciidoc[] diff --git a/heartbeat/heartbeat.full.yml b/heartbeat/heartbeat.full.yml index bc07003d2939..6b5c1e86ca6c 100644 --- a/heartbeat/heartbeat.full.yml +++ b/heartbeat/heartbeat.full.yml @@ -339,24 +339,6 @@ output.elasticsearch: # requests are made. #flush_interval: 1s - # A template is used to set the mapping in Elasticsearch - # By default template loading is enabled and the template is loaded. - # These settings can be adjusted to load your own template or overwrite existing ones. - - # Set to false to disable template loading. - #template.enabled: true - - # Template name. By default the template name is heartbeat. - # The version of the beat will always be appended to the given name - # so the final name is heartbeat-%{[beat.version]}. - #template.name: "heartbeat" - - # Path to fields.yml file to generate the template - #template.fields: "${path.config}/fields.yml" - - # Overwrite existing template - #template.overwrite: false - # Use SSL settings for HTTPS. Default is true. #ssl.enabled: true @@ -783,6 +765,27 @@ output.elasticsearch: # dashboards and index pattern. Example: testbeat-* #setup.dashboards.index: +#============================== Template ===================================== + +# A template is used to set the mapping in Elasticsearch +# By default template loading is enabled and the template is loaded. +# These settings can be adjusted to load your own template or overwrite existing ones. + +# Set to false to disable template loading. +#setup.template.enabled: true + +# Template name. By default the template name is heartbeat. +# The version of the beat will always be appended to the given name +# so the final name is heartbeat-%{[beat.version]}. +#setup.template.name: "heartbeat" + +# Path to fields.yml file to generate the template +#setup.template.fields: "${path.config}/fields.yml" + +# Overwrite existing template +#setup.template.overwrite: false + + #================================ HTTP Endpoint ====================================== # Each beat can expose internal data points through a http endpoint. For security # reason the endpoint is disabled by default. This feature is currently in beta. diff --git a/libbeat/_meta/config.full.yml b/libbeat/_meta/config.full.yml index fcc06ab8ffad..f2e2f40166b1 100644 --- a/libbeat/_meta/config.full.yml +++ b/libbeat/_meta/config.full.yml @@ -141,24 +141,6 @@ output.elasticsearch: # requests are made. #flush_interval: 1s - # A template is used to set the mapping in Elasticsearch - # By default template loading is enabled and the template is loaded. - # These settings can be adjusted to load your own template or overwrite existing ones. - - # Set to false to disable template loading. - #template.enabled: true - - # Template name. By default the template name is beatname. - # The version of the beat will always be appended to the given name - # so the final name is beatname-%{[beat.version]}. - #template.name: "beatname" - - # Path to fields.yml file to generate the template - #template.fields: "${path.config}/fields.yml" - - # Overwrite existing template - #template.overwrite: false - # Use SSL settings for HTTPS. Default is true. #ssl.enabled: true @@ -585,6 +567,27 @@ output.elasticsearch: # dashboards and index pattern. Example: testbeat-* #setup.dashboards.index: +#============================== Template ===================================== + +# A template is used to set the mapping in Elasticsearch +# By default template loading is enabled and the template is loaded. +# These settings can be adjusted to load your own template or overwrite existing ones. + +# Set to false to disable template loading. +#setup.template.enabled: true + +# Template name. By default the template name is beatname. +# The version of the beat will always be appended to the given name +# so the final name is beatname-%{[beat.version]}. +#setup.template.name: "beatname" + +# Path to fields.yml file to generate the template +#setup.template.fields: "${path.config}/fields.yml" + +# Overwrite existing template +#setup.template.overwrite: false + + #================================ HTTP Endpoint ====================================== # Each beat can expose internal data points through a http endpoint. For security # reason the endpoint is disabled by default. This feature is currently in beta. diff --git a/libbeat/beat/beat.go b/libbeat/beat/beat.go index e9151c97ed7b..51b6838df459 100644 --- a/libbeat/beat/beat.go +++ b/libbeat/beat/beat.go @@ -58,6 +58,7 @@ import ( "github.com/elastic/beats/libbeat/processors" "github.com/elastic/beats/libbeat/publisher" svc "github.com/elastic/beats/libbeat/service" + "github.com/elastic/beats/libbeat/template" "github.com/elastic/beats/libbeat/version" // Register default processors. @@ -113,6 +114,7 @@ type BeatConfig struct { Processors processors.PluginConfig `config:"processors"` Path paths.Path `config:"path"` Dashboards *common.Config `config:"setup.dashboards"` + Template *common.Config `config:"setup.template"` Http *common.Config `config:"http"` } @@ -215,6 +217,11 @@ func (b *Beat) launch(bt Creator) error { return fmt.Errorf("error initializing processors: %v", err) } + err = b.registerTemplateLoading() + if err != nil { + return err + } + debugf("Initializing output plugins") publisher, err := publisher.New(b.Info, b.Config.Output, b.Config.Shipper, processors) if err != nil { @@ -447,6 +454,54 @@ func (b *Beat) loadDashboards() error { return nil } +// registerTemplateLoading registers the loading of the template as a callback with +// the elasticsearch output. It is important the the registration happens before +// the publisher is created. +func (b *Beat) registerTemplateLoading() error { + if *setup { + // -setup implies template.enabled=true + if b.Config.Template == nil { + b.Config.Template = common.NewConfig() + } + err := b.Config.Template.SetBool("enabled", -1, true) + if err != nil { + return fmt.Errorf("Error setting template.enabled=true: %v", err) + } + } + + esConfig := b.Config.Output["elasticsearch"] + + // Loads template by default if esOutput is enabled + if (b.Config.Template == nil && esConfig.Enabled()) || (b.Config.Template != nil && b.Config.Template.Enabled()) { + if esConfig == nil || !esConfig.Enabled() { + return fmt.Errorf("Template loading requested but the Elasticsearch output is not configured/enabled") + } + + // load template through callback to make sure it is also loaded + // on reconnecting + callback := func(esClient *elasticsearch.Client) error { + + if b.Config.Template == nil { + b.Config.Template = common.NewConfig() + } + + loader, err := template.NewLoader(b.Config.Template, esClient, b.Info) + if err != nil { + return fmt.Errorf("Error loading elasticsearch template: %v", err) + } + + loader.Load() + + logp.Info("ES template successfully loaded.") + return nil + } + + elasticsearch.RegisterConnectCallback(callback) + } + + return nil +} + // handleError handles the given error by logging it and then returning the // error. If the err is nil or is a GracefulExit error then the method will // return nil without logging anything. diff --git a/libbeat/beat/setup.go b/libbeat/beat/setup.go new file mode 100644 index 000000000000..061e9091270b --- /dev/null +++ b/libbeat/beat/setup.go @@ -0,0 +1,10 @@ +package beat + +type TemplateConfig struct { + Enabled bool `config:"enabled"` + Name string `config:"name"` + Fields string `config:"fields"` + Overwrite bool `config:"overwrite"` + OutputToFile string `config:"output_to_file"` + Settings map[string]string `config:"settings"` +} diff --git a/libbeat/docs/outputconfig.asciidoc b/libbeat/docs/outputconfig.asciidoc index 7d9eef9f90ef..1a8b2d8aad84 100644 --- a/libbeat/docs/outputconfig.asciidoc +++ b/libbeat/docs/outputconfig.asciidoc @@ -240,59 +240,6 @@ output.elasticsearch: type: "normal" ------------------------------------------------------------------------------ -===== template - -The http://www.elastic.co/guide/en/elasticsearch/reference/current/indices-templates.html[index -template] to use for setting mappings in Elasticsearch. By default, template loading is -enabled. - -You can adjust the following settings to load your own template or overwrite an existing one: - -*`enabled`*:: Set to false to disable template loading. If set this to false, -you must <>. - -*`name`*:: The name of the template. The default is +{beatname_lc}+. - -*`path`*:: The path to the template file. The default is +fields.yml+. If a relative -path is set, it is considered relative to the config path. See the <> section for -details. - -*`overwrite`*:: A boolean that specifies whether to overwrite the existing template. The default -is false. - -For example: - -["source","yaml",subs="attributes,callouts"] ----------------------------------------------------------------------- -output.elasticsearch: - hosts: ["localhost:9200"] - template.name: "{beatname_lc}" - template.fields: "fields.yml" - template.overwrite: false ----------------------------------------------------------------------- - -===== template.versions - -In the default configuration, {beatname_uc} automatically checks the -Elasticsearch version and loads the recommended template file for the particular -version. This behaviour can be controlled from the following options: - -*`2x.path`*:: The path to the template file to load for -Elasticsearch versions 2.x.y. The default is +{beatname_lc}.template-es2x.json+. - -*`2x.enabled`*:: If set to +false+, the +2x.path+ option is ignored and the -default template is loaded regardless of the Elasticsearch version. - -For example: - -["source","yaml",subs="attributes,callouts"] ----------------------------------------------------------------------- -output.elasticsearch: - hosts: ["localhost:9200"] - template.fields: "{beatname_lc}.template.json" - template.overwrite: false ----------------------------------------------------------------------- - ===== max_retries The number of times to retry publishing an event after a publishing failure. @@ -346,7 +293,7 @@ See <> for more information. *Prerequisite:* To use Logstash as an output, you must {libbeat}/logstash-installation.html#logstash-setup[install and configure] the Beats input -plugin for Logstash. +plugin for Logstash. The Logstash output sends the events directly to Logstash by using the lumberjack protocol, which runs over TCP. Logstash allows for additional processing and routing of @@ -377,13 +324,13 @@ use in Logstash for indexing and filtering: ------------------------------------------------------------------------------ <1> {beatname_uc} uses the `@metadata` field to send metadata to Logstash. The contents of the `@metadata` field only exist in Logstash and are not part of any -events sent from Logstash. See the +events sent from Logstash. See the {logstashdoc}/event-dependent-configuration.html#metadata[Logstash documentation] for more about the `@metadata` field. <2> The default is {beatname_lc}. To change this value, set the <> option in the {beatname_uc} config file. -<3> The value of `type` varies depending on the event type. - +<3> The value of `type` varies depending on the event type. + You can access this metadata from within the Logstash config file to set values dynamically based on the contents of the metadata. diff --git a/libbeat/docs/setup-config.asciidoc b/libbeat/docs/setup-config.asciidoc new file mode 100644 index 000000000000..2e8b942c2e3c --- /dev/null +++ b/libbeat/docs/setup-config.asciidoc @@ -0,0 +1,4 @@ + + +include::./dashboardsconfig.asciidoc[] +include::./template-config.asciidoc[] diff --git a/libbeat/docs/template-config.asciidoc b/libbeat/docs/template-config.asciidoc new file mode 100644 index 000000000000..c046922b9e4f --- /dev/null +++ b/libbeat/docs/template-config.asciidoc @@ -0,0 +1,30 @@ +[[configuration-template]] + +=== Template + +The http://www.elastic.co/guide/en/elasticsearch/reference/current/indices-templates.html[index +template] to use for setting mappings in Elasticsearch. By default, template loading is +enabled. + +You can adjust the following settings to load your own template or overwrite an existing one: + +*`enabled`*:: Set to false to disable template loading. If set this to false, +you must <>. + +*`name`*:: The name of the template. The default is +{beatname_lc}+. + +*`path`*:: The path to the template file. The default is +fields.yml+. If a relative +path is set, it is considered relative to the config path. See the <> section for +details. + +*`overwrite`*:: A boolean that specifies whether to overwrite the existing template. The default +is false. + +For example: + +["source","yaml",subs="attributes,callouts"] +---------------------------------------------------------------------- +setup.template.name: "{beatname_lc}" +setup.template.fields: "fields.yml" +setup.template.overwrite: false +---------------------------------------------------------------------- diff --git a/libbeat/outputs/elasticsearch/client.go b/libbeat/outputs/elasticsearch/client.go index f3ec5f8316f3..b56621ce3bc9 100644 --- a/libbeat/outputs/elasticsearch/client.go +++ b/libbeat/outputs/elasticsearch/client.go @@ -590,21 +590,6 @@ func (client *Client) PublishEvent(data outputs.Data) error { return nil } -// LoadTemplate loads a template into Elasticsearch overwriting the existing -// template if it exists. If you wish to not overwrite an existing template -// then use CheckTemplate prior to calling this method. -func (client *Client) LoadTemplate(templateName string, template map[string]interface{}) error { - - logp.Info("load template: %s", templateName) - path := "/_template/" + templateName - body, err := client.LoadJSON(path, template) - if err != nil { - return fmt.Errorf("couldn't load template: %v. Response body: %s", err, body) - } - logp.Info("Elasticsearch template with name '%s' loaded", templateName) - return nil -} - // LoadJSON creates a PUT request based on a JSON document. func (client *Client) LoadJSON(path string, json map[string]interface{}) ([]byte, error) { status, body, err := client.Request("PUT", path, "", nil, json) @@ -618,17 +603,9 @@ func (client *Client) LoadJSON(path string, json map[string]interface{}) ([]byte return body, nil } -// CheckTemplate checks if a given template already exist. It returns true if -// and only if Elasticsearch returns with HTTP status code 200. -func (client *Client) CheckTemplate(templateName string) bool { - - status, _, _ := client.Request("HEAD", "/_template/"+templateName, "", nil, nil) - - if status != 200 { - return false - } - - return true +// GetVersion returns the elasticsearch version the client is connected to +func (client *Client) GetVersion() string { + return client.Connection.version } // Connect connects the client. diff --git a/libbeat/outputs/elasticsearch/client_integration_test.go b/libbeat/outputs/elasticsearch/client_integration_test.go index 1b3cf27bcf2e..ea186ba94e60 100644 --- a/libbeat/outputs/elasticsearch/client_integration_test.go +++ b/libbeat/outputs/elasticsearch/client_integration_test.go @@ -4,7 +4,6 @@ package elasticsearch import ( "os" - "path/filepath" "strings" "testing" "time" @@ -12,8 +11,6 @@ import ( "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/libbeat/outputs" - "github.com/elastic/beats/libbeat/template" - "github.com/elastic/beats/libbeat/version" "github.com/stretchr/testify/assert" ) @@ -25,176 +22,6 @@ func TestClientConnect(t *testing.T) { assert.NoError(t, err) } -func TestCheckTemplate(t *testing.T) { - - client := GetTestingElasticsearch() - err := client.Connect(5 * time.Second) - assert.Nil(t, err) - - // Check for non existent template - assert.False(t, client.CheckTemplate("libbeat-notexists")) -} - -func TestLoadTemplate(t *testing.T) { - - // Setup ES - client := GetTestingElasticsearch() - err := client.Connect(5 * time.Second) - assert.Nil(t, err) - - // Load template - absPath, err := filepath.Abs("../../") - assert.NotNil(t, absPath) - assert.Nil(t, err) - - fieldsPath := absPath + "/fields.yml" - index := "testbeat" - - tmpl, err := template.New(version.GetDefaultVersion(), client.Connection.version, index) - assert.NoError(t, err) - content, err := tmpl.Load(fieldsPath) - assert.NoError(t, err) - - // Load template - err = client.LoadTemplate(tmpl.GetName(), content) - assert.Nil(t, err) - - // Make sure template was loaded - assert.True(t, client.CheckTemplate(tmpl.GetName())) - - // Delete template again to clean up - client.Request("DELETE", "/_template/"+tmpl.GetName(), "", nil, nil) - - // Make sure it was removed - assert.False(t, client.CheckTemplate(tmpl.GetName())) - -} - -func TestLoadInvalidTemplate(t *testing.T) { - - // Invalid Template - template := map[string]interface{}{ - "json": "invalid", - } - - // Setup ES - client := GetTestingElasticsearch() - err := client.Connect(5 * time.Second) - assert.Nil(t, err) - - templateName := "invalidtemplate" - - // Try to load invalid template - err = client.LoadTemplate(templateName, template) - assert.Error(t, err) - - // Make sure template was not loaded - assert.False(t, client.CheckTemplate(templateName)) -} - -// Tests loading the templates for each beat -func TestLoadBeatsTemplate(t *testing.T) { - - beats := []string{ - "libbeat", - } - - for _, beat := range beats { - // Load template - absPath, err := filepath.Abs("../../../" + beat) - assert.NotNil(t, absPath) - assert.Nil(t, err) - - // Setup ES - client := GetTestingElasticsearch() - - err = client.Connect(5 * time.Second) - assert.Nil(t, err) - - fieldsPath := absPath + "/fields.yml" - index := beat - - tmpl, err := template.New(version.GetDefaultVersion(), client.Connection.version, index) - assert.NoError(t, err) - content, err := tmpl.Load(fieldsPath) - assert.NoError(t, err) - - // Load template - err = client.LoadTemplate(tmpl.GetName(), content) - assert.Nil(t, err) - - // Make sure template was loaded - assert.True(t, client.CheckTemplate(tmpl.GetName())) - - // Delete template again to clean up - client.Request("DELETE", "/_template/"+tmpl.GetName(), "", nil, nil) - - // Make sure it was removed - assert.False(t, client.CheckTemplate(tmpl.GetName())) - } -} - -// TestOutputLoadTemplate checks that the template is inserted before -// the first event is published. -func TestOutputLoadTemplate(t *testing.T) { - - client := GetTestingElasticsearch() - err := client.Connect(5 * time.Second) - if err != nil { - t.Fatal(err) - } - - templateName := "libbeat-" + version.GetDefaultVersion() - - // delete template if it exists - client.Request("DELETE", "/_template/"+templateName, "", nil, nil) - - // Make sure template is not yet there - assert.False(t, client.CheckTemplate(templateName)) - - templatePath := "../../fields.yml" - tPath, err := filepath.Abs(templatePath) - if err != nil { - t.Fatal(err) - } - - config := map[string]interface{}{ - "hosts": GetEsHost(), - "template": map[string]interface{}{ - "name": "libbeat", - "fields": tPath, - }, - } - - cfg, err := common.NewConfigFrom(config) - if err != nil { - t.Fatal(err) - } - - output, err := New(common.BeatInfo{Beat: "libbeat", Version: version.GetDefaultVersion()}, cfg) - if err != nil { - t.Fatal(err) - } - - event := outputs.Data{Event: common.MapStr{ - "@timestamp": common.Time(time.Now()), - "host": "test-host", - "type": "libbeat", - "message": "Test message from libbeat", - "beat": common.MapStr{ - "version": version.GetDefaultVersion(), - }, - }} - - err = output.PublishEvent(nil, outputs.Options{Guaranteed: true}, event) - if err != nil { - t.Fatal(err) - } - - // Guaranteed publish, so the template should be there - assert.True(t, client.CheckTemplate(templateName)) -} - func TestClientPublishEvent(t *testing.T) { index := "beat-int-pub-single-event" output, client := connectTestEs(t, map[string]interface{}{ diff --git a/libbeat/outputs/elasticsearch/config.go b/libbeat/outputs/elasticsearch/config.go index a3f12cb1ddaf..7eb4cdaabf16 100644 --- a/libbeat/outputs/elasticsearch/config.go +++ b/libbeat/outputs/elasticsearch/config.go @@ -19,26 +19,6 @@ type elasticsearchConfig struct { TLS *outputs.TLSConfig `config:"ssl"` MaxRetries int `config:"max_retries"` Timeout time.Duration `config:"timeout"` - Template Template `config:"template"` -} - -// Template contains the elasticsearch template. -type Template struct { - Enabled bool `config:"enabled"` - Name string `config:"name"` - Fields string `config:"fields"` - Overwrite bool `config:"overwrite"` -} - -// TemplateVersions contains the template versions. -type TemplateVersions struct { - Es2x TemplateVersion `config:"2x"` -} - -// TemplateVersion contains a template version. -type TemplateVersion struct { - Enabled bool `config:"enabled"` - Path string `config:"path"` } const ( @@ -58,10 +38,6 @@ var ( CompressionLevel: 0, TLS: nil, LoadBalance: true, - Template: Template{ - Enabled: true, - Fields: "fields.yml", - }, } ) diff --git a/libbeat/outputs/elasticsearch/output.go b/libbeat/outputs/elasticsearch/output.go index 38a25d60aefa..51071a600c09 100644 --- a/libbeat/outputs/elasticsearch/output.go +++ b/libbeat/outputs/elasticsearch/output.go @@ -4,7 +4,6 @@ import ( "errors" "fmt" "net/url" - "sync" "time" "github.com/elastic/beats/libbeat/common" @@ -15,8 +14,6 @@ import ( "github.com/elastic/beats/libbeat/outputs/mode/modeutil" "github.com/elastic/beats/libbeat/outputs/outil" "github.com/elastic/beats/libbeat/outputs/transport" - "github.com/elastic/beats/libbeat/paths" - "github.com/elastic/beats/libbeat/template" ) type elasticsearchOutput struct { @@ -26,8 +23,6 @@ type elasticsearchOutput struct { clients []mode.ProtocolClient mode mode.ConnectionMode - - templateMutex sync.Mutex } func init() { @@ -49,6 +44,14 @@ var ( ErrResponseRead = errors.New("bulk item status parse failed") ) +var connectCallbackRegistry connectCallback + +// RegisterConnectCallback registers a callback for the elasticsearch output +// The callback is called each time the client connects to elasticsearch. +func RegisterConnectCallback(callback connectCallback) { + connectCallbackRegistry = callback +} + // New instantiates a new output plugin instance publishing to elasticsearch. func New(beat common.BeatInfo, cfg *common.Config) (outputs.Outputer, error) { if !cfg.HasField("bulk_max_size") { @@ -223,48 +226,6 @@ func (out *elasticsearchOutput) init( return nil } -// loadTemplate checks if the index mapping template should be loaded -// In case the template is not already loaded or overwriting is enabled, the -// template is written to index -func (out *elasticsearchOutput) loadTemplate(config Template, client *Client) error { - out.templateMutex.Lock() - defer out.templateMutex.Unlock() - - logp.Info("Trying to load template for client: %s", client.Connection.URL) - - // Check if template already exist or should be overwritten - exists := client.CheckTemplate(config.Name) - if !exists || config.Overwrite { - - logp.Info("Loading template for elasticsearch version: %s", client.Connection.version) - - if config.Overwrite { - logp.Info("Existing template will be overwritten, as overwrite is enabled.") - } - - tmpl, err := template.New(out.beat.Version, client.Connection.version, config.Name) - if err != nil { - return fmt.Errorf("error creating template instance: %v", err) - } - - fieldsPath := paths.Resolve(paths.Config, config.Fields) - - output, err := tmpl.Load(fieldsPath) - if err != nil { - return fmt.Errorf("error creating template from file %s: %v", fieldsPath, err) - } - - err = client.LoadTemplate(tmpl.GetName(), output) - if err != nil { - return fmt.Errorf("could not load template: %v", err) - } - } else { - logp.Info("Template already exists and will not be overwritten.") - } - - return nil -} - func makeClientFactory( tls *transport.TLSConfig, config *elasticsearchConfig, @@ -292,19 +253,6 @@ func makeClientFactory( params = nil } - var onConnected connectCallback - // TODO: should we check if fields.yml exists? - if config.Template.Enabled { - // Set beat name as default if name not set - if config.Template.Name == "" { - config.Template.Name = out.beat.Beat - } - // define a callback to be called on connection - onConnected = func(client *Client) error { - return out.loadTemplate(config.Template, client) - } - } - return NewClient(ClientSettings{ URL: esURL, Index: out.index, @@ -317,7 +265,7 @@ func makeClientFactory( Headers: config.Headers, Timeout: config.Timeout, CompressionLevel: config.CompressionLevel, - }, onConnected) + }, connectCallbackRegistry) } } diff --git a/libbeat/outputs/elasticsearch/output_test.go b/libbeat/outputs/elasticsearch/output_test.go index d941c47fff23..f9c9323327a8 100644 --- a/libbeat/outputs/elasticsearch/output_test.go +++ b/libbeat/outputs/elasticsearch/output_test.go @@ -26,16 +26,15 @@ func createElasticsearchConnection(flushInterval int, bulkSize int) *elasticsear } config, _ := common.NewConfigFrom(map[string]interface{}{ - "hosts": []string{GetEsHost()}, - "port": esPort, - "username": os.Getenv("ES_USER"), - "password": os.Getenv("ES_PASS"), - "path": "", - "index": fmt.Sprintf("%v-%%{+yyyy.MM.dd}", index), - "protocol": "http", - "flush_interval": flushInterval, - "bulk_max_size": bulkSize, - "template.enabled": false, + "hosts": []string{GetEsHost()}, + "port": esPort, + "username": os.Getenv("ES_USER"), + "password": os.Getenv("ES_PASS"), + "path": "", + "index": fmt.Sprintf("%v-%%{+yyyy.MM.dd}", index), + "protocol": "http", + "flush_interval": flushInterval, + "bulk_max_size": bulkSize, }) output := &elasticsearchOutput{beat: common.BeatInfo{Beat: "test"}} diff --git a/libbeat/template/config.go b/libbeat/template/config.go new file mode 100644 index 000000000000..7be6f7702ab9 --- /dev/null +++ b/libbeat/template/config.go @@ -0,0 +1,17 @@ +package template + +type TemplateConfig struct { + Enabled bool `config:"enabled"` + Name string `config:"name"` + Fields string `config:"fields"` + Overwrite bool `config:"overwrite"` + OutputToFile string `config:"output_to_file"` + Settings map[string]string `config:"settings"` +} + +var ( + defaultConfig = TemplateConfig{ + Enabled: true, + Fields: "fields.yml", + } +) diff --git a/libbeat/template/load.go b/libbeat/template/load.go new file mode 100644 index 000000000000..d6ac02c1deae --- /dev/null +++ b/libbeat/template/load.go @@ -0,0 +1,109 @@ +package template + +import ( + "fmt" + + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/logp" + "github.com/elastic/beats/libbeat/paths" +) + +// TemplateLoader is a subset of the Elasticsearch client API capable of +// loading the template. +type ESClient interface { + LoadJSON(path string, json map[string]interface{}) ([]byte, error) + Request(method, path string, pipeline string, params map[string]string, body interface{}) (int, []byte, error) + GetVersion() string +} + +type Loader struct { + config TemplateConfig + client ESClient + beatInfo common.BeatInfo +} + +func NewLoader(cfg *common.Config, client ESClient, beatInfo common.BeatInfo) (*Loader, error) { + + config := defaultConfig + + err := cfg.Unpack(&config) + if err != nil { + return nil, err + } + + return &Loader{ + config: config, + client: client, + beatInfo: beatInfo, + }, nil +} + +// loadTemplate checks if the index mapping template should be loaded +// In case the template is not already loaded or overwriting is enabled, the +// template is written to index +func (l *Loader) Load() error { + + // Check if template already exist or should be overwritten + exists := l.CheckTemplate(l.config.Name) + if !exists || l.config.Overwrite { + + logp.Info("Loading template for elasticsearch version: %s", l.client.GetVersion()) + + if l.config.Overwrite { + logp.Info("Existing template will be overwritten, as overwrite is enabled.") + } + + if l.config.Name == "" { + l.config.Name = l.beatInfo.Beat + } + + tmpl, err := New(l.beatInfo.Version, l.client.GetVersion(), l.config.Name) + if err != nil { + return fmt.Errorf("error creating template instance: %v", err) + } + + fieldsPath := paths.Resolve(paths.Config, l.config.Fields) + + output, err := tmpl.Load(fieldsPath) + if err != nil { + return fmt.Errorf("error creating template from file %s: %v", fieldsPath, err) + } + + err = l.LoadTemplate(tmpl.GetName(), output) + if err != nil { + return fmt.Errorf("could not load template: %v", err) + } + } else { + logp.Info("Template already exists and will not be overwritten.") + } + + return nil +} + +// LoadTemplate loads a template into Elasticsearch overwriting the existing +// template if it exists. If you wish to not overwrite an existing template +// then use CheckTemplate prior to calling this method. +func (l *Loader) LoadTemplate(templateName string, template map[string]interface{}) error { + + logp.Info("load template: %s", templateName) + path := "/_template/" + templateName + body, err := l.client.LoadJSON(path, template) + if err != nil { + return fmt.Errorf("couldn't load template: %v. Response body: %s", err, body) + } + logp.Info("Elasticsearch template with name '%s' loaded", templateName) + return nil +} + +// CheckTemplate checks if a given template already exist. It returns true if +// and only if Elasticsearch returns with HTTP status code 200. +func (l *Loader) CheckTemplate(templateName string) bool { + + status, _, _ := l.client.Request("HEAD", "/_template/"+templateName, "", nil, nil) + + if status != 200 { + return false + } + + return true +} diff --git a/libbeat/template/load_integration_test.go b/libbeat/template/load_integration_test.go new file mode 100644 index 000000000000..75e89464e7a7 --- /dev/null +++ b/libbeat/template/load_integration_test.go @@ -0,0 +1,139 @@ +// +build integration + +package template + +import ( + "path/filepath" + "testing" + "time" + + "github.com/elastic/beats/libbeat/outputs/elasticsearch" + "github.com/elastic/beats/libbeat/version" + + "github.com/stretchr/testify/assert" +) + +func TestCheckTemplate(t *testing.T) { + + client := elasticsearch.GetTestingElasticsearch() + + err := client.Connect(5 * time.Second) + assert.Nil(t, err) + + loader := &Loader{ + client: client, + } + + // Check for non existent template + assert.False(t, loader.CheckTemplate("libbeat-notexists")) +} + +func TestLoadTemplate(t *testing.T) { + + // Setup ES + client := elasticsearch.GetTestingElasticsearch() + err := client.Connect(5 * time.Second) + assert.Nil(t, err) + + // Load template + absPath, err := filepath.Abs("../") + assert.NotNil(t, absPath) + assert.Nil(t, err) + + fieldsPath := absPath + "/fields.yml" + index := "testbeat" + + tmpl, err := New(version.GetDefaultVersion(), client.GetVersion(), index) + assert.NoError(t, err) + content, err := tmpl.Load(fieldsPath) + assert.NoError(t, err) + + loader := &Loader{ + client: client, + } + + // Load template + err = loader.LoadTemplate(tmpl.GetName(), content) + assert.Nil(t, err) + + // Make sure template was loaded + assert.True(t, loader.CheckTemplate(tmpl.GetName())) + + // Delete template again to clean up + client.Request("DELETE", "/_template/"+tmpl.GetName(), "", nil, nil) + + // Make sure it was removed + assert.False(t, loader.CheckTemplate(tmpl.GetName())) +} + +func TestLoadInvalidTemplate(t *testing.T) { + + // Invalid Template + template := map[string]interface{}{ + "json": "invalid", + } + + // Setup ES + client := elasticsearch.GetTestingElasticsearch() + err := client.Connect(5 * time.Second) + assert.Nil(t, err) + + templateName := "invalidtemplate" + + loader := &Loader{ + client: client, + } + + // Try to load invalid template + err = loader.LoadTemplate(templateName, template) + assert.Error(t, err) + + // Make sure template was not loaded + assert.False(t, loader.CheckTemplate(templateName)) +} + +// Tests loading the templates for each beat +func TestLoadBeatsTemplate(t *testing.T) { + + beats := []string{ + "libbeat", + } + + for _, beat := range beats { + // Load template + absPath, err := filepath.Abs("../../" + beat) + assert.NotNil(t, absPath) + assert.Nil(t, err) + + // Setup ES + client := elasticsearch.GetTestingElasticsearch() + + err = client.Connect(5 * time.Second) + assert.Nil(t, err) + + fieldsPath := absPath + "/fields.yml" + index := beat + + tmpl, err := New(version.GetDefaultVersion(), client.GetVersion(), index) + assert.NoError(t, err) + content, err := tmpl.Load(fieldsPath) + assert.NoError(t, err) + + loader := &Loader{ + client: client, + } + + // Load template + err = loader.LoadTemplate(tmpl.GetName(), content) + assert.Nil(t, err) + + // Make sure template was loaded + assert.True(t, loader.CheckTemplate(tmpl.GetName())) + + // Delete template again to clean up + client.Request("DELETE", "/_template/"+tmpl.GetName(), "", nil, nil) + + // Make sure it was removed + assert.False(t, loader.CheckTemplate(tmpl.GetName())) + } +} diff --git a/metricbeat/docs/reference/configuration/metricbeat-options.asciidoc b/metricbeat/docs/reference/configuration/metricbeat-options.asciidoc index b76a66b53489..6b2ddf4c74c9 100644 --- a/metricbeat/docs/reference/configuration/metricbeat-options.asciidoc +++ b/metricbeat/docs/reference/configuration/metricbeat-options.asciidoc @@ -75,7 +75,7 @@ include::../../../../libbeat/docs/outputconfig.asciidoc[] include::../../../../libbeat/docs/shared-path-config.asciidoc[] -include::../../../../libbeat/docs/dashboardsconfig.asciidoc[] +include::../../../../libbeat/docs/setup-config.asciidoc[] include::../../../../libbeat/docs/loggingconfig.asciidoc[] diff --git a/metricbeat/metricbeat.full.yml b/metricbeat/metricbeat.full.yml index 4e07b61c4fc4..e6aee6445511 100644 --- a/metricbeat/metricbeat.full.yml +++ b/metricbeat/metricbeat.full.yml @@ -551,24 +551,6 @@ output.elasticsearch: # requests are made. #flush_interval: 1s - # A template is used to set the mapping in Elasticsearch - # By default template loading is enabled and the template is loaded. - # These settings can be adjusted to load your own template or overwrite existing ones. - - # Set to false to disable template loading. - #template.enabled: true - - # Template name. By default the template name is metricbeat. - # The version of the beat will always be appended to the given name - # so the final name is metricbeat-%{[beat.version]}. - #template.name: "metricbeat" - - # Path to fields.yml file to generate the template - #template.fields: "${path.config}/fields.yml" - - # Overwrite existing template - #template.overwrite: false - # Use SSL settings for HTTPS. Default is true. #ssl.enabled: true @@ -995,6 +977,27 @@ output.elasticsearch: # dashboards and index pattern. Example: testbeat-* #setup.dashboards.index: +#============================== Template ===================================== + +# A template is used to set the mapping in Elasticsearch +# By default template loading is enabled and the template is loaded. +# These settings can be adjusted to load your own template or overwrite existing ones. + +# Set to false to disable template loading. +#setup.template.enabled: true + +# Template name. By default the template name is metricbeat. +# The version of the beat will always be appended to the given name +# so the final name is metricbeat-%{[beat.version]}. +#setup.template.name: "metricbeat" + +# Path to fields.yml file to generate the template +#setup.template.fields: "${path.config}/fields.yml" + +# Overwrite existing template +#setup.template.overwrite: false + + #================================ HTTP Endpoint ====================================== # Each beat can expose internal data points through a http endpoint. For security # reason the endpoint is disabled by default. This feature is currently in beta. diff --git a/packetbeat/docs/reference/configuration/packetbeat-options.asciidoc b/packetbeat/docs/reference/configuration/packetbeat-options.asciidoc index 26322ed82143..18beaae0c908 100644 --- a/packetbeat/docs/reference/configuration/packetbeat-options.asciidoc +++ b/packetbeat/docs/reference/configuration/packetbeat-options.asciidoc @@ -838,7 +838,7 @@ include::../../../../libbeat/docs/outputconfig.asciidoc[] include::../../../../libbeat/docs/shared-path-config.asciidoc[] -include::../../../../libbeat/docs/dashboardsconfig.asciidoc[] +include::../../../../libbeat/docs/setup-config.asciidoc[] include::../../../../libbeat/docs/loggingconfig.asciidoc[] diff --git a/packetbeat/packetbeat.full.yml b/packetbeat/packetbeat.full.yml index 9c393222d5b6..ff4842fcbd12 100644 --- a/packetbeat/packetbeat.full.yml +++ b/packetbeat/packetbeat.full.yml @@ -596,24 +596,6 @@ output.elasticsearch: # requests are made. #flush_interval: 1s - # A template is used to set the mapping in Elasticsearch - # By default template loading is enabled and the template is loaded. - # These settings can be adjusted to load your own template or overwrite existing ones. - - # Set to false to disable template loading. - #template.enabled: true - - # Template name. By default the template name is packetbeat. - # The version of the beat will always be appended to the given name - # so the final name is packetbeat-%{[beat.version]}. - #template.name: "packetbeat" - - # Path to fields.yml file to generate the template - #template.fields: "${path.config}/fields.yml" - - # Overwrite existing template - #template.overwrite: false - # Use SSL settings for HTTPS. Default is true. #ssl.enabled: true @@ -1040,6 +1022,27 @@ output.elasticsearch: # dashboards and index pattern. Example: testbeat-* #setup.dashboards.index: +#============================== Template ===================================== + +# A template is used to set the mapping in Elasticsearch +# By default template loading is enabled and the template is loaded. +# These settings can be adjusted to load your own template or overwrite existing ones. + +# Set to false to disable template loading. +#setup.template.enabled: true + +# Template name. By default the template name is packetbeat. +# The version of the beat will always be appended to the given name +# so the final name is packetbeat-%{[beat.version]}. +#setup.template.name: "packetbeat" + +# Path to fields.yml file to generate the template +#setup.template.fields: "${path.config}/fields.yml" + +# Overwrite existing template +#setup.template.overwrite: false + + #================================ HTTP Endpoint ====================================== # Each beat can expose internal data points through a http endpoint. For security # reason the endpoint is disabled by default. This feature is currently in beta. diff --git a/winlogbeat/docs/reference/configuration/winlogbeat-options.asciidoc b/winlogbeat/docs/reference/configuration/winlogbeat-options.asciidoc index 710544c85c98..2e7ffa765640 100644 --- a/winlogbeat/docs/reference/configuration/winlogbeat-options.asciidoc +++ b/winlogbeat/docs/reference/configuration/winlogbeat-options.asciidoc @@ -332,7 +332,7 @@ include::../../../../libbeat/docs/outputconfig.asciidoc[] include::../../../../libbeat/docs/shared-path-config.asciidoc[] -include::../../../../libbeat/docs/dashboardsconfig.asciidoc[] +include::../../../../libbeat/docs/setup-config.asciidoc[] include::../../../../libbeat/docs/loggingconfig.asciidoc[] diff --git a/winlogbeat/winlogbeat.full.yml b/winlogbeat/winlogbeat.full.yml index 189f27bde944..b9562b132309 100644 --- a/winlogbeat/winlogbeat.full.yml +++ b/winlogbeat/winlogbeat.full.yml @@ -170,24 +170,6 @@ output.elasticsearch: # requests are made. #flush_interval: 1s - # A template is used to set the mapping in Elasticsearch - # By default template loading is enabled and the template is loaded. - # These settings can be adjusted to load your own template or overwrite existing ones. - - # Set to false to disable template loading. - #template.enabled: true - - # Template name. By default the template name is winlogbeat. - # The version of the beat will always be appended to the given name - # so the final name is winlogbeat-%{[beat.version]}. - #template.name: "winlogbeat" - - # Path to fields.yml file to generate the template - #template.fields: "${path.config}/fields.yml" - - # Overwrite existing template - #template.overwrite: false - # Use SSL settings for HTTPS. Default is true. #ssl.enabled: true @@ -614,6 +596,27 @@ output.elasticsearch: # dashboards and index pattern. Example: testbeat-* #setup.dashboards.index: +#============================== Template ===================================== + +# A template is used to set the mapping in Elasticsearch +# By default template loading is enabled and the template is loaded. +# These settings can be adjusted to load your own template or overwrite existing ones. + +# Set to false to disable template loading. +#setup.template.enabled: true + +# Template name. By default the template name is winlogbeat. +# The version of the beat will always be appended to the given name +# so the final name is winlogbeat-%{[beat.version]}. +#setup.template.name: "winlogbeat" + +# Path to fields.yml file to generate the template +#setup.template.fields: "${path.config}/fields.yml" + +# Overwrite existing template +#setup.template.overwrite: false + + #================================ HTTP Endpoint ====================================== # Each beat can expose internal data points through a http endpoint. For security # reason the endpoint is disabled by default. This feature is currently in beta.