Load template by default, depending on the version (#1993)

This queries Elasticsearch for the version and choses between the
default template file (specified by the `template.path` option) and
the ES 2x version of the template file (specified by the new
`template.versions.2x.path` option).

There's also a change in the defaults. With all the template
options commented out, the code now loads the template. Previously,
the template loading was the default via the sample configs only.

I also did a small change in the output plugins API, now they receive
the beatname in a clean way.

Fixes #1740.

CHANGELOG.asciidoc

@@ -15,6 +15,7 @@ https://github.com/elastic/beats/compare/v5.0.0-alpha4...master[Check the HEAD d
 *Affecting all Beats*
 - Rename the `filters` section to `processors`. {pull}1944[1944]
 - Introduce the condition with `when` in the processor configuration. {pull}1949[1949]
+- The Elasticsearch template is now loaded by default. {pull}1993[1993]
 
 *Metricbeat*
 
@@ -53,6 +54,7 @@ https://github.com/elastic/beats/compare/v5.0.0-alpha4...master[Check the HEAD d
 - Command line flag -c can be used multiple times. {pull}1985[1985]
 - Add OR/AND/NOT to the condition associated with the processors. {pull}1983[1983]
 - Add '-E' CLI flag for overwriting single config options via command line. {pull}1986[1986]
+- Choose the mapping template file based on the Elasticsearch version. {pull}1993[1993]
 
 *Metricbeat*
 

filebeat/filebeat.full.yml

@@ -357,16 +357,28 @@ output.elasticsearch:
 
   # A template is used to set the mapping in Elasticsearch
   # By default template loading is enabled and the template is loaded.
-  # These settings can be adjusted to load your own template or overwrite existing ones
+  # These settings can be adjusted to load your own template or overwrite existing ones.
+
+  # Set to false to disable template loading.
+  #template.enabled: true
 
   # Template name. By default the template name is filebeat.
-  template.name: "filebeat"
+  #template.name: "filebeat"
 
   # Path to template file
-  template.path: "filebeat.template.json"
+  #template.path: "filebeat.template.json"
 
   # Overwrite existing template
-  template.overwrite: false
+  #template.overwrite: false
+
+  # If set to true, filebeat checks the Elasticsearch version at connect time, and if it
+  # is 2.x, it loads the file specified by the template.versions.2x.path setting. The 
+  # default is true.
+  #template.versions.2x.enabled: true
+
+  # Path to the Elasticsearch 2.x version of the template file.
+  #template.versions.2x.path: "filebeat.template-es2x.json"
+
 
   # TLS configuration. By default is off.
   # List of root certificates for HTTPS server verifications

filebeat/filebeat.yml

@@ -82,15 +82,6 @@ output.elasticsearch:
   # Array of hosts to connect to.
   hosts: ["localhost:9200"]
 
-  # Template name. By default the template name is filebeat.
-  template.name: "filebeat"
-
-  # Path to template file
-  template.path: "filebeat.template.json"
-
-  # Overwrite existing template
-  template.overwrite: false
-
 #----------------------------- Logstash output --------------------------------
 #output.logstash:
   # The Logstash hosts

libbeat/_meta/config.full.yml

@@ -139,16 +139,28 @@ output.elasticsearch:
 
   # A template is used to set the mapping in Elasticsearch
   # By default template loading is enabled and the template is loaded.
-  # These settings can be adjusted to load your own template or overwrite existing ones
+  # These settings can be adjusted to load your own template or overwrite existing ones.
+
+  # Set to false to disable template loading.
+  #template.enabled: true
 
   # Template name. By default the template name is beatname.
-  template.name: "beatname"
+  #template.name: "beatname"
 
   # Path to template file
-  template.path: "beatname.template.json"
+  #template.path: "beatname.template.json"
 
   # Overwrite existing template
-  template.overwrite: false
+  #template.overwrite: false
+
+  # If set to true, beatname checks the Elasticsearch version at connect time, and if it
+  # is 2.x, it loads the file specified by the template.versions.2x.path setting. The 
+  # default is true.
+  #template.versions.2x.enabled: true
+
+  # Path to the Elasticsearch 2.x version of the template file.
+  #template.versions.2x.path: "beatname.template-es2x.json"
+
 
   # TLS configuration. By default is off.
   # List of root certificates for HTTPS server verifications

libbeat/_meta/config.yml

@@ -24,15 +24,6 @@ output.elasticsearch:
   # Array of hosts to connect to.
   hosts: ["localhost:9200"]
 
-  # Template name. By default the template name is beatname.
-  template.name: "beatname"
-
-  # Path to template file
-  template.path: "beatname.template.json"
-
-  # Overwrite existing template
-  template.overwrite: false
-
 #----------------------------- Logstash output --------------------------------
 #output.logstash:
   # The Logstash hosts

libbeat/docs/outputconfig.asciidoc

@@ -24,8 +24,8 @@ output.elasticsearch:
   # The Elasticsearch cluster
   hosts: ["http://localhost:9200"]
 
-  # Template name. By default the template name is {beatname_lc}.
-  template.name: "{beatname_lc}"
+  # Load the Elasticsearch template.
+  template.enabled: true
 
   # Path to template file
   template.path: "{beatname_lc}.template.json"
@@ -195,6 +195,9 @@ enabled.
 
 You can adjust the following settings to load your own template or overwrite an existing one:
 
+*`enabled`*:: Set to false to disable template loading. If set this to false,
+you must <<load-template-manually,load the template manually>>.
+
 *`name`*:: The name of the template. The default is +{beatname_lc}+.
 
 *`path`*:: The path to the template file. The default is +{beatname_lc}.template.json+. If a relative
@@ -215,8 +218,28 @@ output.elasticsearch:
   template.overwrite: false
 ----------------------------------------------------------------------
 
-To disable automatic template loading, comment out the template part under elasticsearch output.
-If you disable this option, you must <<load-template-manually,load the template manually>>. 
+===== template.versions
+
+In the default configuration, {beatname_uc} automatically checks the
+Elasticsearch version and loads the recommended template file for the particular
+version. This behaviour can be controlled from the following options:
+
+*`2x.path`*:: The path to the template file to load for
+Elasticsearch versions 2.x.y. The default is +{beatname_lc}.template-es2x.json+.
+
+*`2x.enabled`*:: If set to +false+, the +2x.path+ option is ignored and the
+default template is loaded regardless of the Elasticsearch version.
+
+For example:
+
+["source","yaml",subs="attributes,callouts"]
+----------------------------------------------------------------------
+output.elasticsearch:
+  hosts: ["localhost:9200"]
+  template.path: "{beatname_lc}.template.json"
+  template.overwrite: false
+  template.versions.2x.path: "{beatname_lc}.template-es2x.json
+----------------------------------------------------------------------
 
 ===== max_retries
 

libbeat/outputs/console/console.go

@@ -18,7 +18,7 @@ type console struct {
 	config config
 }
 
-func New(config *common.Config, _ int) (outputs.Outputer, error) {
+func New(_ string, config *common.Config, _ int) (outputs.Outputer, error) {
 	c := &console{config: defaultConfig}
 	err := config.Unpack(&c.config)
 	if err != nil {

libbeat/outputs/elasticsearch/api_mock_test.go

@@ -19,12 +19,14 @@ import (
 func ElasticsearchMock(code int, body []byte) *httptest.Server {
 
 	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		respCode := code
-		if r.Method == "HEAD" { // send ok on ping
-			respCode = 200
+		if r.URL.Path == "/" { // send ok and a minimal JSON on ping
+			w.WriteHeader(200)
+			w.Header().Set("Content-Type", "application/json")
+			w.Write([]byte(`{"version":{"number":"5.0.0"}}`))
+			return
 		}
 
-		w.WriteHeader(respCode)
+		w.WriteHeader(code)
 		if body != nil {
 			w.Header().Set("Content-Type", "application/json")
 			w.Write(body)

libbeat/outputs/elasticsearch/client.go

@@ -3,6 +3,7 @@ package elasticsearch
 import (
 	"bytes"
 	"crypto/tls"
+	"encoding/json"
 	"errors"
 	"expvar"
 	"fmt"
@@ -46,6 +47,7 @@ type Connection struct {
 	onConnectCallback func() error
 
 	encoder bodyEncoder
+	version string
 }
 
 // Metrics that can retrieved through the expvar web interface.
@@ -495,7 +497,7 @@ func (client *Client) CheckTemplate(templateName string) bool {
 
 func (conn *Connection) Connect(timeout time.Duration) error {
 	var err error
-	conn.connected, err = conn.Ping(timeout)
+	conn.connected, conn.version, err = conn.Ping(timeout)
 	if err != nil {
 		return err
 	}
@@ -510,18 +512,35 @@ func (conn *Connection) Connect(timeout time.Duration) error {
 	return nil
 }
 
-func (conn *Connection) Ping(timeout time.Duration) (bool, error) {
+// Ping sends a GET request to the Elasticsearch
+func (conn *Connection) Ping(timeout time.Duration) (bool, string, error) {
 	debugf("ES Ping(url=%v, timeout=%v)", conn.URL, timeout)
 
 	conn.http.Timeout = timeout
-	status, _, err := conn.execRequest("HEAD", conn.URL, nil)
+	status, body, err := conn.execRequest("GET", conn.URL, nil)
 	if err != nil {
 		debugf("Ping request failed with: %v", err)
-		return false, err
+		return false, "", err
+	}
+
+	if status >= 300 {
+		return false, "", fmt.Errorf("Non 2xx response code: %d", status)
+	}
+
+	var response struct {
+		Version struct {
+			Number string
+		}
+	}
+
+	err = json.Unmarshal(body, &response)
+	if err != nil {
+		return false, "", fmt.Errorf("Failed to parse JSON response: %v", err)
 	}
 
 	debugf("Ping status code: %v", status)
-	return status < 300, nil
+	logp.Info("Connected to Elasticsearch version %s", response.Version.Number)
+	return true, response.Version.Number, nil
 }
 
 func (conn *Connection) IsConnected() bool {

libbeat/outputs/elasticsearch/client_integration_test.go

@@ -152,8 +152,9 @@ func TestOutputLoadTemplate(t *testing.T) {
 	config := map[string]interface{}{
 		"hosts": GetEsHost(),
 		"template": map[string]interface{}{
-			"name": "libbeat",
-			"path": tPath,
+			"name":                "libbeat",
+			"path":                tPath,
+			"versions.2x.enabled": false,
 		},
 	}
 
@@ -162,7 +163,7 @@ func TestOutputLoadTemplate(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	output, err := New(cfg, 0)
+	output, err := New("libbeat", cfg, 0)
 	if err != nil {
 		t.Fatal(err)
 	}

libbeat/outputs/elasticsearch/config.go

@@ -24,9 +24,20 @@ type elasticsearchConfig struct {
 }
 
 type Template struct {
-	Name      string `config:"name"`
-	Path      string `config:"path"`
-	Overwrite bool   `config:"overwrite"`
+	Enabled   bool             `config:"enabled"`
+	Name      string           `config:"name"`
+	Path      string           `config:"path"`
+	Overwrite bool             `config:"overwrite"`
+	Versions  TemplateVersions `config:"versions"`
+}
+
+type TemplateVersions struct {
+	Es2x TemplateVersion `config:"2x"`
+}
+
+type TemplateVersion struct {
+	Enabled bool   `config:"enabled"`
+	Path    string `config:"path"`
 }
 
 const (
@@ -46,6 +57,10 @@ var (
 		CompressionLevel: 0,
 		TLS:              nil,
 		LoadBalance:      true,
+		Template: Template{
+			Enabled:  true,
+			Versions: TemplateVersions{Es2x: TemplateVersion{Enabled: true}},
+		},
 	}
 )