Improvements and fixes to the Host Overview dashboard (elastic#5340) (e…

…lastic#5351)

* Use `beat.name` instead of `beat.hostname` in visualizations. See elastic#5276 for
  the motivation
* Add a "tip" widget that tells user how they can select another host.
* Automatically set a search pattern of the form `beat.name: "HOSTNAME"` where
  HOSTNAME is the Beat name (hostname) that uploads the dashboards.

The way the last point works is that I saved the dashbaord using this filter:
`beat.name:"CHANGEME_HOSTNAME"`. The kibana loader code does a string replacement
and replaces `CHANGEME_HOSTNAME` with the actual hostname. The disadvantage of this
approach (vs doing JSON parsing) is that we must remember to always save that
dashboard with the `CHANGEME_HOSTNAME` wildcard in place. I have added a unit test
to the system module that checks for that, so it should be relatively hard for
us to forget.

Closes elastic#5276.

(cherry picked from commit 4778c51)

CHANGELOG.asciidoc

@@ -45,6 +45,8 @@ https://github.com/elastic/beats/compare/v6.0.0-rc1...master[Check the HEAD diff
 
 *Metricbeat*
 
+- Use `beat.name` instead of `beat.hostname` in the Host Overview dashboard. {pull}5340[5340]
+
 *Packetbeat*
 
 *Winlogbeat*
@@ -61,6 +63,8 @@ https://github.com/elastic/beats/compare/v6.0.0-rc1...master[Check the HEAD diff
 
 *Metricbeat*
 
+- Auto-select a hostname (based on the host on which the Beat is running) in the Host Overview dashboard. {pull}5340[5340]
+
 *Packetbeat*
 
 *Winlogbeat*

libbeat/cmd/instance/beat.go

@@ -533,7 +533,7 @@ func (b *Beat) loadDashboards(force bool) error {
 		if b.Config.Output.Name() == "elasticsearch" {
 			esConfig = b.Config.Output.Config()
 		}
-		err := dashboards.ImportDashboards(b.Info.Beat, b.Info.Version, paths.Resolve(paths.Home, ""),
+		err := dashboards.ImportDashboards(b.Info.Beat, b.Info.Name, paths.Resolve(paths.Home, ""),
 			b.Config.Kibana, esConfig, b.Config.Dashboards, nil)
 		if err != nil {
 			return fmt.Errorf("Error importing Kibana dashboards: %v", err)

libbeat/dashboards/dashboards.go

@@ -10,7 +10,7 @@ import (
 	"github.com/elastic/beats/libbeat/logp"
 )
 
-func ImportDashboards(beatName, beatVersion, homePath string,
+func ImportDashboards(beatName, hostname, homePath string,
 	kibanaConfig *common.Config, esConfig *common.Config,
 	dashboardsConfig *common.Config, msgOutputter MessageOutputter) error {
 
@@ -62,7 +62,7 @@ func ImportDashboards(beatName, beatVersion, homePath string,
 		kibanaConfig.SetString("password", -1, esLoader.client.Password)
 	}
 
-	kibanaLoader, err := NewKibanaLoader(kibanaConfig, &dashConfig, msgOutputter)
+	kibanaLoader, err := NewKibanaLoader(kibanaConfig, &dashConfig, hostname, msgOutputter)
 	if err != nil {
 		return fmt.Errorf("fail to create the Kibana loader: %v", err)
 	}

libbeat/dashboards/kibana_loader.go

@@ -17,10 +17,11 @@ type KibanaLoader struct {
 	client       *kibana.Client
 	config       *Config
 	version      string
+	hostname     string
 	msgOutputter MessageOutputter
 }
 
-func NewKibanaLoader(cfg *common.Config, dashboardsConfig *Config, msgOutputter MessageOutputter) (*KibanaLoader, error) {
+func NewKibanaLoader(cfg *common.Config, dashboardsConfig *Config, hostname string, msgOutputter MessageOutputter) (*KibanaLoader, error) {
 
 	if cfg == nil || !cfg.Enabled() {
 		return nil, fmt.Errorf("Kibana is not configured or enabled")
@@ -35,6 +36,7 @@ func NewKibanaLoader(cfg *common.Config, dashboardsConfig *Config, msgOutputter
 		client:       client,
 		config:       dashboardsConfig,
 		version:      client.GetVersion(),
+		hostname:     hostname,
 		msgOutputter: msgOutputter,
 	}
 
@@ -50,13 +52,13 @@ func (loader KibanaLoader) ImportIndex(file string) error {
 	// read json file
 	reader, err := ioutil.ReadFile(file)
 	if err != nil {
-		return fmt.Errorf("fail to read index-pattern: %v", err)
+		return fmt.Errorf("fail to read index-pattern from file %s: %v", file, err)
 	}
 
 	var indexContent common.MapStr
 	err = json.Unmarshal(reader, &indexContent)
 	if err != nil {
-		return fmt.Errorf("fail to unmarshal the index content: %v", err)
+		return fmt.Errorf("fail to unmarshal the index content from file %s: %v", file, err)
 	}
 
 	indexContent = ReplaceIndexInIndexPattern(loader.config.Index, indexContent)
@@ -72,16 +74,21 @@ func (loader KibanaLoader) ImportDashboard(file string) error {
 	// read json file
 	reader, err := ioutil.ReadFile(file)
 	if err != nil {
-		return fmt.Errorf("fail to read index-pattern: %v", err)
+		return fmt.Errorf("fail to read dashboard from file %s: %v", file, err)
 	}
 	var content common.MapStr
 	err = json.Unmarshal(reader, &content)
 	if err != nil {
-		return fmt.Errorf("fail to unmarshal the index content: %v", err)
+		return fmt.Errorf("fail to unmarshal the dashboard content from file %s: %v", file, err)
 	}
 
 	content = ReplaceIndexInDashboardObject(loader.config.Index, content)
 
+	content, err = ReplaceStringInDashboard("CHANGEME_HOSTNAME", loader.hostname, content)
+	if err != nil {
+		return fmt.Errorf("fail to replace the hostname in dashboard %s: %v", file, err)
+	}
+
 	return loader.client.ImportJSON(importAPI, params, content)
 }
 

libbeat/dashboards/modify_json.go

@@ -1,6 +1,7 @@
 package dashboards
 
 import (
+	"bytes"
 	"encoding/json"
 	"fmt"
 
@@ -103,3 +104,16 @@ func ReplaceIndexInDashboardObject(index string, content common.MapStr) common.M
 	}
 	return content
 }
+
+func ReplaceStringInDashboard(old, new string, content common.MapStr) (common.MapStr, error) {
+	marshaled, err := json.Marshal(content)
+	if err != nil {
+		return nil, fmt.Errorf("fail to marshal dashboard object: %v", content)
+	}
+
+	replaced := bytes.Replace(marshaled, []byte(old), []byte(new), -1)
+
+	var result common.MapStr
+	err = json.Unmarshal(replaced, &result)
+	return result, nil
+}

libbeat/dashboards/modify_json_test.go

@@ -0,0 +1,54 @@
+package dashboards
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/elastic/beats/libbeat/common"
+)
+
+func TestReplaceStringInDashboard(t *testing.T) {
+	tests := []struct {
+		content  common.MapStr
+		old      string
+		new      string
+		expected common.MapStr
+	}{
+		{
+			content:  common.MapStr{"test": "CHANGEME"},
+			old:      "CHANGEME",
+			new:      "hostname",
+			expected: common.MapStr{"test": "hostname"},
+		},
+		{
+			content:  common.MapStr{"test": "hello"},
+			old:      "CHANGEME",
+			new:      "hostname",
+			expected: common.MapStr{"test": "hello"},
+		},
+		{
+			content:  common.MapStr{"test": map[string]interface{}{"key": "\"CHANGEME\""}},
+			old:      "CHANGEME",
+			new:      "hostname.local",
+			expected: common.MapStr{"test": map[string]interface{}{"key": "\"hostname.local\""}},
+		},
+		{
+			content: common.MapStr{
+				"kibanaSavedObjectMeta": map[string]interface{}{
+					"searchSourceJSON": "{\"filter\":[],\"highlightAll\":true,\"version\":true,\"query\":{\"query\":\"beat.name:\\\"CHANGEME_HOSTNAME\\\"\",\"language\":\"lucene\"}}"}},
+
+			old: "CHANGEME_HOSTNAME",
+			new: "hostname.local",
+			expected: common.MapStr{
+				"kibanaSavedObjectMeta": map[string]interface{}{
+					"searchSourceJSON": "{\"filter\":[],\"highlightAll\":true,\"version\":true,\"query\":{\"query\":\"beat.name:\\\"hostname.local\\\"\",\"language\":\"lucene\"}}"}},
+		},
+	}
+
+	for _, test := range tests {
+		result, err := ReplaceStringInDashboard(test.old, test.new, test.content)
+		assert.NoError(t, err)
+		assert.Equal(t, test.expected, result)
+	}
+}

metricbeat/module/system/_meta/kibana/default/dashboard/Metricbeat-containers-overview.json

@@ -13,7 +13,7 @@
       },
       "id": "Container-CPU-usage",
       "type": "visualization",
-      "version": 2
+      "version": 1
     },
     {
       "attributes": {
@@ -28,7 +28,7 @@
       },
       "id": "System-Navigation",
       "type": "visualization",
-      "version": 6
+      "version": 3
     },
     {
       "attributes": {
@@ -43,7 +43,7 @@
       },
       "id": "Container-Memory-stats",
       "type": "visualization",
-      "version": 2
+      "version": 1
     },
     {
       "attributes": {
@@ -58,7 +58,7 @@
       },
       "id": "Container-Block-IO",
       "type": "visualization",
-      "version": 3
+      "version": 1
     },
     {
       "attributes": {
@@ -76,8 +76,8 @@
       },
       "id": "CPU-slash-Memory-per-container",
       "type": "dashboard",
-      "version": 4
+      "version": 1
     }
   ],
-  "version": "6.0.0-beta1-SNAPSHOT"
+  "version": "6.0.0-rc1-SNAPSHOT"
 }