From 2a04e2c093b029af4c5d83fe72c188a5b80a91ee Mon Sep 17 00:00:00 2001
From: Shaunak Kashyap <ycombinator@gmail.com>
Date: Wed, 3 Oct 2018 05:21:41 -0700
Subject: [PATCH] Support Kibana Spaces (#8045)

* Support Kibana Spaces via setup.kibana.space.id setting

* Adding CHANGELOG entry

* Add space-id option to export_dashboards script

* Add integration tests for importing and exporting dashboards with space ID

* Adding -space-id option for export_dashboards.go to documentation

* Adding entry to dev CHANGELOG

* Prefix space path with /

* Create Kibana space before trying to use it

* Make spaces tests conditional on version

* Add semver to requirements.txt

* Add kbn-xsrf header to POST request

* Don't recreate space in second space test
---
 CHANGELOG-developer.asciidoc                  |  1 +
 CHANGELOG.asciidoc                            |  1 +
 auditbeat/auditbeat.yml                       |  5 +
 dev-tools/cmd/dashboards/export_dashboards.go | 11 ++-
 docs/devguide/newdashboards.asciidoc          |  9 ++
 filebeat/filebeat.yml                         |  5 +
 heartbeat/heartbeat.yml                       |  5 +
 libbeat/_meta/config.yml                      |  5 +
 libbeat/kibana/client.go                      |  7 +-
 libbeat/kibana/client_config.go               |  2 +
 libbeat/tests/system/requirements.txt         |  1 +
 libbeat/tests/system/test_dashboard.py        | 92 ++++++++++++++++++-
 metricbeat/metricbeat.yml                     |  5 +
 packetbeat/packetbeat.yml                     |  5 +
 winlogbeat/winlogbeat.yml                     |  5 +
 15 files changed, 154 insertions(+), 5 deletions(-)

diff --git a/CHANGELOG-developer.asciidoc b/CHANGELOG-developer.asciidoc
index ff80d070070..1581c93df95 100644
--- a/CHANGELOG-developer.asciidoc
+++ b/CHANGELOG-developer.asciidoc
@@ -53,3 +53,4 @@ The list below covers the major changes between 6.3.0 and master only.
   `cmd.GenRootCmd`, `cmd.GenRootCmdWithRunFlags`, and `cmd.GenRootCmdWithIndexPrefixWithRunFlags`. {pull}7850[7850]
 - Set current year in generator templates. {pull}8396[8396]
 - You can now override default settings of libbeat by using instance.Settings. {pull}8449[8449]
+- Add `-space-id` option to `export_dashboards.go` script to support Kibana Spaces {pull}7942[7942]
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
index 70fa946cd7e..3edfa840779 100644
--- a/CHANGELOG.asciidoc
+++ b/CHANGELOG.asciidoc
@@ -110,6 +110,7 @@ https://github.com/elastic/beats/compare/v6.4.0...master[Check the HEAD diff]
 - Added the `add_process_metadata` processor to enrich events with process information. {pull}6789[6789]
 - Report number of open file handles on Windows. {pull}8329[8329]
 - Support for Kafka 2.0.0 in kafka output {pull}8399[8399]
+- Add setting `setup.kibana.space.id` to support Kibana Spaces {pull}7942[7942]
 
 *Auditbeat*
 
diff --git a/auditbeat/auditbeat.yml b/auditbeat/auditbeat.yml
index 56789249281..a3749aa2af0 100644
--- a/auditbeat/auditbeat.yml
+++ b/auditbeat/auditbeat.yml
@@ -95,6 +95,11 @@ setup.kibana:
   # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601
   #host: "localhost:5601"
 
+  # Kibana Space ID
+  # ID of the Kibana Space into which the dashboards should be loaded. By default,
+  # the Default Space will be used.
+  #space.id:
+
 #============================= Elastic Cloud ==================================
 
 # These settings simplify using auditbeat with the Elastic Cloud (https://cloud.elastic.co/).
diff --git a/dev-tools/cmd/dashboards/export_dashboards.go b/dev-tools/cmd/dashboards/export_dashboards.go
index f208b8e6c16..421e006017d 100644
--- a/dev-tools/cmd/dashboards/export_dashboards.go
+++ b/dev-tools/cmd/dashboards/export_dashboards.go
@@ -27,6 +27,7 @@ import (
 	"net/http"
 	"net/url"
 	"os"
+	"path"
 	"path/filepath"
 	"strings"
 
@@ -48,11 +49,14 @@ func makeURL(url, path string, params url.Values) string {
 	return strings.Join([]string{url, path, "?", params.Encode()}, "")
 }
 
-func Export(client *http.Client, conn string, dashboard string, out string) error {
+func Export(client *http.Client, conn string, spaceID string, dashboard string, out string) error {
 	params := url.Values{}
 
 	params.Add("dashboard", dashboard)
 
+	if spaceID != "" {
+		exportAPI = path.Join("/s", spaceID, exportAPI)
+	}
 	fullURL := makeURL(conn, exportAPI, params)
 	if !quiet {
 		log.Printf("Calling HTTP GET %v\n", fullURL)
@@ -138,6 +142,7 @@ var quiet = false
 
 func main() {
 	kibanaURL := flag.String("kibana", "http://localhost:5601", "Kibana URL")
+	spaceID := flag.String("space-id", "", "Space ID")
 	dashboard := flag.String("dashboard", "", "Dashboard ID")
 	fileOutput := flag.String("output", "output.json", "Output file")
 	ymlFile := flag.String("yml", "", "Path to the module.yml file containing the dashboards")
@@ -171,7 +176,7 @@ func main() {
 			if err != nil {
 				log.Fatalf("fail to create directory %s: %v", directory, err)
 			}
-			err = Export(client, *kibanaURL, dashboard["id"], filepath.Join(directory, dashboard["file"]))
+			err = Export(client, *kibanaURL, *spaceID, dashboard["id"], filepath.Join(directory, dashboard["file"]))
 			if err != nil {
 				log.Fatalf("fail to export the dashboards: %s", err)
 			}
@@ -180,7 +185,7 @@ func main() {
 	}
 
 	if len(*dashboard) > 0 {
-		err := Export(client, *kibanaURL, *dashboard, *fileOutput)
+		err := Export(client, *kibanaURL, *spaceID, *dashboard, *fileOutput)
 		if err != nil {
 			log.Fatalf("fail to export the dashboards: %s", err)
 		}
diff --git a/docs/devguide/newdashboards.asciidoc b/docs/devguide/newdashboards.asciidoc
index b874dafbfe3..b532655fc20 100644
--- a/docs/devguide/newdashboards.asciidoc
+++ b/docs/devguide/newdashboards.asciidoc
@@ -283,6 +283,15 @@ go run dev-tools/cmd/dashboards/export_dashboards.go -yml filebeat/module/system
 -------------------
 
 
+===== Export dashboards from a Kibana Space
+
+If you are using the Kibana Spaces feature and want to export dashboards from a specific Space, pass the Space ID to the `export_dashboards.go` script:
+
+[source,shell]
+-------------------
+go run dev-tools/cmd/dashboards/export_dashboards.go -space-id my-space [other-options]
+-------------------
+
 
 ==== Exporting Kibana 5.x dashboards
 
diff --git a/filebeat/filebeat.yml b/filebeat/filebeat.yml
index 5ae43216b7d..52888b19ea5 100644
--- a/filebeat/filebeat.yml
+++ b/filebeat/filebeat.yml
@@ -122,6 +122,11 @@ setup.kibana:
   # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601
   #host: "localhost:5601"
 
+  # Kibana Space ID
+  # ID of the Kibana Space into which the dashboards should be loaded. By default,
+  # the Default Space will be used.
+  #space.id:
+
 #============================= Elastic Cloud ==================================
 
 # These settings simplify using filebeat with the Elastic Cloud (https://cloud.elastic.co/).
diff --git a/heartbeat/heartbeat.yml b/heartbeat/heartbeat.yml
index d80f7bc0f5f..044616ad816 100644
--- a/heartbeat/heartbeat.yml
+++ b/heartbeat/heartbeat.yml
@@ -69,6 +69,11 @@ setup.kibana:
   # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601
   #host: "localhost:5601"
 
+  # Kibana Space ID
+  # ID of the Kibana Space into which the dashboards should be loaded. By default,
+  # the Default Space will be used.
+  #space.id:
+
 #============================= Elastic Cloud ==================================
 
 # These settings simplify using heartbeat with the Elastic Cloud (https://cloud.elastic.co/).
diff --git a/libbeat/_meta/config.yml b/libbeat/_meta/config.yml
index 0277cd84fdd..6f460928bac 100644
--- a/libbeat/_meta/config.yml
+++ b/libbeat/_meta/config.yml
@@ -39,6 +39,11 @@ setup.kibana:
   # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601
   #host: "localhost:5601"
 
+  # Kibana Space ID
+  # ID of the Kibana Space into which the dashboards should be loaded. By default,
+  # the Default Space will be used.
+  #space.id:
+
 #============================= Elastic Cloud ==================================
 
 # These settings simplify using beatname with the Elastic Cloud (https://cloud.elastic.co/).
diff --git a/libbeat/kibana/client.go b/libbeat/kibana/client.go
index 69b34700ae6..2efa9b0bfa7 100644
--- a/libbeat/kibana/client.go
+++ b/libbeat/kibana/client.go
@@ -25,6 +25,7 @@ import (
 	"io/ioutil"
 	"net/http"
 	"net/url"
+	"path"
 	"strings"
 
 	"github.com/pkg/errors"
@@ -88,7 +89,11 @@ func NewKibanaClient(cfg *common.Config) (*Client, error) {
 
 // NewClientWithConfig creates and returns a kibana client using the given config
 func NewClientWithConfig(config *ClientConfig) (*Client, error) {
-	kibanaURL, err := common.MakeURL(config.Protocol, config.Path, config.Host, 5601)
+	p := config.Path
+	if config.SpaceID != "" {
+		p = path.Join(p, "s", config.SpaceID)
+	}
+	kibanaURL, err := common.MakeURL(config.Protocol, p, config.Host, 5601)
 	if err != nil {
 		return nil, fmt.Errorf("invalid Kibana host: %v", err)
 	}
diff --git a/libbeat/kibana/client_config.go b/libbeat/kibana/client_config.go
index 25af90a810c..c5dc101075c 100644
--- a/libbeat/kibana/client_config.go
+++ b/libbeat/kibana/client_config.go
@@ -28,6 +28,7 @@ type ClientConfig struct {
 	Protocol string            `config:"protocol"`
 	Host     string            `config:"host"`
 	Path     string            `config:"path"`
+	SpaceID  string            `config:"space.id"`
 	Username string            `config:"username"`
 	Password string            `config:"password"`
 	TLS      *tlscommon.Config `config:"ssl"`
@@ -39,6 +40,7 @@ var (
 		Protocol: "http",
 		Host:     "localhost:5601",
 		Path:     "",
+		SpaceID:  "",
 		Username: "",
 		Password: "",
 		Timeout:  90 * time.Second,
diff --git a/libbeat/tests/system/requirements.txt b/libbeat/tests/system/requirements.txt
index 743fdf14c66..35eba8977d5 100644
--- a/libbeat/tests/system/requirements.txt
+++ b/libbeat/tests/system/requirements.txt
@@ -29,3 +29,4 @@ urllib3==1.22
 websocket-client==0.47.0
 parameterized==0.6.1
 jsondiff==1.1.2
+semver==2.8.1
diff --git a/libbeat/tests/system/test_dashboard.py b/libbeat/tests/system/test_dashboard.py
index e4b6f69f387..e76d738a9d5 100644
--- a/libbeat/tests/system/test_dashboard.py
+++ b/libbeat/tests/system/test_dashboard.py
@@ -4,7 +4,8 @@
 import subprocess
 from nose.plugins.attrib import attr
 import unittest
-
+import requests
+import semver
 
 INTEGRATION_TESTS = os.environ.get('INTEGRATION_TESTS', False)
 
@@ -36,6 +37,40 @@ def test_load_dashboard(self):
 
         assert self.log_contains("Kibana dashboards successfully loaded") is True
 
+    @unittest.skipUnless(INTEGRATION_TESTS, "integration test")
+    @attr('integration')
+    def test_load_dashboard_into_space(self, create_space=True):
+        """
+        Test loading dashboards into Kibana space
+        """
+        version = self.get_version()
+        if semver.compare(version, "6.5.0") == -1:
+            # Skip for Kibana versions < 6.5.0 as Kibana Spaces not available
+            raise SkipTest
+
+        self.render_config_template()
+        if create_space:
+            self.create_kibana_space()
+
+        beat = self.start_beat(
+            logging_args=["-e", "-d", "*"],
+            extra_args=["setup",
+                        "--dashboards",
+                        "-E", "setup.dashboards.file=" +
+                        os.path.join(self.beat_path, "tests", "files", "testbeat-dashboards.zip"),
+                        "-E", "setup.dashboards.beat=testbeat",
+                        "-E", "setup.kibana.protocol=http",
+                        "-E", "setup.kibana.host=" + self.get_kibana_host(),
+                        "-E", "setup.kibana.port=" + self.get_kibana_port(),
+                        "-E", "setup.kibana.space.id=foo-bar",
+                        "-E", "output.elasticsearch.hosts=['" + self.get_host() + "']",
+                        "-E", "output.file.enabled=false"]
+        )
+
+        beat.check_wait(exit_code=0)
+
+        assert self.log_contains("Kibana dashboards successfully loaded") is True
+
     @unittest.skipUnless(INTEGRATION_TESTS, "integration test")
     @attr('integration')
     def test_load_only_index_patterns(self):
@@ -88,6 +123,36 @@ def test_export_dashboard(self):
 
         os.remove("output.json")
 
+    @unittest.skipUnless(INTEGRATION_TESTS, "integration test")
+    @attr('integration')
+    def test_export_dashboard_from_space(self):
+        """
+        Test export dashboards from Kibana space and remove unsupported characters
+        """
+        version = self.get_version()
+        if semver.compare(version, "6.5.0") == -1:
+            # Skip for Kibana versions < 6.5.0 as Kibana Spaces not available
+            raise SkipTest
+
+        self.test_load_dashboard_into_space(False)
+
+        path = os.path.normpath(self.beat_path + "/../dev-tools/cmd/dashboards/export_dashboards.go")
+        command = path + " -kibana http://" + self.get_kibana_host() + ":" + self.get_kibana_port()
+        command = "go run " + command + " -dashboard Metricbeat-system-overview -space-id foo-bar"
+
+        p = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        content, err = p.communicate()
+
+        assert p.returncode == 0
+
+        assert os.path.isfile("output.json") is True
+
+        with open('output.json') as f:
+            content = f.read()
+            assert "Metricbeat-system-overview" in content
+
+        os.remove("output.json")
+
     def get_host(self):
         return os.getenv('ES_HOST', 'localhost') + ':' + os.getenv('ES_PORT', '9200')
 
@@ -96,3 +161,28 @@ def get_kibana_host(self):
 
     def get_kibana_port(self):
         return os.getenv('KIBANA_PORT', '5601')
+
+    def create_kibana_space(self):
+        url = "http://" + self.get_kibana_host() + ":" + self.get_kibana_port() + \
+            "/api/spaces/space"
+        data = {
+            "id": "foo-bar",
+            "name": "Foo bar space"
+        }
+
+        headers = {
+            "kbn-xsrf": "1"
+        }
+
+        r = requests.post(url, json=data, headers=headers)
+        assert r.status_code == 200
+
+    def get_version(self):
+        url = "http://" + self.get_kibana_host() + ":" + self.get_kibana_port() + \
+            "/api/status"
+
+        r = requests.get(url)
+        body = r.json()
+        version = body["version"]["number"]
+
+        return version
diff --git a/metricbeat/metricbeat.yml b/metricbeat/metricbeat.yml
index 8363541d426..5bb313ded91 100644
--- a/metricbeat/metricbeat.yml
+++ b/metricbeat/metricbeat.yml
@@ -66,6 +66,11 @@ setup.kibana:
   # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601
   #host: "localhost:5601"
 
+  # Kibana Space ID
+  # ID of the Kibana Space into which the dashboards should be loaded. By default,
+  # the Default Space will be used.
+  #space.id:
+
 #============================= Elastic Cloud ==================================
 
 # These settings simplify using metricbeat with the Elastic Cloud (https://cloud.elastic.co/).
diff --git a/packetbeat/packetbeat.yml b/packetbeat/packetbeat.yml
index 79babc2232a..5bdaa0f6911 100644
--- a/packetbeat/packetbeat.yml
+++ b/packetbeat/packetbeat.yml
@@ -149,6 +149,11 @@ setup.kibana:
   # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601
   #host: "localhost:5601"
 
+  # Kibana Space ID
+  # ID of the Kibana Space into which the dashboards should be loaded. By default,
+  # the Default Space will be used.
+  #space.id:
+
 #============================= Elastic Cloud ==================================
 
 # These settings simplify using packetbeat with the Elastic Cloud (https://cloud.elastic.co/).
diff --git a/winlogbeat/winlogbeat.yml b/winlogbeat/winlogbeat.yml
index d5c53bbe686..d711c7f10db 100644
--- a/winlogbeat/winlogbeat.yml
+++ b/winlogbeat/winlogbeat.yml
@@ -70,6 +70,11 @@ setup.kibana:
   # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601
   #host: "localhost:5601"
 
+  # Kibana Space ID
+  # ID of the Kibana Space into which the dashboards should be loaded. By default,
+  # the Default Space will be used.
+  #space.id:
+
 #============================= Elastic Cloud ==================================
 
 # These settings simplify using winlogbeat with the Elastic Cloud (https://cloud.elastic.co/).