Cherry-pick to 7.0: Add username password into kibana config (#10675)

CHANGELOG.next.asciidoc

@@ -168,6 +168,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix encoding of timestamps when using disk spool. {issue}10099[10099]
 - Fix stopping of modules started by kubernetes autodiscover. {pull}10476[10476]
 - Fix a issue when remote and local configuration didn't match when fetching configuration from Central Management. {issue}10587[10587]
+- Fix unauthorized error when loading dashboards by adding username and password into kibana config. {issue}10513[10513] {pull}10675[10675]
 
 *Auditbeat*
 

libbeat/cmd/instance/beat.go

@@ -681,11 +681,6 @@ func (b *Beat) loadDashboards(ctx context.Context, force bool) error {
 	}
 
 	if b.Config.Dashboards.Enabled() {
-		var esConfig *common.Config
-		if b.Config.Output.Name() == "elasticsearch" {
-			esConfig = b.Config.Output.Config()
-		}
-
 		var withMigration bool
 		if b.RawConfig.HasField("migration") {
 			sub, err := b.RawConfig.Child("migration", -1)
@@ -695,10 +690,11 @@ func (b *Beat) loadDashboards(ctx context.Context, force bool) error {
 			withMigration = sub.Enabled()
 		}
 
-		// init kibana config object
-		kibanaConfig := b.Config.Kibana
-		if kibanaConfig == nil {
-			kibanaConfig = common.NewConfig()
+		// Initialize kibana config. If username and password is set in elasticsearch output config but not in kibana,
+		// initKibanaConfig will attach the ussername and password into kibana config as a part of the initialization.
+		kibanaConfig, err := initKibanaConfig(b.Config)
+		if err != nil {
+			return fmt.Errorf("error initKibanaConfig: %v", err)
 		}
 
 		client, err := kibana.NewKibanaClient(kibanaConfig)
@@ -720,7 +716,7 @@ func (b *Beat) loadDashboards(ctx context.Context, force bool) error {
 		}
 
 		err = dashboards.ImportDashboards(ctx, b.Info, paths.Resolve(paths.Home, ""),
-			kibanaConfig, esConfig, b.Config.Dashboards, nil, pattern)
+			kibanaConfig, b.Config.Dashboards, nil, pattern)
 		if err != nil {
 			return errw.Wrap(err, "Error importing Kibana dashboards")
 		}
@@ -886,3 +882,29 @@ func LoadKeystore(cfg *common.Config, name string) (keystore.Keystore, error) {
 	defaultPathConfig := paths.Resolve(paths.Data, fmt.Sprintf("%s.keystore", name))
 	return keystore.Factory(keystoreCfg, defaultPathConfig)
 }
+
+func initKibanaConfig(beatConfig beatConfig) (*common.Config, error) {
+	var esConfig *common.Config
+	if beatConfig.Output.Name() == "elasticsearch" {
+		esConfig = beatConfig.Output.Config()
+	}
+
+	// init kibana config object
+	kibanaConfig := beatConfig.Kibana
+	if kibanaConfig == nil {
+		kibanaConfig = common.NewConfig()
+	}
+
+	if esConfig.Enabled() {
+		username, _ := esConfig.String("username", -1)
+		password, _ := esConfig.String("password", -1)
+
+		if !kibanaConfig.HasField("username") && username != "" {
+			kibanaConfig.SetString("username", -1, username)
+		}
+		if !kibanaConfig.HasField("password") && password != "" {
+			kibanaConfig.SetString("password", -1, password)
+		}
+	}
+	return kibanaConfig, nil
+}

libbeat/cmd/instance/beat_test.go

@@ -22,6 +22,8 @@ package instance
 import (
 	"testing"
 
+	"github.com/elastic/beats/libbeat/cfgfile"
+
 	"github.com/gofrs/uuid"
 	"github.com/stretchr/testify/assert"
 )
@@ -63,3 +65,30 @@ func TestNewInstanceUUID(t *testing.T) {
 	}
 	assert.NotEqual(t, b.Info.ID, differentUUID)
 }
+
+func TestInitKibanaConfig(t *testing.T) {
+	b, err := NewBeat("filebeat", "testidx", "0.9")
+	if err != nil {
+		panic(err)
+	}
+
+	assert.Equal(t, "filebeat", b.Info.Beat)
+	assert.Equal(t, "testidx", b.Info.IndexPrefix)
+	assert.Equal(t, "0.9", b.Info.Version)
+
+	cfg, err := cfgfile.Load("../test/filebeat_test.yml", nil)
+	err = cfg.Unpack(&b.Config)
+	assert.NoError(t, err)
+
+	kibanaConfig, err := initKibanaConfig(b.Config)
+	assert.NoError(t, err)
+	username, err := kibanaConfig.String("username", -1)
+	password, err := kibanaConfig.String("password", -1)
+	protocol, err := kibanaConfig.String("protocol", -1)
+	host, err := kibanaConfig.String("host", -1)
+
+	assert.Equal(t, "elastic-test-username", username)
+	assert.Equal(t, "elastic-test-password", password)
+	assert.Equal(t, "https", protocol)
+	assert.Equal(t, "127.0.0.1:5601", host)
+}

libbeat/cmd/test/filebeat_test.yml

@@ -0,0 +1,26 @@
+#============================== Kibana =====================================
+
+# Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API.
+# This requires a Kibana endpoint configuration.
+setup.kibana:
+
+  # Kibana Host
+  # Scheme and port can be left out and will be set to the default (http and 5601)
+  # In case you specify and additional path, the scheme is required: http://localhost:5601/path
+  # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601
+  host: "127.0.0.1:5601"
+  protocol: https
+
+#================================ Outputs =====================================
+
+# Configure what output to use when sending the data collected by the beat.
+
+#-------------------------- Elasticsearch output ------------------------------
+output.elasticsearch:
+  # Array of hosts to connect to.
+  hosts: ["127.0.0.1:9200"]
+
+  # Optional protocol and basic auth credentials.
+  username: "elastic-test-username"
+  password: "elastic-test-password"
+  protocal: "https"

libbeat/dashboards/dashboards.go

@@ -33,7 +33,7 @@ import (
 func ImportDashboards(
 	ctx context.Context,
 	beatInfo beat.Info, homePath string,
-	kibanaConfig, esConfig, dashboardsConfig *common.Config,
+	kibanaConfig, dashboardsConfig *common.Config,
 	msgOutputter MessageOutputter,
 	pattern common.MapStr,
 ) error {
@@ -50,18 +50,6 @@ func ImportDashboards(
 		return err
 	}
 
-	if esConfig.Enabled() {
-		username, _ := esConfig.String("username", -1)
-		password, _ := esConfig.String("password", -1)
-
-		if !kibanaConfig.HasField("username") && username != "" {
-			kibanaConfig.SetString("username", -1, username)
-		}
-		if !kibanaConfig.HasField("password") && password != "" {
-			kibanaConfig.SetString("password", -1, password)
-		}
-	}
-
 	if !kibanaConfig.Enabled() {
 		return errors.New("kibana configuration missing for loading dashboards.")
 	}