[Elastic Agent] Adjust to Fleet Server connection information from Ki…

…bana in configuration (#24713)

* Adjust fleet.kibana.* to just fleet.*

* Switch from fleet.kibana.* to fleet.* for connection.

* Add changelog entry.

* Add check to ensure no possible panic in mapDict.

(cherry picked from commit 974f255)

x-pack/elastic-agent/.gitignore

@@ -2,6 +2,9 @@
 build/
 elastic-agent
 elastic-agent.dev.yml
+elastic-agent.yml.*
+fleet.yml
+fleet.yml.lock
 pkg/agent/operation/tests/scripts/short--1.0.yml
 pkg/agent/operation/tests/scripts/configurable-1.0-darwin-x86/configurable
 pkg/agent/operation/tests/scripts/servicable-1.0-darwin-x86/configurable

x-pack/elastic-agent/CHANGELOG.next.asciidoc

@@ -8,6 +8,7 @@
 ==== Breaking changes
 
 - Docker container is not run as root by default. {pull}21213[21213]
+- Read Fleet connection information from `fleet.*` instead of `fleet.kibana.*`. {pull}24713[24713]
 
 ==== Bugfixes
 - Fix rename *ConfigChange to *PolicyChange to align on changes in the UI. {pull}20779[20779]

x-pack/elastic-agent/pkg/agent/application/managed_mode.go

@@ -84,12 +84,12 @@ func newManaged(
 		return nil, err
 	}
 
-	client, err := client.NewAuthWithConfig(log, cfg.Fleet.AccessAPIKey, cfg.Fleet.Kibana)
+	client, err := client.NewAuthWithConfig(log, cfg.Fleet.AccessAPIKey, cfg.Fleet.Client)
 	if err != nil {
 		return nil, errors.New(err,
 			"fail to create API client",
 			errors.TypeNetwork,
-			errors.M(errors.MetaKeyURI, cfg.Fleet.Kibana.Host))
+			errors.M(errors.MetaKeyURI, cfg.Fleet.Client.Host))
 	}
 
 	sysInfo, err := sysinfo.Host()

x-pack/elastic-agent/pkg/agent/application/pipeline/actions/handlers/handler_action_policy_change.go

@@ -12,21 +12,20 @@ import (
 	"sort"
 	"time"
 
+	"gopkg.in/yaml.v2"
+
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/info"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/pipeline"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/pipeline/actions"
-	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/storage/store"
-	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/fleetapi/client"
-
-	"gopkg.in/yaml.v2"
-
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/configuration"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/storage"
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/storage/store"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/config"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/fleetapi"
-	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/kibana"
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/fleetapi/client"
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/remote"
 )
 
 const (
@@ -106,61 +105,61 @@ func (h *PolicyChange) handleKibanaHosts(ctx context.Context, c *config.Config)
 	if err != nil {
 		return errors.New(err, "could not parse the configuration from the policy", errors.TypeConfig)
 	}
-	if kibanaEqual(h.config.Fleet.Kibana, cfg.Fleet.Kibana) {
+	if clientEqual(h.config.Fleet.Client, cfg.Fleet.Client) {
 		// already the same hosts
 		return nil
 	}
 
 	// only set protocol/hosts as that is all Fleet currently sends
-	prevProtocol := h.config.Fleet.Kibana.Protocol
-	prevPath := h.config.Fleet.Kibana.Path
-	prevHosts := h.config.Fleet.Kibana.Hosts
-	h.config.Fleet.Kibana.Protocol = cfg.Fleet.Kibana.Protocol
-	h.config.Fleet.Kibana.Path = cfg.Fleet.Kibana.Path
-	h.config.Fleet.Kibana.Hosts = cfg.Fleet.Kibana.Hosts
+	prevProtocol := h.config.Fleet.Client.Protocol
+	prevPath := h.config.Fleet.Client.Path
+	prevHosts := h.config.Fleet.Client.Hosts
+	h.config.Fleet.Client.Protocol = cfg.Fleet.Client.Protocol
+	h.config.Fleet.Client.Path = cfg.Fleet.Client.Path
+	h.config.Fleet.Client.Hosts = cfg.Fleet.Client.Hosts
 
 	// rollback on failure
 	defer func() {
 		if err != nil {
-			h.config.Fleet.Kibana.Protocol = prevProtocol
-			h.config.Fleet.Kibana.Path = prevPath
-			h.config.Fleet.Kibana.Hosts = prevHosts
+			h.config.Fleet.Client.Protocol = prevProtocol
+			h.config.Fleet.Client.Path = prevPath
+			h.config.Fleet.Client.Hosts = prevHosts
 		}
 	}()
 
-	client, err := client.NewAuthWithConfig(h.log, h.config.Fleet.AccessAPIKey, h.config.Fleet.Kibana)
+	client, err := client.NewAuthWithConfig(h.log, h.config.Fleet.AccessAPIKey, h.config.Fleet.Client)
 	if err != nil {
 		return errors.New(
 			err, "fail to create API client with updated hosts",
-			errors.TypeNetwork, errors.M("hosts", h.config.Fleet.Kibana.Hosts))
+			errors.TypeNetwork, errors.M("hosts", h.config.Fleet.Client.Hosts))
 	}
 	ctx, cancel := context.WithTimeout(ctx, apiStatusTimeout)
 	defer cancel()
 	_, err = client.Send(ctx, "GET", "/api/status", nil, nil, nil)
 	if err != nil {
 		return errors.New(
 			err, "fail to communicate with updated API client hosts",
-			errors.TypeNetwork, errors.M("hosts", h.config.Fleet.Kibana.Hosts))
+			errors.TypeNetwork, errors.M("hosts", h.config.Fleet.Client.Hosts))
 	}
 	reader, err := fleetToReader(h.agentInfo, h.config)
 	if err != nil {
 		return errors.New(
 			err, "fail to persist updated API client hosts",
-			errors.TypeUnexpected, errors.M("hosts", h.config.Fleet.Kibana.Hosts))
+			errors.TypeUnexpected, errors.M("hosts", h.config.Fleet.Client.Hosts))
 	}
 	err = h.store.Save(reader)
 	if err != nil {
 		return errors.New(
 			err, "fail to persist updated API client hosts",
-			errors.TypeFilesystem, errors.M("hosts", h.config.Fleet.Kibana.Hosts))
+			errors.TypeFilesystem, errors.M("hosts", h.config.Fleet.Client.Hosts))
 	}
 	for _, setter := range h.setters {
 		setter.SetClient(client)
 	}
 	return nil
 }
 
-func kibanaEqual(k1 *kibana.Config, k2 *kibana.Config) bool {
+func clientEqual(k1 remote.Config, k2 remote.Config) bool {
 	if k1.Protocol != k2.Protocol {
 		return false
 	}

x-pack/elastic-agent/pkg/agent/application/pipeline/emitter/modifiers/fleet_decorator.go

@@ -32,37 +32,50 @@ func InjectFleet(cfg *config.Config, hostInfo types.HostInfo, agentInfo *info.Ag
 		if err != nil {
 			return err
 		}
-		token, ok := transpiler.Lookup(ast, "fleet.access_api_key")
+		fleet, ok := transpiler.Lookup(ast, "fleet")
 		if !ok {
-			return fmt.Errorf("failed to get api key from fleet config")
-		}
-
-		kbn, ok := transpiler.Lookup(ast, "fleet.kibana")
-		if !ok {
-			return fmt.Errorf("failed to get kibana config key from fleet config")
+			return fmt.Errorf("failed to get fleet from config")
 		}
 
+		// copy top-level agent.* into fleet.agent.* (this gets sent to Applications in this structure)
 		agent, ok := transpiler.Lookup(ast, "agent")
 		if !ok {
-			return fmt.Errorf("failed to get agent key from fleet config")
+			return fmt.Errorf("failed to get agent key from config")
+		}
+		if err := transpiler.Insert(ast, agent, "fleet"); err != nil {
+			return err
 		}
 
+		// ensure that the agent.logging.level is present
 		if _, found := transpiler.Lookup(ast, "agent.logging.level"); !found {
 			transpiler.Insert(ast, transpiler.NewKey("level", transpiler.NewStrVal(logLevel)), "agent.logging")
 		}
 
+		// fleet.host to Agent can be the host to connect to Fleet Server, but to Applications it should
+		// be the fleet.host.id. move fleet.host to fleet.hosts if fleet.hosts doesn't exist
+		if _, ok := transpiler.Lookup(ast, "fleet.hosts"); !ok {
+			if host, ok := transpiler.Lookup(ast, "fleet.host"); ok {
+				if key, ok := host.(*transpiler.Key); ok {
+					if value, ok := key.Value().(*transpiler.StrVal); ok {
+						hosts := transpiler.NewList([]transpiler.Node{transpiler.NewStrVal(value.String())})
+						if err := transpiler.Insert(ast, hosts, "fleet.hosts"); err != nil {
+							return err
+						}
+					}
+				}
+			}
+		}
+
+		// inject host.* into fleet.host.* (this gets sent to Applications in this structure)
 		host := transpiler.NewKey("host", transpiler.NewDict([]transpiler.Node{
 			transpiler.NewKey("id", transpiler.NewStrVal(hostInfo.UniqueID)),
 		}))
-
-		nodes := []transpiler.Node{agent, token, kbn, host}
-		server, ok := transpiler.Lookup(ast, "fleet.server")
-		if ok {
-			nodes = append(nodes, server)
+		if err := transpiler.Insert(ast, host, "fleet"); err != nil {
+			return err
 		}
-		fleet := transpiler.NewDict(nodes)
 
-		err = transpiler.Insert(rootAst, fleet, "fleet")
+		// inject fleet.* from local AST to the rootAST so its present when sending to Applications.
+		err = transpiler.Insert(rootAst, fleet.Value().(transpiler.Node), "fleet")
 		if err != nil {
 			return err
 		}

x-pack/elastic-agent/pkg/agent/cmd/enroll_cmd.go

@@ -31,8 +31,8 @@ import (
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/process"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/fleetapi"
 	fleetclient "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/fleetapi/client"
-	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/kibana"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/release"
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/remote"
 )
 
 const (
@@ -58,7 +58,7 @@ type enrollCmd struct {
 	options      *enrollCmdOption
 	client       fleetclient.Sender
 	configStore  saver
-	kibanaConfig *kibana.Config
+	remoteConfig remote.Config
 	agentProc    *process.Info
 }
 
@@ -88,13 +88,13 @@ type enrollCmdOption struct {
 	FleetServer          enrollCmdFleetServerOption
 }
 
-func (e *enrollCmdOption) kibanaConfig() (*kibana.Config, error) {
-	cfg, err := kibana.NewConfigFromURL(e.URL)
+func (e *enrollCmdOption) remoteConfig() (remote.Config, error) {
+	cfg, err := remote.NewConfigFromURL(e.URL)
 	if err != nil {
-		return nil, err
+		return remote.Config{}, err
 	}
-	if cfg.Protocol == kibana.ProtocolHTTP && !e.Insecure {
-		return nil, fmt.Errorf("connection to Kibana is insecure, strongly recommended to use a secure connection (override with --insecure)")
+	if cfg.Protocol == remote.ProtocolHTTP && !e.Insecure {
+		return remote.Config{}, fmt.Errorf("connection to Kibana is insecure, strongly recommended to use a secure connection (override with --insecure)")
 	}
 
 	// Add any SSL options from the CLI.
@@ -160,15 +160,15 @@ func (c *enrollCmd) Execute(ctx context.Context) error {
 		}
 	}
 
-	c.kibanaConfig, err = c.options.kibanaConfig()
+	c.remoteConfig, err = c.options.remoteConfig()
 	if err != nil {
 		return errors.New(
 			err, "Error",
 			errors.TypeConfig,
 			errors.M(errors.MetaKeyURI, c.options.URL))
 	}
 
-	c.client, err = fleetclient.NewWithConfig(c.log, c.kibanaConfig)
+	c.client, err = fleetclient.NewWithConfig(c.log, c.remoteConfig)
 	if err != nil {
 		return errors.New(
 			err, "Error",
@@ -364,7 +364,7 @@ func (c *enrollCmd) enroll(ctx context.Context) error {
 			errors.TypeNetwork)
 	}
 
-	fleetConfig, err := createFleetConfigFromEnroll(resp.Item.AccessAPIKey, c.kibanaConfig)
+	fleetConfig, err := createFleetConfigFromEnroll(resp.Item.AccessAPIKey, c.remoteConfig)
 	if err != nil {
 		return err
 	}
@@ -678,11 +678,11 @@ func createFleetServerBootstrapConfig(connStr string, policyID string, host stri
 	return cfg, nil
 }
 
-func createFleetConfigFromEnroll(accessAPIKey string, kbn *kibana.Config) (*configuration.FleetAgentConfig, error) {
+func createFleetConfigFromEnroll(accessAPIKey string, cli remote.Config) (*configuration.FleetAgentConfig, error) {
 	cfg := configuration.DefaultFleetAgentConfig()
 	cfg.Enabled = true
 	cfg.AccessAPIKey = accessAPIKey
-	cfg.Kibana = kbn
+	cfg.Client = cli
 
 	if err := cfg.Valid(); err != nil {
 		return nil, errors.New(err, "invalid enrollment options", errors.TypeConfig)

x-pack/elastic-agent/pkg/agent/cmd/enroll_cmd_test.go

@@ -155,9 +155,9 @@ func TestEnroll(t *testing.T) {
 
 			require.NoError(t, err)
 			require.Equal(t, "my-access-api-key", config.AccessAPIKey)
-			require.Equal(t, host, config.Kibana.Host)
-			require.Equal(t, "", config.Kibana.Username)
-			require.Equal(t, "", config.Kibana.Password)
+			require.Equal(t, host, config.Client.Host)
+			require.Equal(t, "", config.Client.Username)
+			require.Equal(t, "", config.Client.Password)
 		},
 	))
 
@@ -215,9 +215,9 @@ func TestEnroll(t *testing.T) {
 
 			require.NoError(t, err)
 			require.Equal(t, "my-access-api-key", config.AccessAPIKey)
-			require.Equal(t, host, config.Kibana.Host)
-			require.Equal(t, "", config.Kibana.Username)
-			require.Equal(t, "", config.Kibana.Password)
+			require.Equal(t, host, config.Client.Host)
+			require.Equal(t, "", config.Client.Username)
+			require.Equal(t, "", config.Client.Password)
 		},
 	))
 
@@ -275,9 +275,9 @@ func TestEnroll(t *testing.T) {
 
 			require.NoError(t, err)
 			require.Equal(t, "my-access-api-key", config.AccessAPIKey)
-			require.Equal(t, host, config.Kibana.Host)
-			require.Equal(t, "", config.Kibana.Username)
-			require.Equal(t, "", config.Kibana.Password)
+			require.Equal(t, host, config.Client.Host)
+			require.Equal(t, "", config.Client.Username)
+			require.Equal(t, "", config.Client.Password)
 		},
 	))
 

x-pack/elastic-agent/pkg/agent/configuration/fleet.go

@@ -6,7 +6,7 @@ package configuration
 
 import (
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors"
-	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/kibana"
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/remote"
 	fleetreporterConfig "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/reporter/fleet/config"
 )
 
@@ -15,7 +15,7 @@ import (
 type FleetAgentConfig struct {
 	Enabled      bool                        `config:"enabled" yaml:"enabled"`
 	AccessAPIKey string                      `config:"access_api_key" yaml:"access_api_key"`
-	Kibana       *kibana.Config              `config:"kibana" yaml:"kibana"`
+	Client       remote.Config               `config:",inline" yaml:",inline"`
 	Reporting    *fleetreporterConfig.Config `config:"reporting" yaml:"reporting"`
 	Info         *AgentInfo                  `config:"agent" yaml:"agent"`
 	Server       *FleetServerConfig          `config:"server" yaml:"server,omitempty"`
@@ -33,8 +33,8 @@ func (e *FleetAgentConfig) Valid() error {
 			return errors.New("empty access token", errors.TypeConfig)
 		}
 
-		if e.Kibana == nil || len(e.Kibana.Host) == 0 {
-			return errors.New("missing Kibana host configuration", errors.TypeConfig)
+		if len(e.Client.Host) == 0 {
+			return errors.New("missing fleet host configuration", errors.TypeConfig)
 		}
 	}
 
@@ -45,7 +45,7 @@ func (e *FleetAgentConfig) Valid() error {
 func DefaultFleetAgentConfig() *FleetAgentConfig {
 	return &FleetAgentConfig{
 		Enabled:   false,
-		Kibana:    kibana.DefaultClientConfig(),
+		Client:    remote.DefaultClientConfig(),
 		Reporting: fleetreporterConfig.DefaultConfig(),
 		Info:      &AgentInfo{},
 	}

x-pack/elastic-agent/pkg/agent/program/program_test.go

@@ -395,6 +395,10 @@ func TestConfiguration(t *testing.T) {
 		// 	programs: []string{"journalbeat"},
 		// 	expected: 1,
 		// },
+		"fleet_server": {
+			programs: []string{"fleet-server"},
+			expected: 1,
+		},
 		"synthetics_config": {
 			programs: []string{"heartbeat"},
 			expected: 1,