[Elastic-Agent] Agent push ECS meta to fleet (#17894)

[Elastic-Agent] Agent push ECS meta to fleet (#17894)

x-pack/elastic-agent/CHANGELOG.asciidoc

@@ -36,3 +36,4 @@
 - Display the stability of the agent at enroll and start.  {pull}17336[17336]
 - Expose stream.* variables in events {pull}17468[17468]
 - Monitoring configuration reloadable {pull}17855[17855]
+- Pack ECS metadata to request payload send to fleet {pull}17894[17894]

x-pack/elastic-agent/pkg/agent/application/enroll_cmd.go

@@ -141,7 +141,7 @@ func (c *EnrollCmd) Execute() error {
 
 	metadata, err := metadata()
 	if err != nil {
-		return errors.New(err, "acquiring hostname")
+		return errors.New(err, "acquiring metadata failed")
 	}
 
 	r := &fleetapi.EnrollRequest{

x-pack/elastic-agent/pkg/agent/application/filters/constraints_filter.go

@@ -6,7 +6,6 @@ package filters
 
 import (
 	"fmt"
-	"runtime"
 
 	"github.com/Masterminds/semver"
 
@@ -15,8 +14,6 @@ import (
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/transpiler"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/boolexp"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger"
-	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/release"
-	"github.com/elastic/go-sysinfo"
 )
 
 const (
@@ -25,24 +22,6 @@ const (
 	validateVersionFuncName = "validate_version"
 )
 
-// List of variables available to be used in constraint definitions.
-const (
-	// `agent.id` is a generated (in standalone) or assigned (in fleet) agent identifier.
-	agentIDKey = "agent.id"
-	// `agent.version` specifies current version of an agent.
-	agentVersionKey = "agent.version"
-	// `host.architecture` defines architecture of a host (e.g. x86_64, arm, ppc, mips).
-	hostArchKey = "host.architecture"
-	// `os.family` defines a family of underlying operating system (e.g. redhat, debian, freebsd, windows).
-	osFamilyKey = "os.family"
-	// `os.kernel` specifies current version of a kernel in a semver format.
-	osKernelKey = "os.kernel"
-	// `os.platform` specifies platform agent is running on (e.g. centos, ubuntu, windows).
-	osPlatformKey = "os.platform"
-	// `os.version` specifies version of underlying operating system (e.g. 10.12.6).
-	osVersionKey = "os.version"
-)
-
 var (
 	boolexpVarStore    *constraintVarStore
 	boolexpMethodsRegs *boolexp.MethodsReg
@@ -245,30 +224,20 @@ func newVarStore() (*constraintVarStore, error) {
 }
 
 func initVarStore(store *constraintVarStore) error {
-	sysInfo, err := sysinfo.Host()
+	agentInfo, err := info.NewAgentInfo()
 	if err != nil {
 		return err
 	}
 
-	agentInfo, err := info.NewAgentInfo()
+	meta, err := agentInfo.ECSMetadata()
 	if err != nil {
-		return err
+		return errors.New(err, "failed to gather host metadata")
 	}
 
-	info := sysInfo.Info()
-
-	// 	Agent
-	store.vars[agentIDKey] = agentInfo.AgentID()
-	store.vars[agentVersionKey] = release.Version()
-
-	// Host
-	store.vars[hostArchKey] = info.Architecture
-
-	// Operating system
-	store.vars[osFamilyKey] = runtime.GOOS
-	store.vars[osKernelKey] = info.KernelVersion
-	store.vars[osPlatformKey] = info.OS.Family
-	store.vars[osVersionKey] = info.OS.Version
+	// keep existing, overwrite gathered
+	for k, v := range meta {
+		store.vars[k] = v
+	}
 
 	return nil
 }

x-pack/elastic-agent/pkg/agent/application/fleet_gateway.go

@@ -182,6 +182,8 @@ func (f *fleetGateway) execute(ctx context.Context) (*fleetapi.CheckinResponse,
 	var metaData map[string]interface{}
 	if m, err := metadata(); err == nil {
 		metaData = m
+	} else {
+		f.log.Error(errors.New("failed to load metadata", err))
 	}
 
 	// checkin

x-pack/elastic-agent/pkg/agent/application/info/agent_metadata.go

@@ -0,0 +1,115 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package info
+
+import (
+	"fmt"
+	"os"
+	"runtime"
+	"strings"
+
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/release"
+	"github.com/elastic/go-sysinfo"
+	"github.com/elastic/go-sysinfo/types"
+)
+
+// List of variables available to be used in constraint definitions.
+const (
+	// `agent.id` is a generated (in standalone) or assigned (in fleet) agent identifier.
+	agentIDKey = "agent.id"
+	// `agent.version` specifies current version of an agent.
+	agentVersionKey = "agent.version"
+
+	// `os.family` defines a family of underlying operating system (e.g. redhat, debian, freebsd, windows).
+	osFamilyKey = "os.family"
+	// `os.kernel` specifies current version of a kernel in a semver format.
+	osKernelKey = "os.kernel"
+	// `os.platform` specifies platform agent is running on (e.g. centos, ubuntu, windows).
+	osPlatformKey = "os.platform"
+	// `os.version` specifies version of underlying operating system (e.g. 10.12.6).
+	osVersionKey = "os.version"
+	// `os.name` is a operating system name.
+	// Currently we just normalize the name (i.e. macOS, Windows, Linux). See https://www.elastic.co/guide/en/ecs/current/ecs-os.html
+	osNameKey = "os.name"
+	// `os.full` is an operating system name, including the version or code name.
+	osFullKey = "os.full"
+
+	// `host.architecture` defines architecture of a host (e.g. x86_64, arm, ppc, mips).
+	hostArchKey = "host.architecture"
+	// `host.hostname` specifies hostname of the host.
+	hostHostnameKey = "host.hostname"
+	// `host.name` specifies hostname of the host.
+	hostNameKey = "host.name"
+	// `host.id` is a Unique host id.
+	// As hostname is not always unique, use values that are meaningful in your environment.
+	hostIDKey = "host.id"
+	// `host.ip` is Host ip addresses.
+	// Note: this field should contain an array of values.
+	hostIPKey = "host.ip"
+	// `host.mac` is Host mac addresses.
+	// Note: this field should contain an array of values.
+	hostMACKey = "host.mac"
+)
+
+// ECSMetadata returns an agent ECS compliant metadata.
+func (i *AgentInfo) ECSMetadata() (map[string]interface{}, error) {
+	hostname, err := os.Hostname()
+	if err != nil {
+		return nil, err
+	}
+
+	// TODO: remove these values when kibana migrates to ECS
+	meta := map[string]interface{}{
+		"platform": runtime.GOOS,
+		"version":  release.Version(),
+		"host":     hostname,
+	}
+
+	sysInfo, err := sysinfo.Host()
+	if err != nil {
+		return nil, err
+	}
+
+	info := sysInfo.Info()
+
+	// Agent
+	meta[agentIDKey] = i.agentID
+	meta[agentVersionKey] = release.Version()
+
+	// Host
+	meta[hostArchKey] = info.Architecture
+	meta[hostHostnameKey] = hostname
+	meta[hostNameKey] = hostname
+	meta[hostIDKey] = info.UniqueID
+	meta[hostIPKey] = fmt.Sprintf("[%s]", strings.Join(info.IPs, ","))
+	meta[hostMACKey] = fmt.Sprintf("[%s]", strings.Join(info.MACs, ","))
+
+	// Operating system
+	meta[osFamilyKey] = runtime.GOOS
+	meta[osKernelKey] = info.KernelVersion
+	meta[osPlatformKey] = info.OS.Family
+	meta[osVersionKey] = info.OS.Version
+	meta[osNameKey] = info.OS.Name
+	meta[osFullKey] = getFullOSName(info)
+
+	return meta, nil
+}
+
+func getFullOSName(info types.HostInfo) string {
+	var sb strings.Builder
+	sb.WriteString(info.OS.Name)
+	if codeName := info.OS.Codename; codeName != "" {
+		sb.WriteString(" ")
+		sb.WriteString(codeName)
+	}
+
+	if version := info.OS.Version; version != "" {
+		sb.WriteString("(")
+		sb.WriteString(version)
+		sb.WriteString(")")
+	}
+
+	return sb.String()
+}

x-pack/elastic-agent/pkg/agent/application/local_meta.go

@@ -5,21 +5,20 @@
 package application
 
 import (
-	"os"
-	"runtime"
-
-	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/release"
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/info"
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors"
 )
 
 func metadata() (map[string]interface{}, error) {
-	hostname, err := os.Hostname()
+	agentInfo, err := info.NewAgentInfo()
 	if err != nil {
 		return nil, err
 	}
 
-	return map[string]interface{}{
-		"platform": runtime.GOOS,
-		"version":  release.Version(),
-		"host":     hostname,
-	}, nil
+	meta, err := agentInfo.ECSMetadata()
+	if err != nil {
+		return nil, errors.New(err, "failed to gather host metadata")
+	}
+
+	return meta, nil
 }

x-pack/elastic-agent/pkg/core/plugin/app/monitoring/beats/beats_monitor.go

@@ -46,7 +46,12 @@ func (b *Monitor) Reload(rawConfig *config.Config) error {
 		return err
 	}
 
-	b.config = cfg.MonitoringConfig
+	if cfg == nil || cfg.MonitoringConfig == nil {
+		b.config = &monitoringConfig.MonitoringConfig{}
+	} else {
+		b.config = cfg.MonitoringConfig
+	}
+
 	return nil
 }