Cherry-pick #17894 to 7.x: Agent push ECS meta to fleet (#17921)

* [Elastic-Agent] Agent push ECS meta to fleet (#17894)

[Elastic-Agent] Agent push ECS meta to fleet (#17894)

* [Elastic-Agent] ECS compliant Elastic agent metadata sent to fleet  (#18006)

* use meta object

* changelog

* agent.* => elastic.agent.*

x-pack/elastic-agent/CHANGELOG.asciidoc

@@ -38,6 +38,7 @@
 - Display the stability of the agent at enroll and start.  {pull}17336[17336]
 - Expose stream.* variables in events {pull}17468[17468]
 - Monitoring configuration reloadable {pull}17855[17855]
+- Pack ECS metadata to request payload send to fleet {pull}17894[17894]
 - Allow CLI overrides of paths {pull}17781[17781]
 - Enable Filebeat input: S3, Azureeventhub, cloudfoundry, httpjson, netflow, o365audit. {pull}17909[17909]
 - Use data subfolder as default for process logs {pull}17960[17960]

x-pack/elastic-agent/pkg/agent/application/enroll_cmd.go

@@ -141,7 +141,7 @@ func (c *EnrollCmd) Execute() error {
 
 	metadata, err := metadata()
 	if err != nil {
-		return errors.New(err, "acquiring hostname")
+		return errors.New(err, "acquiring metadata failed")
 	}
 
 	r := &fleetapi.EnrollRequest{

x-pack/elastic-agent/pkg/agent/application/filters/constraints_filter.go

@@ -6,7 +6,6 @@ package filters
 
 import (
 	"fmt"
-	"runtime"
 
 	"github.com/Masterminds/semver"
 
@@ -15,8 +14,6 @@ import (
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/transpiler"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/boolexp"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger"
-	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/release"
-	"github.com/elastic/go-sysinfo"
 )
 
 const (
@@ -25,24 +22,6 @@ const (
 	validateVersionFuncName = "validate_version"
 )
 
-// List of variables available to be used in constraint definitions.
-const (
-	// `agent.id` is a generated (in standalone) or assigned (in fleet) agent identifier.
-	agentIDKey = "agent.id"
-	// `agent.version` specifies current version of an agent.
-	agentVersionKey = "agent.version"
-	// `host.architecture` defines architecture of a host (e.g. x86_64, arm, ppc, mips).
-	hostArchKey = "host.architecture"
-	// `os.family` defines a family of underlying operating system (e.g. redhat, debian, freebsd, windows).
-	osFamilyKey = "os.family"
-	// `os.kernel` specifies current version of a kernel in a semver format.
-	osKernelKey = "os.kernel"
-	// `os.platform` specifies platform agent is running on (e.g. centos, ubuntu, windows).
-	osPlatformKey = "os.platform"
-	// `os.version` specifies version of underlying operating system (e.g. 10.12.6).
-	osVersionKey = "os.version"
-)
-
 var (
 	boolexpVarStore    *constraintVarStore
 	boolexpMethodsRegs *boolexp.MethodsReg
@@ -245,30 +224,20 @@ func newVarStore() (*constraintVarStore, error) {
 }
 
 func initVarStore(store *constraintVarStore) error {
-	sysInfo, err := sysinfo.Host()
+	agentInfo, err := info.NewAgentInfo()
 	if err != nil {
 		return err
 	}
 
-	agentInfo, err := info.NewAgentInfo()
+	meta, err := agentInfo.ECSMetadataFlatMap()
 	if err != nil {
-		return err
+		return errors.New(err, "failed to gather host metadata")
 	}
 
-	info := sysInfo.Info()
-
-	// 	Agent
-	store.vars[agentIDKey] = agentInfo.AgentID()
-	store.vars[agentVersionKey] = release.Version()
-
-	// Host
-	store.vars[hostArchKey] = info.Architecture
-
-	// Operating system
-	store.vars[osFamilyKey] = runtime.GOOS
-	store.vars[osKernelKey] = info.KernelVersion
-	store.vars[osPlatformKey] = info.OS.Family
-	store.vars[osVersionKey] = info.OS.Version
+	// keep existing, overwrite gathered
+	for k, v := range meta {
+		store.vars[k] = v
+	}
 
 	return nil
 }

x-pack/elastic-agent/pkg/agent/application/fleet_gateway.go

@@ -179,16 +179,16 @@ func (f *fleetGateway) execute(ctx context.Context) (*fleetapi.CheckinResponse,
 	// get events
 	ee, ack := f.reporter.Events()
 
-	var metaData map[string]interface{}
-	if m, err := metadata(); err == nil {
-		metaData = m
+	ecsMeta, err := metadata()
+	if err != nil {
+		f.log.Error(errors.New("failed to load metadata", err))
 	}
 
 	// checkin
 	cmd := fleetapi.NewCheckinCmd(f.agentInfo, f.client)
 	req := &fleetapi.CheckinRequest{
 		Events:   ee,
-		Metadata: metaData,
+		Metadata: ecsMeta,
 	}
 
 	resp, err := cmd.Execute(ctx, req)

x-pack/elastic-agent/pkg/agent/application/info/agent_metadata.go

@@ -0,0 +1,202 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package info
+
+import (
+	"fmt"
+	"os"
+	"runtime"
+	"strings"
+
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/release"
+	"github.com/elastic/go-sysinfo"
+	"github.com/elastic/go-sysinfo/types"
+)
+
+// ECSMeta is a collection of agent related metadata in ECS compliant object form.
+type ECSMeta struct {
+	Agent *AgentECSMeta  `json:"elastic.agent"`
+	Host  *HostECSMeta   `json:"host"`
+	OS    *SystemECSMeta `json:"os"`
+}
+
+// AgentECSMeta is a collection of agent metadata in ECS compliant object form.
+type AgentECSMeta struct {
+	// ID is a generated (in standalone) or assigned (in fleet) agent identifier.
+	ID string `json:"id"`
+	// Version specifies current version of an agent.
+	Version string `json:"version"`
+}
+
+// SystemECSMeta is a collection of operating system metadata in ECS compliant object form.
+type SystemECSMeta struct {
+	// Family defines a family of underlying operating system (e.g. redhat, debian, freebsd, windows).
+	Family string `json:"family"`
+	// Kernel specifies current version of a kernel in a semver format.
+	Kernel string `json:"kernel"`
+	// Platform specifies platform agent is running on (e.g. centos, ubuntu, windows).
+	Platform string `json:"platform"`
+	// Version specifies version of underlying operating system (e.g. 10.12.6).
+	Version string `json:"version"`
+	// Name is a operating system name.
+	// Currently we just normalize the name (i.e. macOS, Windows, Linux). See https://www.elastic.co/guide/en/ecs/current/ecs-html
+	Name string `json:"name"`
+	// Full is an operating system name, including the version or code name.
+	FullName string `json:"full"`
+}
+
+// HostECSMeta is a collection of host metadata in ECS compliant object form.
+type HostECSMeta struct {
+	// Arch defines architecture of a host (e.g. x86_64, arm, ppc, mips).
+	Arch string `json:"architecture"`
+	// Hostname specifies hostname of the host.
+	Hostname string `json:"hostname"`
+	// Name specifies hostname of the host.
+	Name string `json:"name"`
+	// ID is a Unique host id.
+	// As hostname is not always unique, use values that are meaningful in your environment.
+	ID string `json:"id"`
+	// IP is Host ip addresses.
+	// Note: this field should contain an array of values.
+	IP []string `json:"ip"`
+	// Mac is Host mac addresses.
+	// Note: this field should contain an array of values.
+	MAC []string `json:"mac"`
+}
+
+// List of variables available to be used in constraint definitions.
+const (
+	// `agent.id` is a generated (in standalone) or assigned (in fleet) agent identifier.
+	agentIDKey = "agent.id"
+	// `agent.version` specifies current version of an agent.
+	agentVersionKey = "agent.version"
+
+	// `os.family` defines a family of underlying operating system (e.g. redhat, debian, freebsd, windows).
+	osFamilyKey = "os.family"
+	// `os.kernel` specifies current version of a kernel in a semver format.
+	osKernelKey = "os.kernel"
+	// `os.platform` specifies platform agent is running on (e.g. centos, ubuntu, windows).
+	osPlatformKey = "os.platform"
+	// `os.version` specifies version of underlying operating system (e.g. 10.12.6).
+	osVersionKey = "os.version"
+	// `os.name` is a operating system name.
+	// Currently we just normalize the name (i.e. macOS, Windows, Linux). See https://www.elastic.co/guide/en/ecs/current/ecs-os.html
+	osNameKey = "os.name"
+	// `os.full` is an operating system name, including the version or code name.
+	osFullKey = "os.full"
+
+	// `host.architecture` defines architecture of a host (e.g. x86_64, arm, ppc, mips).
+	hostArchKey = "host.architecture"
+	// `host.hostname` specifies hostname of the host.
+	hostHostnameKey = "host.hostname"
+	// `host.name` specifies hostname of the host.
+	hostNameKey = "host.name"
+	// `host.id` is a Unique host id.
+	// As hostname is not always unique, use values that are meaningful in your environment.
+	hostIDKey = "host.id"
+	// `host.ip` is Host ip addresses.
+	// Note: this field should contain an array of values.
+	hostIPKey = "host.ip"
+	// `host.mac` is Host mac addresses.
+	// Note: this field should contain an array of values.
+	hostMACKey = "host.mac"
+)
+
+// ECSMetadata returns an agent ECS compliant metadata.
+func (i *AgentInfo) ECSMetadata() (*ECSMeta, error) {
+	hostname, err := os.Hostname()
+	if err != nil {
+		return nil, err
+	}
+
+	sysInfo, err := sysinfo.Host()
+	if err != nil {
+		return nil, err
+	}
+
+	info := sysInfo.Info()
+
+	return &ECSMeta{
+		Agent: &AgentECSMeta{
+			ID:      i.agentID,
+			Version: release.Version(),
+		},
+		Host: &HostECSMeta{
+			Arch:     info.Architecture,
+			Hostname: hostname,
+			Name:     hostname,
+			ID:       info.UniqueID,
+			IP:       info.IPs,
+			MAC:      info.MACs,
+		},
+
+		// Operating system
+		OS: &SystemECSMeta{
+			Family:   runtime.GOOS,
+			Kernel:   info.KernelVersion,
+			Platform: info.OS.Family,
+			Version:  info.OS.Version,
+			Name:     info.OS.Name,
+			FullName: getFullOSName(info),
+		},
+	}, nil
+}
+
+// ECSMetadataFlatMap returns an agent ECS compliant metadata in a form of flattened map.
+func (i *AgentInfo) ECSMetadataFlatMap() (map[string]interface{}, error) {
+	hostname, err := os.Hostname()
+	if err != nil {
+		return nil, err
+	}
+
+	// TODO: remove these values when kibana migrates to ECS
+	meta := make(map[string]interface{})
+
+	sysInfo, err := sysinfo.Host()
+	if err != nil {
+		return nil, err
+	}
+
+	info := sysInfo.Info()
+
+	// Agent
+	meta[agentIDKey] = i.agentID
+	meta[agentVersionKey] = release.Version()
+
+	// Host
+	meta[hostArchKey] = info.Architecture
+	meta[hostHostnameKey] = hostname
+	meta[hostNameKey] = hostname
+	meta[hostIDKey] = info.UniqueID
+	meta[hostIPKey] = fmt.Sprintf("[%s]", strings.Join(info.IPs, ","))
+	meta[hostMACKey] = fmt.Sprintf("[%s]", strings.Join(info.MACs, ","))
+
+	// Operating system
+	meta[osFamilyKey] = runtime.GOOS
+	meta[osKernelKey] = info.KernelVersion
+	meta[osPlatformKey] = info.OS.Family
+	meta[osVersionKey] = info.OS.Version
+	meta[osNameKey] = info.OS.Name
+	meta[osFullKey] = getFullOSName(info)
+
+	return meta, nil
+}
+
+func getFullOSName(info types.HostInfo) string {
+	var sb strings.Builder
+	sb.WriteString(info.OS.Name)
+	if codeName := info.OS.Codename; codeName != "" {
+		sb.WriteString(" ")
+		sb.WriteString(codeName)
+	}
+
+	if version := info.OS.Version; version != "" {
+		sb.WriteString("(")
+		sb.WriteString(version)
+		sb.WriteString(")")
+	}
+
+	return sb.String()
+}

x-pack/elastic-agent/pkg/agent/application/local_meta.go

@@ -5,21 +5,20 @@
 package application
 
 import (
-	"os"
-	"runtime"
-
-	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/release"
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/info"
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors"
 )
 
-func metadata() (map[string]interface{}, error) {
-	hostname, err := os.Hostname()
+func metadata() (*info.ECSMeta, error) {
+	agentInfo, err := info.NewAgentInfo()
 	if err != nil {
 		return nil, err
 	}
 
-	return map[string]interface{}{
-		"platform": runtime.GOOS,
-		"version":  release.Version(),
-		"host":     hostname,
-	}, nil
+	meta, err := agentInfo.ECSMetadata()
+	if err != nil {
+		return nil, errors.New(err, "failed to gather host metadata")
+	}
+
+	return meta, nil
 }

x-pack/elastic-agent/pkg/core/plugin/app/monitoring/beats/beats_monitor.go

@@ -46,7 +46,12 @@ func (b *Monitor) Reload(rawConfig *config.Config) error {
 		return err
 	}
 
-	b.config = cfg.MonitoringConfig
+	if cfg == nil || cfg.MonitoringConfig == nil {
+		b.config = &monitoringConfig.MonitoringConfig{}
+	} else {
+		b.config = cfg.MonitoringConfig
+	}
+
 	return nil
 }
 

x-pack/elastic-agent/pkg/fleetapi/checkin_cmd.go

@@ -13,15 +13,16 @@ import (
 	"net/http"
 	"time"
 
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/info"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors"
 )
 
 const checkingPath = "/api/ingest_manager/fleet/agents/%s/checkin"
 
 // CheckinRequest consists of multiple events reported to fleet ui.
 type CheckinRequest struct {
-	Events   []SerializableEvent    `json:"events"`
-	Metadata map[string]interface{} `json:"local_metadata,omitempty"`
+	Events   []SerializableEvent `json:"events"`
+	Metadata *info.ECSMeta       `json:"local_metadata,omitempty"`
 }
 
 // SerializableEvent is a representation of the event to be send to the Fleet API via the checkin