From 757e7d134aba455caf12b603c2e32516899b1df5 Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Wed, 22 Apr 2020 16:47:53 +0200 Subject: [PATCH] [Elastic-Agent] Agent push ECS meta to fleet (#17894) [Elastic-Agent] Agent push ECS meta to fleet (#17894) --- x-pack/elastic-agent/CHANGELOG.asciidoc | 1 + .../pkg/agent/application/enroll_cmd.go | 2 +- .../application/filters/constraints_filter.go | 45 ++----- .../pkg/agent/application/fleet_gateway.go | 2 + .../agent/application/info/agent_metadata.go | 115 ++++++++++++++++++ .../pkg/agent/application/local_meta.go | 19 ++- .../app/monitoring/beats/beats_monitor.go | 7 +- 7 files changed, 141 insertions(+), 50 deletions(-) create mode 100644 x-pack/elastic-agent/pkg/agent/application/info/agent_metadata.go diff --git a/x-pack/elastic-agent/CHANGELOG.asciidoc b/x-pack/elastic-agent/CHANGELOG.asciidoc index edb9a65b141..b81acebc9e8 100644 --- a/x-pack/elastic-agent/CHANGELOG.asciidoc +++ b/x-pack/elastic-agent/CHANGELOG.asciidoc @@ -36,3 +36,4 @@ - Display the stability of the agent at enroll and start. {pull}17336[17336] - Expose stream.* variables in events {pull}17468[17468] - Monitoring configuration reloadable {pull}17855[17855] +- Pack ECS metadata to request payload send to fleet {pull}17894[17894] diff --git a/x-pack/elastic-agent/pkg/agent/application/enroll_cmd.go b/x-pack/elastic-agent/pkg/agent/application/enroll_cmd.go index b1743e92417..323937b080c 100644 --- a/x-pack/elastic-agent/pkg/agent/application/enroll_cmd.go +++ b/x-pack/elastic-agent/pkg/agent/application/enroll_cmd.go @@ -141,7 +141,7 @@ func (c *EnrollCmd) Execute() error { metadata, err := metadata() if err != nil { - return errors.New(err, "acquiring hostname") + return errors.New(err, "acquiring metadata failed") } r := &fleetapi.EnrollRequest{ diff --git a/x-pack/elastic-agent/pkg/agent/application/filters/constraints_filter.go b/x-pack/elastic-agent/pkg/agent/application/filters/constraints_filter.go index 9241f3dd3e6..2cb92cffd97 100644 --- a/x-pack/elastic-agent/pkg/agent/application/filters/constraints_filter.go +++ b/x-pack/elastic-agent/pkg/agent/application/filters/constraints_filter.go @@ -6,7 +6,6 @@ package filters import ( "fmt" - "runtime" "github.com/Masterminds/semver" @@ -15,8 +14,6 @@ import ( "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/transpiler" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/boolexp" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger" - "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/release" - "github.com/elastic/go-sysinfo" ) const ( @@ -25,24 +22,6 @@ const ( validateVersionFuncName = "validate_version" ) -// List of variables available to be used in constraint definitions. -const ( - // `agent.id` is a generated (in standalone) or assigned (in fleet) agent identifier. - agentIDKey = "agent.id" - // `agent.version` specifies current version of an agent. - agentVersionKey = "agent.version" - // `host.architecture` defines architecture of a host (e.g. x86_64, arm, ppc, mips). - hostArchKey = "host.architecture" - // `os.family` defines a family of underlying operating system (e.g. redhat, debian, freebsd, windows). - osFamilyKey = "os.family" - // `os.kernel` specifies current version of a kernel in a semver format. - osKernelKey = "os.kernel" - // `os.platform` specifies platform agent is running on (e.g. centos, ubuntu, windows). - osPlatformKey = "os.platform" - // `os.version` specifies version of underlying operating system (e.g. 10.12.6). - osVersionKey = "os.version" -) - var ( boolexpVarStore *constraintVarStore boolexpMethodsRegs *boolexp.MethodsReg @@ -245,30 +224,20 @@ func newVarStore() (*constraintVarStore, error) { } func initVarStore(store *constraintVarStore) error { - sysInfo, err := sysinfo.Host() + agentInfo, err := info.NewAgentInfo() if err != nil { return err } - agentInfo, err := info.NewAgentInfo() + meta, err := agentInfo.ECSMetadata() if err != nil { - return err + return errors.New(err, "failed to gather host metadata") } - info := sysInfo.Info() - - // Agent - store.vars[agentIDKey] = agentInfo.AgentID() - store.vars[agentVersionKey] = release.Version() - - // Host - store.vars[hostArchKey] = info.Architecture - - // Operating system - store.vars[osFamilyKey] = runtime.GOOS - store.vars[osKernelKey] = info.KernelVersion - store.vars[osPlatformKey] = info.OS.Family - store.vars[osVersionKey] = info.OS.Version + // keep existing, overwrite gathered + for k, v := range meta { + store.vars[k] = v + } return nil } diff --git a/x-pack/elastic-agent/pkg/agent/application/fleet_gateway.go b/x-pack/elastic-agent/pkg/agent/application/fleet_gateway.go index 97c1964f8ac..e4f021fdaad 100644 --- a/x-pack/elastic-agent/pkg/agent/application/fleet_gateway.go +++ b/x-pack/elastic-agent/pkg/agent/application/fleet_gateway.go @@ -182,6 +182,8 @@ func (f *fleetGateway) execute(ctx context.Context) (*fleetapi.CheckinResponse, var metaData map[string]interface{} if m, err := metadata(); err == nil { metaData = m + } else { + f.log.Error(errors.New("failed to load metadata", err)) } // checkin diff --git a/x-pack/elastic-agent/pkg/agent/application/info/agent_metadata.go b/x-pack/elastic-agent/pkg/agent/application/info/agent_metadata.go new file mode 100644 index 00000000000..79371e76600 --- /dev/null +++ b/x-pack/elastic-agent/pkg/agent/application/info/agent_metadata.go @@ -0,0 +1,115 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package info + +import ( + "fmt" + "os" + "runtime" + "strings" + + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/release" + "github.com/elastic/go-sysinfo" + "github.com/elastic/go-sysinfo/types" +) + +// List of variables available to be used in constraint definitions. +const ( + // `agent.id` is a generated (in standalone) or assigned (in fleet) agent identifier. + agentIDKey = "agent.id" + // `agent.version` specifies current version of an agent. + agentVersionKey = "agent.version" + + // `os.family` defines a family of underlying operating system (e.g. redhat, debian, freebsd, windows). + osFamilyKey = "os.family" + // `os.kernel` specifies current version of a kernel in a semver format. + osKernelKey = "os.kernel" + // `os.platform` specifies platform agent is running on (e.g. centos, ubuntu, windows). + osPlatformKey = "os.platform" + // `os.version` specifies version of underlying operating system (e.g. 10.12.6). + osVersionKey = "os.version" + // `os.name` is a operating system name. + // Currently we just normalize the name (i.e. macOS, Windows, Linux). See https://www.elastic.co/guide/en/ecs/current/ecs-os.html + osNameKey = "os.name" + // `os.full` is an operating system name, including the version or code name. + osFullKey = "os.full" + + // `host.architecture` defines architecture of a host (e.g. x86_64, arm, ppc, mips). + hostArchKey = "host.architecture" + // `host.hostname` specifies hostname of the host. + hostHostnameKey = "host.hostname" + // `host.name` specifies hostname of the host. + hostNameKey = "host.name" + // `host.id` is a Unique host id. + // As hostname is not always unique, use values that are meaningful in your environment. + hostIDKey = "host.id" + // `host.ip` is Host ip addresses. + // Note: this field should contain an array of values. + hostIPKey = "host.ip" + // `host.mac` is Host mac addresses. + // Note: this field should contain an array of values. + hostMACKey = "host.mac" +) + +// ECSMetadata returns an agent ECS compliant metadata. +func (i *AgentInfo) ECSMetadata() (map[string]interface{}, error) { + hostname, err := os.Hostname() + if err != nil { + return nil, err + } + + // TODO: remove these values when kibana migrates to ECS + meta := map[string]interface{}{ + "platform": runtime.GOOS, + "version": release.Version(), + "host": hostname, + } + + sysInfo, err := sysinfo.Host() + if err != nil { + return nil, err + } + + info := sysInfo.Info() + + // Agent + meta[agentIDKey] = i.agentID + meta[agentVersionKey] = release.Version() + + // Host + meta[hostArchKey] = info.Architecture + meta[hostHostnameKey] = hostname + meta[hostNameKey] = hostname + meta[hostIDKey] = info.UniqueID + meta[hostIPKey] = fmt.Sprintf("[%s]", strings.Join(info.IPs, ",")) + meta[hostMACKey] = fmt.Sprintf("[%s]", strings.Join(info.MACs, ",")) + + // Operating system + meta[osFamilyKey] = runtime.GOOS + meta[osKernelKey] = info.KernelVersion + meta[osPlatformKey] = info.OS.Family + meta[osVersionKey] = info.OS.Version + meta[osNameKey] = info.OS.Name + meta[osFullKey] = getFullOSName(info) + + return meta, nil +} + +func getFullOSName(info types.HostInfo) string { + var sb strings.Builder + sb.WriteString(info.OS.Name) + if codeName := info.OS.Codename; codeName != "" { + sb.WriteString(" ") + sb.WriteString(codeName) + } + + if version := info.OS.Version; version != "" { + sb.WriteString("(") + sb.WriteString(version) + sb.WriteString(")") + } + + return sb.String() +} diff --git a/x-pack/elastic-agent/pkg/agent/application/local_meta.go b/x-pack/elastic-agent/pkg/agent/application/local_meta.go index 47e358b6262..3456075baa9 100644 --- a/x-pack/elastic-agent/pkg/agent/application/local_meta.go +++ b/x-pack/elastic-agent/pkg/agent/application/local_meta.go @@ -5,21 +5,20 @@ package application import ( - "os" - "runtime" - - "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/release" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/info" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors" ) func metadata() (map[string]interface{}, error) { - hostname, err := os.Hostname() + agentInfo, err := info.NewAgentInfo() if err != nil { return nil, err } - return map[string]interface{}{ - "platform": runtime.GOOS, - "version": release.Version(), - "host": hostname, - }, nil + meta, err := agentInfo.ECSMetadata() + if err != nil { + return nil, errors.New(err, "failed to gather host metadata") + } + + return meta, nil } diff --git a/x-pack/elastic-agent/pkg/core/plugin/app/monitoring/beats/beats_monitor.go b/x-pack/elastic-agent/pkg/core/plugin/app/monitoring/beats/beats_monitor.go index 14d191c5477..f1fb92d3a71 100644 --- a/x-pack/elastic-agent/pkg/core/plugin/app/monitoring/beats/beats_monitor.go +++ b/x-pack/elastic-agent/pkg/core/plugin/app/monitoring/beats/beats_monitor.go @@ -46,7 +46,12 @@ func (b *Monitor) Reload(rawConfig *config.Config) error { return err } - b.config = cfg.MonitoringConfig + if cfg == nil || cfg.MonitoringConfig == nil { + b.config = &monitoringConfig.MonitoringConfig{} + } else { + b.config = cfg.MonitoringConfig + } + return nil }