diff --git a/NOTICE.txt b/NOTICE.txt index af6d0a8ca44..b47a66c8d63 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -10311,11 +10311,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-l -------------------------------------------------------------------------------- Dependency : github.com/elastic/elastic-agent-shipper-client -Version: v0.5.0 +Version: v0.5.1-0.20230228231646-f04347b666f3 Licence type (autodetected): Elastic -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-shipper-client@v0.5.0/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-shipper-client@v0.5.1-0.20230228231646-f04347b666f3/LICENSE.txt: Elastic License 2.0 @@ -10414,11 +10414,11 @@ these terms. -------------------------------------------------------------------------------- Dependency : github.com/elastic/elastic-agent-system-metrics -Version: v0.4.6-0.20230308003052-ba171438211e +Version: v0.5.1-0.20230328163318-a87a0b315c42 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-system-metrics@v0.4.6-0.20230308003052-ba171438211e/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-system-metrics@v0.5.1-0.20230328163318-a87a0b315c42/LICENSE.txt: Apache License Version 2.0, January 2004 @@ -12354,11 +12354,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/go-structform@v -------------------------------------------------------------------------------- Dependency : github.com/elastic/go-sysinfo -Version: v1.9.1-0.20230215152520-f544eca983fb +Version: v1.9.1-0.20230328042007-6dcfe88b8359 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/go-sysinfo@v1.9.1-0.20230215152520-f544eca983fb/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/elastic/go-sysinfo@v1.9.1-0.20230328042007-6dcfe88b8359/LICENSE.txt: Apache License @@ -13472,6 +13472,37 @@ Contents of probable licence file $GOMODCACHE/github.com/fearful-symmetry/gorapl limitations under the License. +-------------------------------------------------------------------------------- +Dependency : github.com/foxcpp/go-mockdns +Version: v0.0.0-20201212160233-ede2f9158d15 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/foxcpp/go-mockdns@v0.0.0-20201212160233-ede2f9158d15/LICENSE: + +MIT License + +Copyright © 2019 Max Mazurov + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + -------------------------------------------------------------------------------- Dependency : github.com/elastic/fsevents Version: v0.0.0-20181029231046-e1d381a4d270 diff --git a/go.mod b/go.mod index 8d9f3ea9d45..79a864c1072 100644 --- a/go.mod +++ b/go.mod @@ -79,7 +79,7 @@ require ( github.com/elastic/go-perf v0.0.0-20191212140718-9c656876f595 github.com/elastic/go-seccomp-bpf v1.3.0 github.com/elastic/go-structform v0.0.10 - github.com/elastic/go-sysinfo v1.9.1-0.20230215152520-f544eca983fb + github.com/elastic/go-sysinfo v1.9.1-0.20230328042007-6dcfe88b8359 github.com/elastic/go-ucfg v0.8.6 github.com/elastic/gosigar v0.14.2 github.com/fatih/color v1.13.0 @@ -194,11 +194,12 @@ require ( github.com/elastic/bayeux v1.0.5 github.com/elastic/elastic-agent-autodiscover v0.5.0 github.com/elastic/elastic-agent-libs v0.3.3 - github.com/elastic/elastic-agent-shipper-client v0.5.0 - github.com/elastic/elastic-agent-system-metrics v0.4.6-0.20230308003052-ba171438211e + github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3 + github.com/elastic/elastic-agent-system-metrics v0.5.1-0.20230328163318-a87a0b315c42 github.com/elastic/go-elasticsearch/v8 v8.2.0 github.com/elastic/mito v0.0.0-20230302005114-1dda06e81678 github.com/elastic/toutoumomoma v0.0.0-20221026030040-594ef30cb640 + github.com/foxcpp/go-mockdns v0.0.0-20201212160233-ede2f9158d15 github.com/google/cel-go v0.13.0 github.com/googleapis/gax-go/v2 v2.6.0 github.com/gorilla/handlers v1.5.1 diff --git a/go.sum b/go.sum index 7751f91b6d3..eb995d76bd9 100644 --- a/go.sum +++ b/go.sum @@ -616,10 +616,10 @@ github.com/elastic/elastic-agent-client/v7 v7.0.3-0.20230315204017-166fd1fd746f/ github.com/elastic/elastic-agent-libs v0.2.11/go.mod h1:chO3rtcLyGlKi9S0iGVZhYCzDfdDsAQYBc+ui588AFE= github.com/elastic/elastic-agent-libs v0.3.3 h1:iE8XhqQ0zRBLba+eu6ScZED0DYcVP/r2JvjcVoOkxic= github.com/elastic/elastic-agent-libs v0.3.3/go.mod h1:nRkcK96PSJfK232cJRx17n9+/MVAIOzs5ghZdzXJAMo= -github.com/elastic/elastic-agent-shipper-client v0.5.0 h1:rkdq7K8+ESNMXtMPzlwiiENTZz2Y6m4lN8SIMFrHuJA= -github.com/elastic/elastic-agent-shipper-client v0.5.0/go.mod h1:rWarFM7qYxJKsi9WcV6ONcFjH/NA3niDNpTxO+8/GVI= -github.com/elastic/elastic-agent-system-metrics v0.4.6-0.20230308003052-ba171438211e h1:OIfumgZhI6lI7Qy1KD1VzuqvX9DWSBpXJsvj97s7MRM= -github.com/elastic/elastic-agent-system-metrics v0.4.6-0.20230308003052-ba171438211e/go.mod h1:v/t/qgYueW3ZOm7SZhYY3ng9GWDddDLu7pmG4Ra3PBs= +github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3 h1:sb+25XJn/JcC9/VL8HX4r4QXSUq4uTNzGS2kxOE7u1U= +github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3/go.mod h1:rWarFM7qYxJKsi9WcV6ONcFjH/NA3niDNpTxO+8/GVI= +github.com/elastic/elastic-agent-system-metrics v0.5.1-0.20230328163318-a87a0b315c42 h1:oJQbTUpFlD/CXKw6N5gWNOjfofwStp+nR49m1wKv6zc= +github.com/elastic/elastic-agent-system-metrics v0.5.1-0.20230328163318-a87a0b315c42/go.mod h1:jBVJTZFtoxQkGmqutGGRSulkRhjgIO2PY059m+HLG4k= github.com/elastic/elastic-transport-go/v8 v8.1.0 h1:NeqEz1ty4RQz+TVbUrpSU7pZ48XkzGWQj02k5koahIE= github.com/elastic/elastic-transport-go/v8 v8.1.0/go.mod h1:87Tcz8IVNe6rVSLdBux1o/PEItLtyabHU3naC7IoqKI= github.com/elastic/fsevents v0.0.0-20181029231046-e1d381a4d270 h1:cWPqxlPtir4RoQVCpGSRXmLqjEHpJKbR60rxh1nQZY4= @@ -647,8 +647,8 @@ github.com/elastic/go-structform v0.0.9/go.mod h1:CZWf9aIRYY5SuKSmOhtXScE5uQiLZN github.com/elastic/go-structform v0.0.10 h1:oy08o/Ih2hHTkNcRY/1HhaYvIp5z6t8si8gnCJPDo1w= github.com/elastic/go-structform v0.0.10/go.mod h1:CZWf9aIRYY5SuKSmOhtXScE5uQiLZNqAFnwKR4OrIM4= github.com/elastic/go-sysinfo v1.7.1/go.mod h1:i1ZYdU10oLNfRzq4vq62BEwD2fH8KaWh6eh0ikPT9F0= -github.com/elastic/go-sysinfo v1.9.1-0.20230215152520-f544eca983fb h1:dTYv5aVpYzI53KJwDM/Hue68YU/PHkLJD/5jDl1fGSs= -github.com/elastic/go-sysinfo v1.9.1-0.20230215152520-f544eca983fb/go.mod h1:j/gGAinRS+z3loQ/1pO+s9pCyQsna8U13kqhF1Aa0fg= +github.com/elastic/go-sysinfo v1.9.1-0.20230328042007-6dcfe88b8359 h1:qSCKykLGlUcO/wzq3xH+wtPHM2KCpQnqXOKClSg5K2w= +github.com/elastic/go-sysinfo v1.9.1-0.20230328042007-6dcfe88b8359/go.mod h1:j/gGAinRS+z3loQ/1pO+s9pCyQsna8U13kqhF1Aa0fg= github.com/elastic/go-ucfg v0.8.5/go.mod h1:4E8mPOLSUV9hQ7sgLEJ4bvt0KhMuDJa8joDT2QGAEKA= github.com/elastic/go-ucfg v0.8.6 h1:stUeyh2goTgGX+/wb9gzKvTv0YB0231LTpKUgCKj4U0= github.com/elastic/go-ucfg v0.8.6/go.mod h1:4E8mPOLSUV9hQ7sgLEJ4bvt0KhMuDJa8joDT2QGAEKA= @@ -705,6 +705,7 @@ github.com/form3tech-oss/jwt-go v3.2.3+incompatible h1:7ZaBxOI7TMoYBfyA3cQHErNNy github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= +github.com/foxcpp/go-mockdns v0.0.0-20201212160233-ede2f9158d15 h1:nLPjjvpUAODOR6vY/7o0hBIk8iTr19Fvmf8aFx/kC7A= github.com/foxcpp/go-mockdns v0.0.0-20201212160233-ede2f9158d15/go.mod h1:tPg4cp4nseejPd+UKxtCVQ2hUxNTZ7qQZJa7CLriIeo= github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= diff --git a/libbeat/cmd/instance/beat.go b/libbeat/cmd/instance/beat.go index de9e171da7b..879323411c1 100644 --- a/libbeat/cmd/instance/beat.go +++ b/libbeat/cmd/instance/beat.go @@ -213,12 +213,6 @@ func NewBeat(name, indexPrefix, v string, elasticLicensed bool) (*Beat, error) { return nil, err } - h, err := sysinfo.Host() - if err != nil { - return nil, fmt.Errorf("failed to get host information: %w", err) - } - fqdn := h.Info().FQDN - fields, err := asset.GetFields(name) if err != nil { return nil, err @@ -237,7 +231,6 @@ func NewBeat(name, indexPrefix, v string, elasticLicensed bool) (*Beat, error) { Version: v, Name: hostname, Hostname: hostname, - FQDN: fqdn, ID: id, FirstStart: time.Now(), StartTime: time.Now(), @@ -555,7 +548,7 @@ func (b *Beat) RegisterHostname(useFQDN bool) { // state.host stateRegistry := monitoring.GetNamespace("state").GetRegistry() - monitoring.NewFunc(stateRegistry, "host", host.ReportInfo(useFQDN), monitoring.Report) + monitoring.NewFunc(stateRegistry, "host", host.ReportInfo(hostname), monitoring.Report) } // TestConfig check all settings are ok and the beat can be run @@ -756,6 +749,22 @@ func (b *Beat) configure(settings Settings) error { logp.Info("Beat ID: %v", b.Info.ID) + // Try to get the host's FQDN and set it. + h, err := sysinfo.Host() + if err != nil { + return fmt.Errorf("failed to get host information: %w", err) + } + + fqdn, err := h.FQDN() + if err != nil { + // FQDN lookup is "best effort". We log the error, fallback to + // the OS-reported hostname, and move on. + logp.Warn("unable to lookup FQDN: %s, using hostname = %s as FQDN", err.Error(), b.Info.Hostname) + b.Info.FQDN = b.Info.Hostname + } else { + b.Info.FQDN = fqdn + } + // initialize config manager b.Manager, err = management.Factory(b.Config.Management)(b.Config.Management, reload.RegisterV2, b.Beat.Info.ID) if err != nil { diff --git a/libbeat/processors/add_host_metadata/add_host_metadata.go b/libbeat/processors/add_host_metadata/add_host_metadata.go index 6fffe4aa919..39ae840e7ff 100644 --- a/libbeat/processors/add_host_metadata/add_host_metadata.go +++ b/libbeat/processors/add_host_metadata/add_host_metadata.go @@ -22,6 +22,8 @@ import ( "sync" "time" + "github.com/elastic/elastic-agent-libs/monitoring" + "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/features" "github.com/elastic/beats/v7/libbeat/processors" @@ -34,9 +36,22 @@ import ( "github.com/elastic/go-sysinfo" ) +const processorName = "add_host_metadata" +const logName = "processor." + processorName + +var ( + reg *monitoring.Registry +) + func init() { - processors.RegisterPlugin("add_host_metadata", New) + processors.RegisterPlugin(processorName, New) jsprocessor.RegisterPlugin("AddHostMetadata", New) + + reg = monitoring.Default.NewRegistry(logName, monitoring.DoNotReport) +} + +type metrics struct { + FQDNLookupFailed *monitoring.Int } type addHostMetadata struct { @@ -48,12 +63,9 @@ type addHostMetadata struct { geoData mapstr.M config Config logger *logp.Logger + metrics metrics } -const ( - processorName = "add_host_metadata" -) - // New constructs a new add_host_metadata processor. func New(cfg *config.C) (processors.Processor, error) { c := defaultConfig() @@ -64,7 +76,10 @@ func New(cfg *config.C) (processors.Processor, error) { p := &addHostMetadata{ config: c, data: mapstr.NewPointer(nil), - logger: logp.NewLogger("add_host_metadata"), + logger: logp.NewLogger(logName), + metrics: metrics{ + FQDNLookupFailed: monitoring.NewInt(reg, "fqdn_lookup_failed"), + }, } if err := p.loadData(); err != nil { return nil, fmt.Errorf("failed to load data: %w", err) @@ -143,7 +158,25 @@ func (p *addHostMetadata) loadData() error { return err } - data := host.MapHostInfo(features.FQDN(), h.Info()) + hostname := h.Info().Hostname + if features.FQDN() { + fqdn, err := h.FQDN() + if err != nil { + // FQDN lookup is "best effort". If it fails, we monitor the failure, fallback to + // the OS-reported hostname, and move on. + p.metrics.FQDNLookupFailed.Inc() + p.logger.Debugf( + "unable to lookup FQDN (failed attempt counter: %d): %s, using hostname = %s as FQDN", + p.metrics.FQDNLookupFailed.Get(), + err.Error(), + hostname, + ) + } else { + hostname = fqdn + } + } + + data := host.MapHostInfo(h.Info(), hostname) if p.config.NetInfoEnabled { // IP-address and MAC-address var ipList, hwList, err = util.GetNetInfo() diff --git a/libbeat/processors/add_host_metadata/add_host_metadata_test.go b/libbeat/processors/add_host_metadata/add_host_metadata_test.go index 44e7584c314..c8a15f54273 100644 --- a/libbeat/processors/add_host_metadata/add_host_metadata_test.go +++ b/libbeat/processors/add_host_metadata/add_host_metadata_test.go @@ -19,19 +19,22 @@ package add_host_metadata import ( "fmt" + "net" + "os" "runtime" "testing" "time" - "github.com/elastic/beats/v7/libbeat/features" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/libbeat/features" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/mapstr" "github.com/elastic/go-sysinfo/types" + + "github.com/foxcpp/go-mockdns" ) var ( @@ -492,17 +495,99 @@ func TestExpireCacheOnFQDNReportingChange(t *testing.T) { // Toggle the FQDN feature flag; this should cause the cache // to expire. - features.UpdateFromConfig(conf.MustNewConfigFrom(map[string]interface{}{ + err = features.UpdateFromConfig(conf.MustNewConfigFrom(map[string]interface{}{ "features.fqdn.enabled": true, })) + require.NoError(t, err) + expired = ahmP.expired() require.True(t, expired) // Set the FQDN feature flag to the same value; this should NOT // cause the cache to expire. - features.UpdateFromConfig(conf.MustNewConfigFrom(map[string]interface{}{ + err = features.UpdateFromConfig(conf.MustNewConfigFrom(map[string]interface{}{ "features.fqdn.enabled": true, })) + require.NoError(t, err) + expired = ahmP.expired() require.False(t, expired) } + +func TestFQDNLookup(t *testing.T) { + hostname, err := os.Hostname() + require.NoError(t, err) + + tests := map[string]struct { + cnameLookupResult string + expectedHostName string + expectedFQDNLookupFailedCount int64 + }{ + "lookup_succeeds": { + cnameLookupResult: "foo.bar.baz.", + expectedHostName: "foo.bar.baz", + expectedFQDNLookupFailedCount: 0, + }, + "lookup_fails": { + cnameLookupResult: "", + expectedHostName: hostname, + expectedFQDNLookupFailedCount: 1, + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + // Mock CNAME resolution + srv, _ := mockdns.NewServer(map[string]mockdns.Zone{ + hostname + ".": { + CNAME: test.cnameLookupResult, + }, + test.cnameLookupResult: { + A: []string{"1.1.1.1"}, + }, + }, false) + defer srv.Close() + + srv.PatchNet(net.DefaultResolver) + defer mockdns.UnpatchNet(net.DefaultResolver) + + // Enable FQDN feature flag + err = features.UpdateFromConfig(fqdnFeatureFlagConfig(true)) + require.NoError(t, err) + defer func() { + err = features.UpdateFromConfig(fqdnFeatureFlagConfig(true)) + require.NoError(t, err) + }() + + // Create processor and check that FQDN lookup failed + testConfig, err := conf.NewConfigFrom(map[string]interface{}{}) + require.NoError(t, err) + + p, err := New(testConfig) + require.NoError(t, err) + + addHostMetadataP, ok := p.(*addHostMetadata) + require.True(t, ok) + require.Equal(t, test.expectedFQDNLookupFailedCount, addHostMetadataP.metrics.FQDNLookupFailed.Get()) + + // Run event through processor and check that hostname reported + // by processor is same as OS-reported hostname + event := &beat.Event{ + Fields: mapstr.M{}, + Timestamp: time.Now(), + } + newEvent, err := p.Run(event) + require.NoError(t, err) + + v, err := newEvent.GetValue("host.name") + require.NoError(t, err) + require.Equal(t, test.expectedHostName, v) + }) + } +} + +func fqdnFeatureFlagConfig(fqdnEnabled bool) *conf.C { + return conf.MustNewConfigFrom(map[string]interface{}{ + "features.fqdn.enabled": fqdnEnabled, + }) +}