From 7b8ea18b6066ff54e4a904b360850b3acfe9fcf4 Mon Sep 17 00:00:00 2001 From: Adrian Serrano Date: Wed, 31 Oct 2018 13:23:53 +0100 Subject: [PATCH 1/3] add_cloud_metadata asynchronous initialisation Now that the add_cloud_metadata is enabled by default in all beats, we are losing 3 precious seconds every time a Beat is started outside a supported cloud environment. This patch makes the cloud detection an asynchronous task so the Beat can start and only block if initialisation is not completed at the time of the first enrichment. Running in debug mode bypasses this parallelism as the processor needs to be initialised when its String() method is called. --- .../add_cloud_metadata/add_cloud_metadata.go | 45 +++++++++++++------ 1 file changed, 32 insertions(+), 13 deletions(-) diff --git a/libbeat/processors/add_cloud_metadata/add_cloud_metadata.go b/libbeat/processors/add_cloud_metadata/add_cloud_metadata.go index c281dd799ecd..8913093764ca 100644 --- a/libbeat/processors/add_cloud_metadata/add_cloud_metadata.go +++ b/libbeat/processors/add_cloud_metadata/add_cloud_metadata.go @@ -25,6 +25,7 @@ import ( "io/ioutil" "net" "net/http" + "sync" "time" "github.com/pkg/errors" @@ -314,34 +315,52 @@ func newCloudMetadata(c *common.Config) (processors.Processor, error) { return nil, err } - result := fetchMetadata(fetchers, config.Timeout) - if result == nil { - logp.Info("add_cloud_metadata: hosting provider type not detected.") - return &addCloudMetadata{}, nil - } + acm := new(addCloudMetadata) - logp.Info("add_cloud_metadata: hosting provider type detected as %v, metadata=%v", - result.provider, result.metadata.String()) + acm.initFn = func() { + result := fetchMetadata(fetchers, config.Timeout) + if result == nil { + logp.Info("add_cloud_metadata: hosting provider type not detected.") + return + } + acm.metadata = result.metadata + acm.initFn = nil + logp.Info("add_cloud_metadata: hosting provider type detected as %v, metadata=%v", + result.provider, result.metadata.String()) + } - return &addCloudMetadata{metadata: result.metadata}, nil + go acm.init() + return acm, nil } type addCloudMetadata struct { + initFn func() + initOnce sync.Once metadata common.MapStr } -func (p addCloudMetadata) Run(event *beat.Event) (*beat.Event, error) { - if len(p.metadata) == 0 { +func (p *addCloudMetadata) init() { + p.initOnce.Do(p.initFn) +} + +func (p *addCloudMetadata) getMeta() common.MapStr { + p.init() + return p.metadata +} + +func (p *addCloudMetadata) Run(event *beat.Event) (*beat.Event, error) { + meta := p.getMeta() + if len(meta) == 0 { return event, nil } // This overwrites the meta.cloud if it exists. But the cloud key should be // reserved for this processor so this should happen. - _, err := event.PutValue("meta.cloud", p.metadata) + _, err := event.PutValue("meta.cloud", meta) return event, err } -func (p addCloudMetadata) String() string { - return "add_cloud_metadata=" + p.metadata.String() +func (p *addCloudMetadata) String() string { + return "add_cloud_metadata=" + p.getMeta().String() } From 6920805d152ad3bc9d7ba5d13b4aa7c457426894 Mon Sep 17 00:00:00 2001 From: Adrian Serrano Date: Fri, 2 Nov 2018 15:17:21 +0100 Subject: [PATCH 2/3] Changelog --- CHANGELOG.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 74e6d3fd4533..d098422ee377 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -133,6 +133,7 @@ https://github.com/elastic/beats/compare/v6.4.0...master[Check the HEAD diff] - Allow Bus to buffer events in case listeners are not configured. {pull}8527[8527] - Enable `host` and `cloud` metadata processors by default. {pull}8596[8596] - Dissect will now flag event on parsing error. {pull}8751[8751] +- add_cloud_metadata initialization is performed asynchronously to avoid delays on startup. {pull}8845[8845] *Auditbeat* From 36380bb23d4bbed8ecd92bd897b5da9b1625dd60 Mon Sep 17 00:00:00 2001 From: Adrian Serrano Date: Tue, 6 Nov 2018 01:28:26 +0100 Subject: [PATCH 3/3] Fix data race --- .../add_cloud_metadata/add_cloud_metadata.go | 37 ++++++++++--------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/libbeat/processors/add_cloud_metadata/add_cloud_metadata.go b/libbeat/processors/add_cloud_metadata/add_cloud_metadata.go index 8913093764ca..c644c9bdd808 100644 --- a/libbeat/processors/add_cloud_metadata/add_cloud_metadata.go +++ b/libbeat/processors/add_cloud_metadata/add_cloud_metadata.go @@ -315,36 +315,39 @@ func newCloudMetadata(c *common.Config) (processors.Processor, error) { return nil, err } - acm := new(addCloudMetadata) - - acm.initFn = func() { - result := fetchMetadata(fetchers, config.Timeout) - if result == nil { - logp.Info("add_cloud_metadata: hosting provider type not detected.") - return - } - acm.metadata = result.metadata - acm.initFn = nil - logp.Info("add_cloud_metadata: hosting provider type detected as %v, metadata=%v", - result.provider, result.metadata.String()) + p := &addCloudMetadata{ + initData: &initData{fetchers, config.Timeout}, } - go acm.init() - return acm, nil + go p.initOnce.Do(p.init) + return p, nil +} + +type initData struct { + fetchers []*metadataFetcher + timeout time.Duration } type addCloudMetadata struct { - initFn func() initOnce sync.Once + initData *initData metadata common.MapStr } func (p *addCloudMetadata) init() { - p.initOnce.Do(p.initFn) + result := fetchMetadata(p.initData.fetchers, p.initData.timeout) + if result == nil { + logp.Info("add_cloud_metadata: hosting provider type not detected.") + return + } + p.metadata = result.metadata + p.initData = nil + logp.Info("add_cloud_metadata: hosting provider type detected as %v, metadata=%v", + result.provider, result.metadata.String()) } func (p *addCloudMetadata) getMeta() common.MapStr { - p.init() + p.initOnce.Do(p.init) return p.metadata }