Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use external autodiscover library for hints helpers #32132

Merged
merged 12 commits into from
Jun 30, 2022
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions NOTICE.txt
Original file line number Diff line number Diff line change
Expand Up @@ -5837,11 +5837,11 @@ SOFTWARE.

--------------------------------------------------------------------------------
Dependency : github.com/elastic/elastic-agent-autodiscover
Version: v0.1.1
Version: v0.2.1
Licence type (autodetected): Apache-2.0
--------------------------------------------------------------------------------

Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-autodiscover@v0.1.1/LICENSE:
Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-autodiscover@v0.2.1/LICENSE:

Apache License
Version 2.0, January 2004
Expand Down Expand Up @@ -23831,11 +23831,11 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

--------------------------------------------------------------------------------
Dependency : github.com/containerd/containerd
Version: v1.5.11
Version: v1.5.13
Licence type (autodetected): Apache-2.0
--------------------------------------------------------------------------------

Contents of probable licence file $GOMODCACHE/github.com/containerd/[email protected].11/LICENSE:
Contents of probable licence file $GOMODCACHE/github.com/containerd/[email protected].13/LICENSE:


Apache License
Expand Down Expand Up @@ -38662,11 +38662,11 @@ Contents of probable licence file $GOMODCACHE/[email protected]/LICENSE:

--------------------------------------------------------------------------------
Dependency : go.uber.org/goleak
Version: v1.1.11
Version: v1.1.12
Licence type (autodetected): MIT
--------------------------------------------------------------------------------

Contents of probable licence file $GOMODCACHE/go.uber.org/[email protected].11/LICENSE:
Contents of probable licence file $GOMODCACHE/go.uber.org/[email protected].12/LICENSE:

The MIT License (MIT)

Expand Down
33 changes: 16 additions & 17 deletions filebeat/autodiscover/builder/hints/logs.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,17 +22,16 @@ import (
"regexp"

"github.com/elastic/elastic-agent-autodiscover/bus"
conf "github.com/elastic/elastic-agent-libs/config"
"github.com/elastic/elastic-agent-libs/mapstr"
"github.com/elastic/go-ucfg"
"github.com/elastic/elastic-agent-autodiscover/utils"

"github.com/elastic/beats/v7/filebeat/fileset"
"github.com/elastic/beats/v7/filebeat/harvester"
"github.com/elastic/beats/v7/libbeat/autodiscover"
"github.com/elastic/beats/v7/libbeat/autodiscover/builder"
"github.com/elastic/beats/v7/libbeat/autodiscover/template"
"github.com/elastic/beats/v7/libbeat/beat"
conf "github.com/elastic/elastic-agent-libs/config"
"github.com/elastic/elastic-agent-libs/logp"
"github.com/elastic/elastic-agent-libs/mapstr"
)

func init() {
Expand Down Expand Up @@ -80,8 +79,8 @@ func (l *logHints) CreateConfig(event bus.Event, options ...ucfg.Option) []*conf
}

// Hint must be explicitly enabled when default_config sets enabled=false.
if !l.config.DefaultConfig.Enabled() && !builder.IsEnabled(hints, l.config.Key) ||
builder.IsDisabled(hints, l.config.Key) {
if !l.config.DefaultConfig.Enabled() && !utils.IsEnabled(hints, l.config.Key) ||
utils.IsDisabled(hints, l.config.Key) {
l.log.Debugw("Hints config is not enabled.", "autodiscover.event", event)
return nil
}
Expand Down Expand Up @@ -168,37 +167,37 @@ func (l *logHints) CreateConfig(event bus.Event, options ...ucfg.Option) []*conf
}

func (l *logHints) getMultiline(hints mapstr.M) mapstr.M {
return builder.GetHintMapStr(hints, l.config.Key, multiline)
return utils.GetHintMapStr(hints, l.config.Key, multiline)
}

func (l *logHints) getIncludeLines(hints mapstr.M) []string {
return builder.GetHintAsList(hints, l.config.Key, includeLines)
return utils.GetHintAsList(hints, l.config.Key, includeLines)
}

func (l *logHints) getExcludeLines(hints mapstr.M) []string {
return builder.GetHintAsList(hints, l.config.Key, excludeLines)
return utils.GetHintAsList(hints, l.config.Key, excludeLines)
}

func (l *logHints) getModule(hints mapstr.M) string {
module := builder.GetHintString(hints, l.config.Key, "module")
module := utils.GetHintString(hints, l.config.Key, "module")
// for security, strip module name
return validModuleNames.ReplaceAllString(module, "")
}

func (l *logHints) getInputsConfigs(hints mapstr.M) []mapstr.M {
return builder.GetHintAsConfigs(hints, l.config.Key)
return utils.GetHintAsConfigs(hints, l.config.Key)
}

func (l *logHints) getProcessors(hints mapstr.M) []mapstr.M {
return builder.GetProcessors(hints, l.config.Key)
return utils.GetProcessors(hints, l.config.Key)
}

func (l *logHints) getPipeline(hints mapstr.M) string {
return builder.GetHintString(hints, l.config.Key, "pipeline")
return utils.GetHintString(hints, l.config.Key, "pipeline")
}

func (l *logHints) getJSONOptions(hints mapstr.M) mapstr.M {
return builder.GetHintMapStr(hints, l.config.Key, json)
return utils.GetHintMapStr(hints, l.config.Key, json)
}

type filesetConfig struct {
Expand All @@ -222,7 +221,7 @@ func (l *logHints) getFilesets(hints mapstr.M, module string) map[string]*filese
}

// If a single fileset is given, pass all streams to it
fileset := builder.GetHintString(hints, l.config.Key, "fileset")
fileset := utils.GetHintString(hints, l.config.Key, "fileset")
if fileset != "" {
if conf, ok := filesets[fileset]; ok {
conf.Enabled = true
Expand All @@ -232,7 +231,7 @@ func (l *logHints) getFilesets(hints mapstr.M, module string) map[string]*filese

// If fileset is defined per stream, return all of them
for _, stream := range []string{"all", "stdout", "stderr"} {
fileset := builder.GetHintString(hints, l.config.Key, "fileset."+stream)
fileset := utils.GetHintString(hints, l.config.Key, "fileset."+stream)
if fileset != "" {
if conf, ok := filesets[fileset]; ok {
conf.Enabled = true
Expand All @@ -253,7 +252,7 @@ func (l *logHints) getFilesets(hints mapstr.M, module string) map[string]*filese
}

func (l *logHints) getInputs(hints mapstr.M) []mapstr.M {
modules := builder.GetHintsAsList(hints, l.config.Key)
modules := utils.GetHintsAsList(hints, l.config.Key)
var output []mapstr.M

for _, mod := range modules {
Expand Down
4 changes: 2 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -160,7 +160,7 @@ require (

require (
github.com/elastic/bayeux v1.0.5
github.com/elastic/elastic-agent-autodiscover v0.1.1
github.com/elastic/elastic-agent-autodiscover v0.2.1
github.com/elastic/elastic-agent-libs v0.2.5
github.com/elastic/elastic-agent-system-metrics v0.4.1
github.com/elastic/go-elasticsearch/v8 v8.2.0
Expand Down Expand Up @@ -197,7 +197,7 @@ require (
github.com/cespare/xxhash v1.1.0 // indirect
github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4 // indirect
github.com/cncf/xds/go v0.0.0-20211130200136-a8f946100490 // indirect
github.com/containerd/containerd v1.5.11 // indirect
github.com/containerd/containerd v1.5.13 // indirect
github.com/cyphar/filepath-securejoin v0.2.3 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/davecgh/go-xdr v0.0.0-20161123171359-e6a2ba005892 // indirect
Expand Down
18 changes: 8 additions & 10 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -160,7 +160,6 @@ github.com/Microsoft/hcsshim v0.8.9/go.mod h1:5692vkUqntj1idxauYlpoINNKeqCiG6Sg3
github.com/Microsoft/hcsshim v0.8.14/go.mod h1:NtVKoYxQuTLx6gEq0L96c9Ju4JbRJ4nY2ow3VK6a9Lg=
github.com/Microsoft/hcsshim v0.8.15/go.mod h1:x38A4YbHbdxJtc0sF6oIz+RG0npwSCAvn69iY6URG00=
github.com/Microsoft/hcsshim v0.8.16/go.mod h1:o5/SZqmR7x9JNKsW3pu+nqHm0MF8vbA+VxGOoXdC600=
github.com/Microsoft/hcsshim v0.8.21/go.mod h1:+w2gRZ5ReXQhFOrvSQeNfhrYB/dg3oDwTOcER2fw4I4=
github.com/Microsoft/hcsshim v0.8.24/go.mod h1:4zegtUJth7lAvFyc6cH2gGQ5B3OFQim01nnU2M8jKDg=
github.com/Microsoft/hcsshim/test v0.0.0-20201218223536-d3e5debf77da/go.mod h1:5hlzMzRKMLyo42nCZ9oml8AdTlq/0cvIaBv6tK1RehU=
github.com/Microsoft/hcsshim/test v0.0.0-20210227013316-43a75bb4edd3/go.mod h1:mw7qgWloBUl75W/gVH3cQszUg1+gUITj7D6NY7ywVnY=
Expand Down Expand Up @@ -347,6 +346,7 @@ github.com/containerd/cgroups v0.0.0-20200710171044-318312a37340/go.mod h1:s5q4S
github.com/containerd/cgroups v0.0.0-20200824123100-0b889c03f102/go.mod h1:s5q4SojHctfxANBDvMeIaIovkq29IP48TKAxnhYRxvo=
github.com/containerd/cgroups v0.0.0-20210114181951-8a68de567b68/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE=
github.com/containerd/cgroups v1.0.1/go.mod h1:0SJrPIenamHDcZhEcJMNBB85rHcUsw4f25ZfBiPYRkU=
github.com/containerd/cgroups v1.0.3/go.mod h1:/ofk34relqNjSGyqPrmEULrO4Sc8LJhvJmWbUCUKqj8=
github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw=
github.com/containerd/console v0.0.0-20181022165439-0650fd9eeb50/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw=
github.com/containerd/console v0.0.0-20191206165004-02ecf6a7291e/go.mod h1:8Pf4gM6VEbTNRIT26AyyU7hxdQU3MvAvxVI0sc00XBE=
Expand All @@ -366,10 +366,8 @@ github.com/containerd/containerd v1.5.0-beta.1/go.mod h1:5HfvG1V2FsKesEGQ17k5/T7
github.com/containerd/containerd v1.5.0-beta.3/go.mod h1:/wr9AVtEM7x9c+n0+stptlo/uBBoBORwEx6ardVcmKU=
github.com/containerd/containerd v1.5.0-beta.4/go.mod h1:GmdgZd2zA2GYIBZ0w09ZvgqEq8EfBp/m3lcVZIvPHhI=
github.com/containerd/containerd v1.5.0-rc.0/go.mod h1:V/IXoMqNGgBlabz3tHD2TWDoTJseu1FGOKuoA4nNb2s=
github.com/containerd/containerd v1.5.1/go.mod h1:0DOxVqwDy2iZvrZp2JUx/E+hS0UNTVn7dJnIOwtYR4g=
github.com/containerd/containerd v1.5.7/go.mod h1:gyvv6+ugqY25TiXxcZC3L5yOeYgEw0QMhscqVp1AR9c=
github.com/containerd/containerd v1.5.11 h1:+biZCY9Kns9t2J8L9hOqubjvNQBr1ULdmR7kL+omKoY=
github.com/containerd/containerd v1.5.11/go.mod h1:FJl/l1urLXpO3oKDx2No2ouBno2GSI56nTl02HfHeZY=
github.com/containerd/containerd v1.5.13 h1:XqvKw9i4P7/mFrC3TSM7yV5cwFZ9avXe6M3YANKnzEE=
github.com/containerd/containerd v1.5.13/go.mod h1:3AlCrzKROjIuP3JALsY14n8YtntaUDBu7vek+rPN5Vc=
github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
github.com/containerd/continuity v0.0.0-20190815185530-f2a389ac0a02/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
github.com/containerd/continuity v0.0.0-20191127005431-f65d91d395eb/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
Expand Down Expand Up @@ -534,11 +532,10 @@ github.com/elastic/bayeux v1.0.5 h1:UceFq01ipmT3S8DzFK+uVAkbCdiPR0Bqei8qIGmUeY0=
github.com/elastic/bayeux v1.0.5/go.mod h1:CSI4iP7qeo5MMlkznGvYKftp8M7qqP/3nzmVZoXHY68=
github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3 h1:lnDkqiRFKm0rxdljqrj3lotWinO9+jFmeDXIC4gvIQs=
github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3/go.mod h1:aPqzac6AYkipvp4hufTyMj5PDIphF3+At8zr7r51xjY=
github.com/elastic/elastic-agent-autodiscover v0.1.1 h1:oSV42MKIcPq30VJFZwc066eVm+pKpxfQNY4RIPE1fyg=
github.com/elastic/elastic-agent-autodiscover v0.1.1/go.mod h1:Gg1fsQI+rVms9FJ2DefBSojfPIzgkV8xlyG8fPG0DE8=
github.com/elastic/elastic-agent-autodiscover v0.2.1 h1:Nbeayh3vq2FNm6xaFo34mhUdOu0EVlpj53CqCsbU0E4=
github.com/elastic/elastic-agent-autodiscover v0.2.1/go.mod h1:gPnzzfdYNdgznAb+iG9eyyXaQXBbAMHa+Y6Z8hXfcGY=
github.com/elastic/elastic-agent-client/v7 v7.0.0-20210727140539-f0905d9377f6 h1:nFvXHBjYK3e9+xF0WKDeAKK4aOO51uC28s+L9rBmilo=
github.com/elastic/elastic-agent-client/v7 v7.0.0-20210727140539-f0905d9377f6/go.mod h1:uh/Gj9a0XEbYoM4NYz4LvaBVARz3QXLmlNjsrKY9fTc=
github.com/elastic/elastic-agent-libs v0.0.0-20220303160015-5b4e674da3dd/go.mod h1://82M1l73IHx0wDbS2Tzkq6Fx9fkmytS1KgkIyzvNTM=
github.com/elastic/elastic-agent-libs v0.2.2/go.mod h1:1xDLBhIqBIjhJ7lr2s+xRFFkQHpitSp8q2zzv1Dqg+s=
github.com/elastic/elastic-agent-libs v0.2.5 h1:8+sYCW/kkWQe5KegGLMYYT3ELXUwibMcjMh+IMr2LRE=
github.com/elastic/elastic-agent-libs v0.2.5/go.mod h1:chO3rtcLyGlKi9S0iGVZhYCzDfdDsAQYBc+ui588AFE=
Expand Down Expand Up @@ -1351,7 +1348,6 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
github.com/opencontainers/image-spec v1.0.0/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
github.com/opencontainers/image-spec v1.0.2-0.20190823105129-775207bd45b6/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
github.com/opencontainers/image-spec v1.0.2 h1:9yCKha/T5XdGtO0q9Q9a6T5NUCsTn/DrBg0D7ufOcFM=
github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=
Expand Down Expand Up @@ -1759,8 +1755,9 @@ go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE=
go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
go.uber.org/goleak v1.0.0/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A=
go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A=
go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI=
go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA=
go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4=
go.uber.org/multierr v1.4.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4=
Expand Down Expand Up @@ -2101,6 +2098,7 @@ golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220405052023-b1e9470b6e64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
Expand Down
17 changes: 8 additions & 9 deletions heartbeat/autodiscover/builder/hints/monitors.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,15 +22,14 @@ import (
"strings"

"github.com/elastic/elastic-agent-autodiscover/bus"
conf "github.com/elastic/elastic-agent-libs/config"
"github.com/elastic/elastic-agent-libs/mapstr"
"github.com/elastic/go-ucfg"
"github.com/elastic/elastic-agent-autodiscover/utils"

"github.com/elastic/beats/v7/libbeat/autodiscover"
"github.com/elastic/beats/v7/libbeat/autodiscover/builder"
"github.com/elastic/beats/v7/libbeat/autodiscover/template"
"github.com/elastic/beats/v7/libbeat/common"
conf "github.com/elastic/elastic-agent-libs/config"
"github.com/elastic/elastic-agent-libs/logp"
"github.com/elastic/elastic-agent-libs/mapstr"
)

func init() {
Expand Down Expand Up @@ -71,7 +70,7 @@ func (hb *heartbeatHints) CreateConfig(event bus.Event, options ...ucfg.Option)
monitorConfig := hb.getRawConfigs(hints)

// If explicty disabled, return nothing
if builder.IsDisabled(hints, hb.config.Key) {
if utils.IsDisabled(hints, hb.config.Key) {
hb.logger.Warnf("heartbeat config disabled by hint: %+v", event)
return []*conf.C{}
}
Expand All @@ -96,7 +95,7 @@ func (hb *heartbeatHints) CreateConfig(event bus.Event, options ...ucfg.Option)
}

tempCfg := mapstr.M{}
monitors := builder.GetHintsAsList(hints, hb.config.Key)
monitors := utils.GetHintsAsList(hints, hb.config.Key)

configs := make([]*conf.C, 0, len(monitors))
for _, monitor := range monitors {
Expand Down Expand Up @@ -126,16 +125,16 @@ func (hb *heartbeatHints) CreateConfig(event bus.Event, options ...ucfg.Option)
}

func (hb *heartbeatHints) getRawConfigs(hints mapstr.M) []mapstr.M {
return builder.GetHintAsConfigs(hints, hb.config.Key)
return utils.GetHintAsConfigs(hints, hb.config.Key)
}

func (hb *heartbeatHints) getProcessors(hints mapstr.M) []mapstr.M {
return builder.GetConfigs(hints, "", "processors")
return utils.GetConfigs(hints, "", "processors")
}

func (hb *heartbeatHints) getHostsWithPort(hints mapstr.M, port int) []string {
var result []string
thosts := builder.GetHintAsList(hints, "", hosts)
thosts := utils.GetHintAsList(hints, "", hosts)
// Only pick hosts that have ${data.port} or the port on current event. This will make
// sure that incorrect meta mapping doesn't happen
for _, h := range thosts {
Expand Down
Loading