Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix and extend logging of light modules load errors #14706

Merged
merged 11 commits into from
Mar 3, 2020
Merged

Fix and extend logging of light modules load errors #14706

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
e2a0676
Fix and extend logging of light modules load errors
jsoriano Nov 22, 2019
944218b
Add BEAT_STRICT_PERMS to beat container in metricbeat docker compose
jsoriano Nov 22, 2019
f228833
Merge remote-tracking branch 'origin/master' into fix-lightmodules-pe…
jsoriano Mar 2, 2020
79dd64d
Add changelog
jsoriano Mar 2, 2020
224fa8e
Use local logger, and fix error messages
jsoriano Mar 2, 2020
794433c
Ignore errors if module directory doesn't exist
jsoriano Mar 2, 2020
10abf94
Fix issue when modules directory contains regular files
jsoriano Mar 2, 2020
c7a391b
Remove empty line
jsoriano Mar 2, 2020
62e282b
Add test for issue with files listed as modules
jsoriano Mar 2, 2020
84cb5df
Merge remote-tracking branch 'origin/master' into fix-lightmodules-pe…
jsoriano Mar 3, 2020
d794fe0
Merge remote-tracking branch 'origin/master' into fix-lightmodules-pe…
jsoriano Mar 3, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.next.asciidoc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
- Made `logstash-xpack` module once again have parity with internally-collected Logstash monitoring data. {pull}16198[16198]
- Change sqs metricset to use average as statistic method. {pull}16438[16438]
- Revert changes in `docker` module: add size flag to docker.container. {pull}16600[16600]
- Fix detection and logging of some error cases with light modules. {pull}14706[14706]

*Packetbeat*

Expand Down
40 changes: 24 additions & 16 deletions metricbeat/mb/lightmodules.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,12 +39,14 @@ const (
// LightModulesSource loads module definitions from files in the provided paths
type LightModulesSource struct {
paths []string
log *logp.Logger
}

// NewLightModulesSource creates a new LightModulesSource
func NewLightModulesSource(paths ...string) *LightModulesSource {
return &LightModulesSource{
paths: paths,
log: logp.NewLogger("registry.lightmodules"),
}
}

Expand All @@ -57,7 +59,7 @@ func (s *LightModulesSource) Modules() ([]string, error) {
func (s *LightModulesSource) HasModule(moduleName string) bool {
names, err := s.moduleNames()
if err != nil {
logp.Error(errors.Wrap(err, "failed to get list of light module names"))
s.log.Errorf("Failed to get list of light module names: %v", err)
return false
}
for _, name := range names {
Expand All @@ -72,7 +74,7 @@ func (s *LightModulesSource) HasModule(moduleName string) bool {
func (s *LightModulesSource) DefaultMetricSets(r *Register, moduleName string) ([]string, error) {
module, err := s.loadModule(r, moduleName)
if err != nil {
return nil, errors.Wrapf(err, "failed to get default metricsets for module '%s'", moduleName)
return nil, errors.Wrapf(err, "getting default metricsets for module '%s'", moduleName)
}
var metricsets []string
for _, ms := range module.MetricSets {
Expand All @@ -87,7 +89,7 @@ func (s *LightModulesSource) DefaultMetricSets(r *Register, moduleName string) (
func (s *LightModulesSource) MetricSets(r *Register, moduleName string) ([]string, error) {
module, err := s.loadModule(r, moduleName)
if err != nil {
return nil, errors.Wrapf(err, "failed to get metricsets for module '%s'", moduleName)
return nil, errors.Wrapf(err, "getting metricsets for module '%s'", moduleName)
}
metricsets := make([]string, 0, len(module.MetricSets))
for _, ms := range module.MetricSets {
Expand All @@ -105,7 +107,7 @@ func (s *LightModulesSource) HasMetricSet(moduleName, metricSetName string) bool

moduleConfig, err := s.loadModuleConfig(modulePath)
if err != nil {
logp.Error(errors.Wrapf(err, "failed to load module config for module '%s'", moduleName))
s.log.Errorf("Failed to load module config for module '%s': %v", moduleName, err)
return false
}

Expand All @@ -121,7 +123,7 @@ func (s *LightModulesSource) HasMetricSet(moduleName, metricSetName string) bool
func (s *LightModulesSource) MetricSetRegistration(register *Register, moduleName, metricSetName string) (MetricSetRegistration, error) {
lightModule, err := s.loadModule(register, moduleName)
if err != nil {
return MetricSetRegistration{}, errors.Wrapf(err, "failed to load module '%s'", moduleName)
return MetricSetRegistration{}, errors.Wrapf(err, "loading module '%s'", moduleName)
}

ms, found := lightModule.MetricSets[metricSetName]
Expand All @@ -135,9 +137,15 @@ func (s *LightModulesSource) MetricSetRegistration(register *Register, moduleNam
// ModulesInfo returns a string representation of this source, with a list of known metricsets
func (s *LightModulesSource) ModulesInfo(r *Register) string {
var metricSets []string
modules, _ := s.Modules()
modules, err := s.Modules()
if err != nil {
s.log.Errorf("Failed to list modules: %s", err)
}
for _, module := range modules {
moduleMetricSets, _ := s.MetricSets(r, module)
moduleMetricSets, err := s.MetricSets(r, module)
if err != nil {
s.log.Errorf("Failed to list light metricsets for module %s: %v", module, err)
}
for _, name := range moduleMetricSets {
metricSets = append(metricSets, fmt.Sprintf("%s/%s", module, name))
}
Expand All @@ -155,7 +163,7 @@ type lightModuleConfig struct {
func (s *LightModulesSource) ProcessorsForMetricSet(r *Register, moduleName string, metricSetName string) (*processors.Processors, error) {
module, err := s.loadModule(r, moduleName)
if err != nil {
return nil, errors.Wrapf(err, "reading processors for metricset '%s' in module '%s' failed", metricSetName, moduleName)
return nil, errors.Wrapf(err, "reading processors for metricset '%s' in module '%s'", metricSetName, moduleName)
}
metricSet, ok := module.MetricSets[metricSetName]
if !ok {
Expand All @@ -178,12 +186,12 @@ func (s *LightModulesSource) loadModule(register *Register, moduleName string) (

moduleConfig, err := s.loadModuleConfig(modulePath)
if err != nil {
return nil, errors.Wrapf(err, "failed to load light module '%s' definition", moduleName)
return nil, errors.Wrapf(err, "loading light module '%s' definition", moduleName)
}

metricSets, err := s.loadMetricSets(register, filepath.Dir(modulePath), moduleConfig.Name, moduleConfig.MetricSets)
if err != nil {
return nil, errors.Wrapf(err, "failed to load metric sets for light module '%s'", moduleName)
return nil, errors.Wrapf(err, "loading metric sets for light module '%s'", moduleName)
}

return &LightModule{Name: moduleName, MetricSets: metricSets}, nil
Expand All @@ -202,12 +210,12 @@ func (s *LightModulesSource) findModulePath(moduleName string) (string, bool) {
func (s *LightModulesSource) loadModuleConfig(modulePath string) (*lightModuleConfig, error) {
config, err := common.LoadFile(modulePath)
if err != nil {
return nil, errors.Wrapf(err, "failed to load module configuration from '%s'", modulePath)
return nil, errors.Wrapf(err, "loading module configuration from '%s'", modulePath)
}

var moduleConfig lightModuleConfig
if err = config.Unpack(&moduleConfig); err != nil {
return nil, errors.Wrapf(err, "failed to parse light module definition from '%s'", modulePath)
return nil, errors.Wrapf(err, "parsing light module definition from '%s'", modulePath)
}
return &moduleConfig, nil
}
Expand All @@ -225,7 +233,7 @@ func (s *LightModulesSource) loadMetricSets(register *Register, moduleDirPath, m

metricSetConfig, err := s.loadMetricSetConfig(manifestPath)
if err != nil {
return nil, errors.Wrapf(err, "failed to load light metricset '%s'", metricSet)
return nil, errors.Wrapf(err, "loading light metricset '%s'", metricSet)
}
metricSetConfig.Name = metricSet
metricSetConfig.Module = moduleName
Expand All @@ -238,11 +246,11 @@ func (s *LightModulesSource) loadMetricSets(register *Register, moduleDirPath, m
func (s *LightModulesSource) loadMetricSetConfig(manifestPath string) (ms LightMetricSet, err error) {
config, err := common.LoadFile(manifestPath)
if err != nil {
return ms, errors.Wrapf(err, "failed to load metricset manifest from '%s'", manifestPath)
return ms, errors.Wrapf(err, "loading metricset manifest from '%s'", manifestPath)
}

if err := config.Unpack(&ms); err != nil {
return ms, errors.Wrapf(err, "failed to parse metricset manifest from '%s'", manifestPath)
return ms, errors.Wrapf(err, "parsing metricset manifest from '%s'", manifestPath)
}
return
}
Expand All @@ -252,7 +260,7 @@ func (s *LightModulesSource) moduleNames() ([]string, error) {
for _, dir := range s.paths {
files, err := ioutil.ReadDir(dir)
if err != nil {
return nil, errors.Wrapf(err, "failed to list modules on path '%s'", dir)
return nil, errors.Wrapf(err, "listing modules on path '%s'", dir)
}
for _, f := range files {
modulePath := filepath.Join(dir, f.Name(), moduleYML)
Expand Down