Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use the Beat version in the Ingest Node pipeline #3516

Merged
merged 1 commit into from
Feb 3, 2017
Merged

Use the Beat version in the Ingest Node pipeline #3516

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion filebeat/beater/filebeat.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ func New(b *beat.Beat, rawConfig *common.Config) (beat.Beater, error) {
return nil, fmt.Errorf("Error reading config file: %v", err)
}

moduleRegistry, err := fileset.NewModuleRegistry(config.Modules)
moduleRegistry, err := fileset.NewModuleRegistry(config.Modules, b.Version)
if err != nil {
return nil, err
}
Expand Down
14 changes: 7 additions & 7 deletions filebeat/fileset/fileset.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ func New(
}

// Read reads the manifest file and evaluates the variables.
func (fs *Fileset) Read() error {
func (fs *Fileset) Read(beatVersion string) error {
var err error
fs.manifest, err = fs.readManifest()
if err != nil {
Expand All @@ -63,7 +63,7 @@ func (fs *Fileset) Read() error {
return err
}

fs.pipelineID, err = fs.getPipelineID()
fs.pipelineID, err = fs.getPipelineID(beatVersion)
if err != nil {
return err
}
Expand Down Expand Up @@ -241,13 +241,13 @@ func (fs *Fileset) getProspectorConfig() (*common.Config, error) {
}

// getPipelineID returns the Ingest Node pipeline ID
func (fs *Fileset) getPipelineID() (string, error) {
func (fs *Fileset) getPipelineID(beatVersion string) (string, error) {
path, err := applyTemplate(fs.vars, fs.manifest.IngestPipeline)
if err != nil {
return "", fmt.Errorf("Error expanding vars on the ingest pipeline path: %v", err)
}

return formatPipelineID(fs.mcfg.Module, fs.name, path), nil
return formatPipelineID(fs.mcfg.Module, fs.name, path, beatVersion), nil
}

func (fs *Fileset) GetPipeline() (pipelineID string, content map[string]interface{}, err error) {
Expand All @@ -266,12 +266,12 @@ func (fs *Fileset) GetPipeline() (pipelineID string, content map[string]interfac
if err != nil {
return "", nil, fmt.Errorf("Error JSON decoding the pipeline file: %s: %v", path, err)
}
return formatPipelineID(fs.mcfg.Module, fs.name, path), content, nil
return fs.pipelineID, content, nil
}

// formatPipelineID generates the ID to be used for the pipeline ID in Elasticsearch
func formatPipelineID(module, fileset, path string) string {
return fmt.Sprintf("%s-%s-%s", module, fileset, removeExt(filepath.Base(path)))
func formatPipelineID(module, fileset, path, beatVersion string) string {
return fmt.Sprintf("filebeat-%s-%s-%s-%s", beatVersion, module, fileset, removeExt(filepath.Base(path)))
}

// removeExt returns the file name without the extension. If no dot is found,
Expand Down
12 changes: 6 additions & 6 deletions filebeat/fileset/fileset_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -149,7 +149,7 @@ func TestResolveVariable(t *testing.T) {

func TestGetProspectorConfigNginx(t *testing.T) {
fs := getModuleForTesting(t, "nginx", "access")
assert.NoError(t, fs.Read())
assert.NoError(t, fs.Read("5.2.0"))

cfg, err := fs.getProspectorConfig()
assert.NoError(t, err)
Expand All @@ -159,7 +159,7 @@ func TestGetProspectorConfigNginx(t *testing.T) {
assert.True(t, cfg.HasField("pipeline"))
pipelineID, err := cfg.String("pipeline", -1)
assert.NoError(t, err)
assert.Equal(t, "nginx-access-with_plugins", pipelineID)
assert.Equal(t, "filebeat-5.2.0-nginx-access-with_plugins", pipelineID)
}

func TestGetProspectorConfigNginxOverrides(t *testing.T) {
Expand All @@ -172,7 +172,7 @@ func TestGetProspectorConfigNginxOverrides(t *testing.T) {
})
assert.NoError(t, err)

assert.NoError(t, fs.Read())
assert.NoError(t, fs.Read("5.2.0"))

cfg, err := fs.getProspectorConfig()
assert.NoError(t, err)
Expand All @@ -183,17 +183,17 @@ func TestGetProspectorConfigNginxOverrides(t *testing.T) {
assert.True(t, cfg.HasField("pipeline"))
pipelineID, err := cfg.String("pipeline", -1)
assert.NoError(t, err)
assert.Equal(t, "nginx-access-with_plugins", pipelineID)
assert.Equal(t, "filebeat-5.2.0-nginx-access-with_plugins", pipelineID)

}

func TestGetPipelineNginx(t *testing.T) {
fs := getModuleForTesting(t, "nginx", "access")
assert.NoError(t, fs.Read())
assert.NoError(t, fs.Read("5.2.0"))

pipelineID, content, err := fs.GetPipeline()
assert.NoError(t, err)
assert.Equal(t, "nginx-access-with_plugins", pipelineID)
assert.Equal(t, "filebeat-5.2.0-nginx-access-with_plugins", pipelineID)
assert.Contains(t, content, "description")
assert.Contains(t, content, "processors")
}
9 changes: 5 additions & 4 deletions filebeat/fileset/modules.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,8 @@ type ModuleRegistry struct {
// newModuleRegistry reads and loads the configured module into the registry.
func newModuleRegistry(modulesPath string,
moduleConfigs []ModuleConfig,
overrides *ModuleOverrides) (*ModuleRegistry, error) {
overrides *ModuleOverrides,
beatVersion string) (*ModuleRegistry, error) {

var reg ModuleRegistry
reg.registry = map[string]map[string]*Fileset{}
Expand Down Expand Up @@ -53,7 +54,7 @@ func newModuleRegistry(modulesPath string,
if err != nil {
return nil, err
}
err = fileset.Read()
err = fileset.Read(beatVersion)
if err != nil {
return nil, fmt.Errorf("Error reading fileset %s/%s: %v", mcfg.Module, filesetName, err)
}
Expand Down Expand Up @@ -81,7 +82,7 @@ func newModuleRegistry(modulesPath string,
}

// NewModuleRegistry reads and loads the configured module into the registry.
func NewModuleRegistry(moduleConfigs []*common.Config) (*ModuleRegistry, error) {
func NewModuleRegistry(moduleConfigs []*common.Config, beatVersion string) (*ModuleRegistry, error) {
modulesPath := paths.Resolve(paths.Home, "module")

stat, err := os.Stat(modulesPath)
Expand All @@ -106,7 +107,7 @@ func NewModuleRegistry(moduleConfigs []*common.Config) (*ModuleRegistry, error)
if err != nil {
return nil, err
}
return newModuleRegistry(modulesPath, mcfgs, modulesOverrides)
return newModuleRegistry(modulesPath, mcfgs, modulesOverrides, beatVersion)
}

func mcfgFromConfig(cfg *common.Config) (*ModuleConfig, error) {
Expand Down
10 changes: 5 additions & 5 deletions filebeat/fileset/modules_integration_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,8 +35,8 @@ func TestLoadPipeline(t *testing.T) {

func TestSetupNginx(t *testing.T) {
client := elasticsearch.GetTestingElasticsearch()
client.Request("DELETE", "/_ingest/pipeline/nginx-access-with_plugins", "", nil, nil)
client.Request("DELETE", "/_ingest/pipeline/nginx-error-pipeline", "", nil, nil)
client.Request("DELETE", "/_ingest/pipeline/filebeat-5.2.0-nginx-access-with_plugins", "", nil, nil)
client.Request("DELETE", "/_ingest/pipeline/filebeat-5.2.0-nginx-error-pipeline", "", nil, nil)

modulesPath, err := filepath.Abs("../module")
assert.NoError(t, err)
Expand All @@ -45,14 +45,14 @@ func TestSetupNginx(t *testing.T) {
{Module: "nginx"},
}

reg, err := newModuleRegistry(modulesPath, configs, nil)
reg, err := newModuleRegistry(modulesPath, configs, nil, "5.2.0")
assert.NoError(t, err)

err = reg.LoadPipelines(client)
assert.NoError(t, err)

status, _, _ := client.Request("GET", "/_ingest/pipeline/nginx-access-with_plugins", "", nil, nil)
status, _, _ := client.Request("GET", "/_ingest/pipeline/filebeat-5.2.0-nginx-access-with_plugins", "", nil, nil)
assert.Equal(t, 200, status)
status, _, _ = client.Request("GET", "/_ingest/pipeline/nginx-error-pipeline", "", nil, nil)
status, _, _ = client.Request("GET", "/_ingest/pipeline/filebeat-5.2.0-nginx-error-pipeline", "", nil, nil)
assert.Equal(t, 200, status)
}
6 changes: 3 additions & 3 deletions filebeat/fileset/modules_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ func TestNewModuleRegistry(t *testing.T) {
{Module: "system"},
}

reg, err := newModuleRegistry(modulesPath, configs, nil)
reg, err := newModuleRegistry(modulesPath, configs, nil, "5.2.0")
assert.NoError(t, err)
assert.NotNil(t, reg)

Expand Down Expand Up @@ -86,7 +86,7 @@ func TestNewModuleRegistryConfig(t *testing.T) {
},
}

reg, err := newModuleRegistry(modulesPath, configs, nil)
reg, err := newModuleRegistry(modulesPath, configs, nil, "5.2.0")
assert.NoError(t, err)
assert.NotNil(t, reg)

Expand Down Expand Up @@ -335,7 +335,7 @@ func TestMissingModuleFolder(t *testing.T) {
load(t, map[string]interface{}{"module": "nginx"}),
}

reg, err := NewModuleRegistry(configs)
reg, err := NewModuleRegistry(configs, "5.2.0")
assert.NoError(t, err)
assert.NotNil(t, reg)

Expand Down