Skip to content

Commit

Permalink
Use the Beat version in the Ingest Node pipeline (#3516)
Browse files Browse the repository at this point in the history 
This adds the Beat version to the pipeline ID, which means that if
we change the pipeline between versions, the new version will be used
automatically. It also means that one can run different versions of the
same Beat and the pipelines won't override each other. The pipelines
are loaded automatically on the Beat start.

Part of #3159.
  • Loading branch information
@tsg @ruflin
tsg authored and ruflin committed Feb 3, 2017
1 parent 65ead9d commit 42686b0
Show file tree
Hide file tree
Showing 6 changed files with 27 additions and 26 deletions.
2 changes: 1 addition & 1 deletion filebeat/beater/filebeat.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ func New(b *beat.Beat, rawConfig *common.Config) (beat.Beater, error) {
return nil, fmt.Errorf("Error reading config file: %v", err)
}

moduleRegistry, err := fileset.NewModuleRegistry(config.Modules)
moduleRegistry, err := fileset.NewModuleRegistry(config.Modules, b.Version)
if err != nil {
return nil, err
}
Expand Down
14 changes: 7 additions & 7 deletions filebeat/fileset/fileset.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ func New(
}

// Read reads the manifest file and evaluates the variables.
func (fs *Fileset) Read() error {
func (fs *Fileset) Read(beatVersion string) error {
var err error
fs.manifest, err = fs.readManifest()
if err != nil {
Expand All @@ -63,7 +63,7 @@ func (fs *Fileset) Read() error {
return err
}

fs.pipelineID, err = fs.getPipelineID()
fs.pipelineID, err = fs.getPipelineID(beatVersion)
if err != nil {
return err
}
Expand Down Expand Up @@ -241,13 +241,13 @@ func (fs *Fileset) getProspectorConfig() (*common.Config, error) {
}

// getPipelineID returns the Ingest Node pipeline ID
func (fs *Fileset) getPipelineID() (string, error) {
func (fs *Fileset) getPipelineID(beatVersion string) (string, error) {
path, err := applyTemplate(fs.vars, fs.manifest.IngestPipeline)
if err != nil {
return "", fmt.Errorf("Error expanding vars on the ingest pipeline path: %v", err)
}

return formatPipelineID(fs.mcfg.Module, fs.name, path), nil
return formatPipelineID(fs.mcfg.Module, fs.name, path, beatVersion), nil
}

func (fs *Fileset) GetPipeline() (pipelineID string, content map[string]interface{}, err error) {
Expand All @@ -266,12 +266,12 @@ func (fs *Fileset) GetPipeline() (pipelineID string, content map[string]interfac
if err != nil {
return "", nil, fmt.Errorf("Error JSON decoding the pipeline file: %s: %v", path, err)
}
return formatPipelineID(fs.mcfg.Module, fs.name, path), content, nil
return fs.pipelineID, content, nil
}

// formatPipelineID generates the ID to be used for the pipeline ID in Elasticsearch
func formatPipelineID(module, fileset, path string) string {
return fmt.Sprintf("%s-%s-%s", module, fileset, removeExt(filepath.Base(path)))
func formatPipelineID(module, fileset, path, beatVersion string) string {
return fmt.Sprintf("filebeat-%s-%s-%s-%s", beatVersion, module, fileset, removeExt(filepath.Base(path)))
}

// removeExt returns the file name without the extension. If no dot is found,
Expand Down
12 changes: 6 additions & 6 deletions filebeat/fileset/fileset_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -149,7 +149,7 @@ func TestResolveVariable(t *testing.T) {

func TestGetProspectorConfigNginx(t *testing.T) {
fs := getModuleForTesting(t, "nginx", "access")
assert.NoError(t, fs.Read())
assert.NoError(t, fs.Read("5.2.0"))

cfg, err := fs.getProspectorConfig()
assert.NoError(t, err)
Expand All @@ -159,7 +159,7 @@ func TestGetProspectorConfigNginx(t *testing.T) {
assert.True(t, cfg.HasField("pipeline"))
pipelineID, err := cfg.String("pipeline", -1)
assert.NoError(t, err)
assert.Equal(t, "nginx-access-with_plugins", pipelineID)
assert.Equal(t, "filebeat-5.2.0-nginx-access-with_plugins", pipelineID)
}

func TestGetProspectorConfigNginxOverrides(t *testing.T) {
Expand All @@ -172,7 +172,7 @@ func TestGetProspectorConfigNginxOverrides(t *testing.T) {
})
assert.NoError(t, err)

assert.NoError(t, fs.Read())
assert.NoError(t, fs.Read("5.2.0"))

cfg, err := fs.getProspectorConfig()
assert.NoError(t, err)
Expand All @@ -183,17 +183,17 @@ func TestGetProspectorConfigNginxOverrides(t *testing.T) {
assert.True(t, cfg.HasField("pipeline"))
pipelineID, err := cfg.String("pipeline", -1)
assert.NoError(t, err)
assert.Equal(t, "nginx-access-with_plugins", pipelineID)
assert.Equal(t, "filebeat-5.2.0-nginx-access-with_plugins", pipelineID)

}

func TestGetPipelineNginx(t *testing.T) {
fs := getModuleForTesting(t, "nginx", "access")
assert.NoError(t, fs.Read())
assert.NoError(t, fs.Read("5.2.0"))

pipelineID, content, err := fs.GetPipeline()
assert.NoError(t, err)
assert.Equal(t, "nginx-access-with_plugins", pipelineID)
assert.Equal(t, "filebeat-5.2.0-nginx-access-with_plugins", pipelineID)
assert.Contains(t, content, "description")
assert.Contains(t, content, "processors")
}
9 changes: 5 additions & 4 deletions filebeat/fileset/modules.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,8 @@ type ModuleRegistry struct {
// newModuleRegistry reads and loads the configured module into the registry.
func newModuleRegistry(modulesPath string,
moduleConfigs []ModuleConfig,
overrides *ModuleOverrides) (*ModuleRegistry, error) {
overrides *ModuleOverrides,
beatVersion string) (*ModuleRegistry, error) {

var reg ModuleRegistry
reg.registry = map[string]map[string]*Fileset{}
Expand Down Expand Up @@ -53,7 +54,7 @@ func newModuleRegistry(modulesPath string,
if err != nil {
return nil, err
}
err = fileset.Read()
err = fileset.Read(beatVersion)
if err != nil {
return nil, fmt.Errorf("Error reading fileset %s/%s: %v", mcfg.Module, filesetName, err)
}
Expand Down Expand Up @@ -81,7 +82,7 @@ func newModuleRegistry(modulesPath string,
}

// NewModuleRegistry reads and loads the configured module into the registry.
func NewModuleRegistry(moduleConfigs []*common.Config) (*ModuleRegistry, error) {
func NewModuleRegistry(moduleConfigs []*common.Config, beatVersion string) (*ModuleRegistry, error) {
modulesPath := paths.Resolve(paths.Home, "module")

stat, err := os.Stat(modulesPath)
Expand All @@ -106,7 +107,7 @@ func NewModuleRegistry(moduleConfigs []*common.Config) (*ModuleRegistry, error)
if err != nil {
return nil, err
}
return newModuleRegistry(modulesPath, mcfgs, modulesOverrides)
return newModuleRegistry(modulesPath, mcfgs, modulesOverrides, beatVersion)
}

func mcfgFromConfig(cfg *common.Config) (*ModuleConfig, error) {
Expand Down
10 changes: 5 additions & 5 deletions filebeat/fileset/modules_integration_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,8 +51,8 @@ func TestLoadPipeline(t *testing.T) {

func TestSetupNginx(t *testing.T) {
client := elasticsearch.GetTestingElasticsearch()
client.Request("DELETE", "/_ingest/pipeline/nginx-access-with_plugins", "", nil, nil)
client.Request("DELETE", "/_ingest/pipeline/nginx-error-pipeline", "", nil, nil)
client.Request("DELETE", "/_ingest/pipeline/filebeat-5.2.0-nginx-access-with_plugins", "", nil, nil)
client.Request("DELETE", "/_ingest/pipeline/filebeat-5.2.0-nginx-error-pipeline", "", nil, nil)

modulesPath, err := filepath.Abs("../module")
assert.NoError(t, err)
Expand All @@ -61,14 +61,14 @@ func TestSetupNginx(t *testing.T) {
{Module: "nginx"},
}

reg, err := newModuleRegistry(modulesPath, configs, nil)
reg, err := newModuleRegistry(modulesPath, configs, nil, "5.2.0")
assert.NoError(t, err)

err = reg.LoadPipelines(client)
assert.NoError(t, err)

status, _, _ := client.Request("GET", "/_ingest/pipeline/nginx-access-with_plugins", "", nil, nil)
status, _, _ := client.Request("GET", "/_ingest/pipeline/filebeat-5.2.0-nginx-access-with_plugins", "", nil, nil)
assert.Equal(t, 200, status)
status, _, _ = client.Request("GET", "/_ingest/pipeline/nginx-error-pipeline", "", nil, nil)
status, _, _ = client.Request("GET", "/_ingest/pipeline/filebeat-5.2.0-nginx-error-pipeline", "", nil, nil)
assert.Equal(t, 200, status)
}
6 changes: 3 additions & 3 deletions filebeat/fileset/modules_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ func TestNewModuleRegistry(t *testing.T) {
{Module: "system"},
}

reg, err := newModuleRegistry(modulesPath, configs, nil)
reg, err := newModuleRegistry(modulesPath, configs, nil, "5.2.0")
assert.NoError(t, err)
assert.NotNil(t, reg)

Expand Down Expand Up @@ -86,7 +86,7 @@ func TestNewModuleRegistryConfig(t *testing.T) {
},
}

reg, err := newModuleRegistry(modulesPath, configs, nil)
reg, err := newModuleRegistry(modulesPath, configs, nil, "5.2.0")
assert.NoError(t, err)
assert.NotNil(t, reg)

Expand Down Expand Up @@ -335,7 +335,7 @@ func TestMissingModuleFolder(t *testing.T) {
load(t, map[string]interface{}{"module": "nginx"}),
}

reg, err := NewModuleRegistry(configs)
reg, err := NewModuleRegistry(configs, "5.2.0")
assert.NoError(t, err)
assert.NotNil(t, reg)

Expand Down

0 comments on commit 42686b0

Please sign in to comment.