diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 72c80626c93..54a0ded55d2 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -529,6 +529,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add add_resource_metadata option setting (always enabled) for add_kubernetes_metadata setting. {pull}22189[22189] - Added Kafka version 2.2 to the list of supported versions. {pull}22328[22328] - Add support for ephemeral containers in kubernetes autodiscover and `add_kubernetes_metadata`. {pull}22389[22389] {pull}22439[22439] +- Added support for wildcard fields and keyword fallback in beats setup commands. {pull}22521[22521] *Auditbeat* diff --git a/auditbeat/cmd/root.go b/auditbeat/cmd/root.go index bf3167e7106..8995f4f98f5 100644 --- a/auditbeat/cmd/root.go +++ b/auditbeat/cmd/root.go @@ -54,19 +54,29 @@ var withECSVersion = processing.WithFields(common.MapStr{ }, }) -func init() { - create := beater.Creator( - beater.WithModuleOptions( - module.WithEventModifier(core.AddDatasetToEvent), - ), - ) +// AuditbeatSettings contains the default settings for auditbeat +func AuditbeatSettings() instance.Settings { var runFlags = pflag.NewFlagSet(Name, pflag.ExitOnError) - settings := instance.Settings{ + return instance.Settings{ RunFlags: runFlags, Name: Name, HasDashboards: true, Processing: processing.MakeDefaultSupport(true, withECSVersion, processing.WithHost, processing.WithAgentMeta()), } - RootCmd = cmd.GenRootCmdWithSettings(create, settings) - RootCmd.AddCommand(ShowCmd) +} + +// Initialize initializes the entrypoint commands for journalbeat +func Initialize(settings instance.Settings) *cmd.BeatsRootCmd { + create := beater.Creator( + beater.WithModuleOptions( + module.WithEventModifier(core.AddDatasetToEvent), + ), + ) + rootCmd := cmd.GenRootCmdWithSettings(create, settings) + rootCmd.AddCommand(ShowCmd) + return rootCmd +} + +func init() { + RootCmd = Initialize(AuditbeatSettings()) } diff --git a/auditbeat/main_test.go b/auditbeat/main_test.go index d6e81c2aad2..54222c69ac5 100644 --- a/auditbeat/main_test.go +++ b/auditbeat/main_test.go @@ -45,5 +45,5 @@ func TestSystem(t *testing.T) { } func TestTemplate(t *testing.T) { - template.TestTemplate(t, cmd.Name) + template.TestTemplate(t, cmd.Name, false) } diff --git a/filebeat/cmd/root.go b/filebeat/cmd/root.go index f2062a35dd7..2b9bff54629 100644 --- a/filebeat/cmd/root.go +++ b/filebeat/cmd/root.go @@ -35,17 +35,23 @@ import ( // Name of this beat const Name = "filebeat" -// Filebeat build the beat root command for executing filebeat and it's subcommands. -func Filebeat(inputs beater.PluginFactory) *cmd.BeatsRootCmd { +// RootCmd to handle beats cli +var RootCmd *cmd.BeatsRootCmd + +// FilebeatSettings contains the default settings for filebeat +func FilebeatSettings() instance.Settings { var runFlags = pflag.NewFlagSet(Name, pflag.ExitOnError) runFlags.AddGoFlag(flag.CommandLine.Lookup("once")) runFlags.AddGoFlag(flag.CommandLine.Lookup("modules")) - settings := instance.Settings{ + return instance.Settings{ RunFlags: runFlags, Name: Name, HasDashboards: true, } +} +// Filebeat build the beat root command for executing filebeat and it's subcommands. +func Filebeat(inputs beater.PluginFactory, settings instance.Settings) *cmd.BeatsRootCmd { command := cmd.GenRootCmdWithSettings(beater.New(inputs), settings) command.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("M")) command.TestCmd.Flags().AddGoFlag(flag.CommandLine.Lookup("modules")) diff --git a/filebeat/main.go b/filebeat/main.go index e8f36acde01..b7f128a266f 100644 --- a/filebeat/main.go +++ b/filebeat/main.go @@ -33,7 +33,7 @@ import ( // Finally, input uses the registrar information, on restart, to // determine where in each file to restart a harvester. func main() { - if err := cmd.Filebeat(inputs.Init).Execute(); err != nil { + if err := cmd.Filebeat(inputs.Init, cmd.FilebeatSettings()).Execute(); err != nil { os.Exit(1) } } diff --git a/filebeat/main_test.go b/filebeat/main_test.go index 5a16694d1bd..d55a1cc9a7f 100644 --- a/filebeat/main_test.go +++ b/filebeat/main_test.go @@ -36,7 +36,7 @@ var fbCommand *cmd.BeatsRootCmd func init() { testing.Init() systemTest = flag.Bool("systemTest", false, "Set to true when running system tests") - fbCommand = fbcmd.Filebeat(inputs.Init) + fbCommand = fbcmd.Filebeat(inputs.Init, fbcmd.FilebeatSettings()) fbCommand.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("systemTest")) fbCommand.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("test.coverprofile")) } @@ -51,5 +51,5 @@ func TestSystem(t *testing.T) { } func TestTemplate(t *testing.T) { - template.TestTemplate(t, fbCommand.Name()) + template.TestTemplate(t, fbCommand.Name(), false) } diff --git a/heartbeat/cmd/root.go b/heartbeat/cmd/root.go index a2b253a535e..cbe2c2063e9 100644 --- a/heartbeat/cmd/root.go +++ b/heartbeat/cmd/root.go @@ -38,23 +38,28 @@ var Name = "heartbeat" // RootCmd to handle beats cli var RootCmd *cmd.BeatsRootCmd -func init() { - settings := instance.Settings{ +// HeartbeatSettings contains the default settings for heartbeat +func HeartbeatSettings() instance.Settings { + return instance.Settings{ Name: Name, Processing: processing.MakeDefaultSupport(true, processing.WithECS, processing.WithAgentMeta()), HasDashboards: false, } - RootCmd = cmd.GenRootCmdWithSettings(beater.New, settings) +} + +// Initialize initializes the entrypoint commands for heartbeat +func Initialize(settings instance.Settings) *cmd.BeatsRootCmd { + rootCmd := cmd.GenRootCmdWithSettings(beater.New, settings) // remove dashboard from export commands - for _, cmd := range RootCmd.ExportCmd.Commands() { + for _, cmd := range rootCmd.ExportCmd.Commands() { if cmd.Name() == "dashboard" { - RootCmd.ExportCmd.RemoveCommand(cmd) + rootCmd.ExportCmd.RemoveCommand(cmd) } } // only add defined flags to setup command - setup := RootCmd.SetupCmd + setup := rootCmd.SetupCmd setup.Short = "Setup Elasticsearch index template and pipelines" setup.Long = `This command does initial setup of the environment: * Index mapping template in Elasticsearch to ensure fields are mapped. @@ -66,4 +71,10 @@ func init() { setup.Flags().MarkDeprecated(cmd.ILMPolicyKey, fmt.Sprintf("use --%s instead", cmd.IndexManagementKey)) setup.Flags().Bool(cmd.TemplateKey, false, "Setup index template") setup.Flags().Bool(cmd.ILMPolicyKey, false, "Setup ILM policy") + + return rootCmd +} + +func init() { + RootCmd = Initialize(HeartbeatSettings()) } diff --git a/heartbeat/main_test.go b/heartbeat/main_test.go index 7de20114fa0..6ea7780a8b1 100644 --- a/heartbeat/main_test.go +++ b/heartbeat/main_test.go @@ -44,5 +44,5 @@ func TestSystem(t *testing.T) { } func TestTemplate(t *testing.T) { - template.TestTemplate(t, cmd.Name) + template.TestTemplate(t, cmd.Name, false) } diff --git a/journalbeat/cmd/root.go b/journalbeat/cmd/root.go index 166bb251780..8b852e9af01 100644 --- a/journalbeat/cmd/root.go +++ b/journalbeat/cmd/root.go @@ -32,4 +32,18 @@ import ( var Name = "journalbeat" // RootCmd to handle beats cli -var RootCmd = cmd.GenRootCmdWithSettings(beater.New, instance.Settings{Name: Name, HasDashboards: false}) +var RootCmd *cmd.BeatsRootCmd + +// JournalbeatSettings contains the default settings for journalbeat +func JournalbeatSettings() instance.Settings { + return instance.Settings{Name: Name, HasDashboards: false} +} + +// Initialize initializes the entrypoint commands for journalbeat +func Initialize(settings instance.Settings) *cmd.BeatsRootCmd { + return cmd.GenRootCmdWithSettings(beater.New, settings) +} + +func init() { + RootCmd = Initialize(JournalbeatSettings()) +} diff --git a/journalbeat/main_test.go b/journalbeat/main_test.go index 63c0e492604..7881c30d52c 100644 --- a/journalbeat/main_test.go +++ b/journalbeat/main_test.go @@ -45,5 +45,5 @@ func TestSystem(t *testing.T) { } func TestTemplate(t *testing.T) { - template.TestTemplate(t, cmd.Name) + template.TestTemplate(t, cmd.Name, false) } diff --git a/libbeat/beat/info.go b/libbeat/beat/info.go index 63f7ac645d3..c808ba214ea 100644 --- a/libbeat/beat/info.go +++ b/libbeat/beat/info.go @@ -21,13 +21,14 @@ import "github.com/gofrs/uuid" // Info stores a beats instance meta data. type Info struct { - Beat string // The actual beat's name - IndexPrefix string // The beat's index prefix in Elasticsearch. - Version string // The beat version. Defaults to the libbeat version when an implementation does not set a version - Name string // configured beat name - Hostname string // hostname - ID uuid.UUID // ID assigned to beat machine - EphemeralID uuid.UUID // ID assigned to beat process invocation (PID) + Beat string // The actual beat's name + IndexPrefix string // The beat's index prefix in Elasticsearch. + Version string // The beat version. Defaults to the libbeat version when an implementation does not set a version + ElasticLicensed bool // Whether the beat is licensed under and Elastic License + Name string // configured beat name + Hostname string // hostname + ID uuid.UUID // ID assigned to beat machine + EphemeralID uuid.UUID // ID assigned to beat process invocation (PID) // Monitoring-related fields Monitoring struct { diff --git a/libbeat/cmd/instance/beat.go b/libbeat/cmd/instance/beat.go index 714b266ea84..b873ddebf95 100644 --- a/libbeat/cmd/instance/beat.go +++ b/libbeat/cmd/instance/beat.go @@ -156,6 +156,7 @@ func Run(settings Settings, bt beat.Creator) error { name := settings.Name idxPrefix := settings.IndexPrefix version := settings.Version + elasticLicensed := settings.ElasticLicensed return handleError(func() error { defer func() { @@ -164,7 +165,7 @@ func Run(settings Settings, bt beat.Creator) error { "panic", r, zap.Stack("stack")) } }() - b, err := NewBeat(name, idxPrefix, version) + b, err := NewBeat(name, idxPrefix, version, elasticLicensed) if err != nil { return err } @@ -191,7 +192,7 @@ func Run(settings Settings, bt beat.Creator) error { // NewInitializedBeat creates a new beat where all information and initialization is derived from settings func NewInitializedBeat(settings Settings) (*Beat, error) { - b, err := NewBeat(settings.Name, settings.IndexPrefix, settings.Version) + b, err := NewBeat(settings.Name, settings.IndexPrefix, settings.Version, settings.ElasticLicensed) if err != nil { return nil, err } @@ -202,7 +203,7 @@ func NewInitializedBeat(settings Settings) (*Beat, error) { } // NewBeat creates a new beat instance -func NewBeat(name, indexPrefix, v string) (*Beat, error) { +func NewBeat(name, indexPrefix, v string, elasticLicensed bool) (*Beat, error) { if v == "" { v = version.GetDefaultVersion() } @@ -227,13 +228,14 @@ func NewBeat(name, indexPrefix, v string) (*Beat, error) { b := beat.Beat{ Info: beat.Info{ - Beat: name, - IndexPrefix: indexPrefix, - Version: v, - Name: hostname, - Hostname: hostname, - ID: id, - EphemeralID: ephemeralID, + Beat: name, + ElasticLicensed: elasticLicensed, + IndexPrefix: indexPrefix, + Version: v, + Name: hostname, + Hostname: hostname, + ID: id, + EphemeralID: ephemeralID, }, Fields: fields, } diff --git a/libbeat/cmd/instance/beat_test.go b/libbeat/cmd/instance/beat_test.go index 8c04e87390f..e302bed1711 100644 --- a/libbeat/cmd/instance/beat_test.go +++ b/libbeat/cmd/instance/beat_test.go @@ -31,7 +31,7 @@ import ( ) func TestNewInstance(t *testing.T) { - b, err := NewBeat("testbeat", "testidx", "0.9") + b, err := NewBeat("testbeat", "testidx", "0.9", false) if err != nil { panic(err) } @@ -45,7 +45,7 @@ func TestNewInstance(t *testing.T) { assert.Equal(t, 36, len(b.Info.ID.String())) // indexPrefix set to name if empty - b, err = NewBeat("testbeat", "", "0.9") + b, err = NewBeat("testbeat", "", "0.9", false) if err != nil { panic(err) } @@ -55,7 +55,7 @@ func TestNewInstance(t *testing.T) { } func TestNewInstanceUUID(t *testing.T) { - b, err := NewBeat("testbeat", "", "0.9") + b, err := NewBeat("testbeat", "", "0.9", false) if err != nil { panic(err) } @@ -69,7 +69,7 @@ func TestNewInstanceUUID(t *testing.T) { } func TestInitKibanaConfig(t *testing.T) { - b, err := NewBeat("filebeat", "testidx", "0.9") + b, err := NewBeat("filebeat", "testidx", "0.9", false) if err != nil { panic(err) } @@ -96,7 +96,7 @@ func TestInitKibanaConfig(t *testing.T) { } func TestEmptyMetaJson(t *testing.T) { - b, err := NewBeat("filebeat", "testidx", "0.9") + b, err := NewBeat("filebeat", "testidx", "0.9", false) if err != nil { panic(err) } diff --git a/libbeat/cmd/instance/settings.go b/libbeat/cmd/instance/settings.go index ee22dc084e0..3d64ccc1fa1 100644 --- a/libbeat/cmd/instance/settings.go +++ b/libbeat/cmd/instance/settings.go @@ -33,6 +33,7 @@ type Settings struct { IndexPrefix string Version string HasDashboards bool + ElasticLicensed bool Monitoring report.Settings RunFlags *pflag.FlagSet ConfigOverrides []cfgfile.ConditionalOverride diff --git a/libbeat/cmd/setup.go b/libbeat/cmd/setup.go index 6fa69d48d6a..711ddfd027c 100644 --- a/libbeat/cmd/setup.go +++ b/libbeat/cmd/setup.go @@ -58,7 +58,7 @@ func genSetupCmd(settings instance.Settings, beatCreator beat.Creator) *cobra.Co * ILM policy (for Elasticsearch 6.5 and newer). `, Run: func(cmd *cobra.Command, args []string) { - beat, err := instance.NewBeat(settings.Name, settings.IndexPrefix, settings.Version) + beat, err := instance.NewBeat(settings.Name, settings.IndexPrefix, settings.Version, settings.ElasticLicensed) if err != nil { fmt.Fprintf(os.Stderr, "Error initializing beat: %s\n", err) os.Exit(1) diff --git a/libbeat/cmd/test/config.go b/libbeat/cmd/test/config.go index dfc93344acf..7e93a618c56 100644 --- a/libbeat/cmd/test/config.go +++ b/libbeat/cmd/test/config.go @@ -32,7 +32,7 @@ func GenTestConfigCmd(settings instance.Settings, beatCreator beat.Creator) *cob Use: "config", Short: "Test configuration settings", Run: func(cmd *cobra.Command, args []string) { - b, err := instance.NewBeat(settings.Name, settings.IndexPrefix, settings.Version) + b, err := instance.NewBeat(settings.Name, settings.IndexPrefix, settings.Version, settings.ElasticLicensed) if err != nil { fmt.Fprintf(os.Stderr, "Error initializing beat: %s\n", err) os.Exit(1) diff --git a/libbeat/cmd/version.go b/libbeat/cmd/version.go index 7019ccd77c9..243c9e04c02 100644 --- a/libbeat/cmd/version.go +++ b/libbeat/cmd/version.go @@ -35,7 +35,7 @@ func GenVersionCmd(settings instance.Settings) *cobra.Command { Short: "Show current version info", Run: cli.RunWith( func(_ *cobra.Command, args []string) error { - beat, err := instance.NewBeat(settings.Name, settings.IndexPrefix, settings.Version) + beat, err := instance.NewBeat(settings.Name, settings.IndexPrefix, settings.Version, settings.ElasticLicensed) if err != nil { return fmt.Errorf("error initializing beat: %s", err) } diff --git a/libbeat/libbeat_test.go b/libbeat/libbeat_test.go index d1d06506c87..e4bac5e309d 100644 --- a/libbeat/libbeat_test.go +++ b/libbeat/libbeat_test.go @@ -42,5 +42,5 @@ func TestSystem(t *testing.T) { } func TestTemplate(t *testing.T) { - template.TestTemplate(t, "mockbeat") + template.TestTemplate(t, "mockbeat", false) } diff --git a/libbeat/template/load.go b/libbeat/template/load.go index 5b0e0f58440..054869998d7 100644 --- a/libbeat/template/load.go +++ b/libbeat/template/load.go @@ -183,7 +183,7 @@ func (b *templateBuilder) template(config TemplateConfig, info beat.Info, esVers b.log.Info("template config not enabled") return nil, nil } - tmpl, err := New(info.Version, info.IndexPrefix, esVersion, config, migration) + tmpl, err := New(info.Version, info.IndexPrefix, info.ElasticLicensed, esVersion, config, migration) if err != nil { return nil, fmt.Errorf("error creating template instance: %v", err) } diff --git a/libbeat/template/processor.go b/libbeat/template/processor.go index 0f372b49838..78decd086fd 100644 --- a/libbeat/template/processor.go +++ b/libbeat/template/processor.go @@ -28,8 +28,9 @@ import ( // Processor struct to process fields to template type Processor struct { - EsVersion common.Version - Migration bool + EsVersion common.Version + Migration bool + ElasticLicensed bool } var ( @@ -73,6 +74,13 @@ func (p *Processor) Process(fields mapping.Fields, state *fieldState, output com indexMapping = p.integer(&field) case "text": indexMapping = p.text(&field) + case "wildcard": + noWildcards := p.EsVersion.LessThan(common.MustNewVersion("7.9.0")) + if !p.ElasticLicensed || noWildcards { + indexMapping = p.keyword(&field) + } else { + indexMapping = p.wildcard(&field) + } case "", "keyword": indexMapping = p.keyword(&field) case "object": @@ -229,6 +237,28 @@ func (p *Processor) keyword(f *mapping.Field) common.MapStr { return property } +func (p *Processor) wildcard(f *mapping.Field) common.MapStr { + property := getDefaultProperties(f) + + property["type"] = "wildcard" + + switch f.IgnoreAbove { + case 0: // Use libbeat default + property["ignore_above"] = defaultIgnoreAbove + case -1: // Use ES default + default: // Use user value + property["ignore_above"] = f.IgnoreAbove + } + + if len(f.MultiFields) > 0 { + fields := common.MapStr{} + p.Process(f.MultiFields, nil, fields) + property["fields"] = fields + } + + return property +} + func (p *Processor) text(f *mapping.Field) common.MapStr { properties := getDefaultProperties(f) diff --git a/libbeat/template/processor_test.go b/libbeat/template/processor_test.go index 2ffc03dad24..78a1588532e 100644 --- a/libbeat/template/processor_test.go +++ b/libbeat/template/processor_test.go @@ -676,3 +676,129 @@ func TestProcessDefaultField(t *testing.T) { "nested.bar", ) } + +func TestProcessWildcardOSS(t *testing.T) { + // Test common fields are combined even if they come from different objects + fields := mapping.Fields{ + mapping.Field{ + Name: "test", + Type: "group", + Fields: mapping.Fields{ + mapping.Field{ + Name: "one", + Type: "wildcard", + }, + }, + }, + } + + output := common.MapStr{} + version, err := common.NewVersion("8.0.0") + if err != nil { + t.Fatal(err) + } + + p := Processor{EsVersion: *version} + err = p.Process(fields, nil, output) + if err != nil { + t.Fatal(err) + } + + // Make sure fields without a name are skipped during template generation + expectedOutput := common.MapStr{ + "test": common.MapStr{ + "properties": common.MapStr{ + "one": common.MapStr{ + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + } + + assert.Equal(t, expectedOutput, output) +} + +func TestProcessWildcardElastic(t *testing.T) { + // Test common fields are combined even if they come from different objects + fields := mapping.Fields{ + mapping.Field{ + Name: "test", + Type: "group", + Fields: mapping.Fields{ + mapping.Field{ + Name: "one", + Type: "wildcard", + }, + }, + }, + } + + output := common.MapStr{} + version, err := common.NewVersion("8.0.0") + if err != nil { + t.Fatal(err) + } + + p := Processor{EsVersion: *version, ElasticLicensed: true} + err = p.Process(fields, nil, output) + if err != nil { + t.Fatal(err) + } + + // Make sure fields without a name are skipped during template generation + expectedOutput := common.MapStr{ + "test": common.MapStr{ + "properties": common.MapStr{ + "one": common.MapStr{ + "ignore_above": 1024, + "type": "wildcard", + }, + }, + }, + } + + assert.Equal(t, expectedOutput, output) +} + +func TestProcessWildcardPreSupport(t *testing.T) { + // Test common fields are combined even if they come from different objects + fields := mapping.Fields{ + mapping.Field{ + Name: "test", + Type: "group", + Fields: mapping.Fields{ + mapping.Field{ + Name: "one", + Type: "wildcard", + }, + }, + }, + } + + output := common.MapStr{} + version, err := common.NewVersion("7.8.0") + if err != nil { + t.Fatal(err) + } + + p := Processor{EsVersion: *version, ElasticLicensed: true} + err = p.Process(fields, nil, output) + if err != nil { + t.Fatal(err) + } + + // Make sure fields without a name are skipped during template generation + expectedOutput := common.MapStr{ + "test": common.MapStr{ + "properties": common.MapStr{ + "one": common.MapStr{ + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + } + + assert.Equal(t, expectedOutput, output) +} diff --git a/libbeat/template/template.go b/libbeat/template/template.go index 2aaa7712d02..0fbfa30ccab 100644 --- a/libbeat/template/template.go +++ b/libbeat/template/template.go @@ -46,22 +46,24 @@ var ( // Template holds information for the ES template. type Template struct { sync.Mutex - name string - pattern string - beatVersion common.Version - beatName string - esVersion common.Version - config TemplateConfig - migration bool - templateType IndexTemplateType - order int - priority int + name string + pattern string + elasticLicensed bool + beatVersion common.Version + beatName string + esVersion common.Version + config TemplateConfig + migration bool + templateType IndexTemplateType + order int + priority int } // New creates a new template instance func New( beatVersion string, beatName string, + elasticLicensed bool, esVersion common.Version, config TemplateConfig, migration bool, @@ -125,16 +127,17 @@ func New( } return &Template{ - pattern: pattern, - name: name, - beatVersion: *bV, - esVersion: esVersion, - beatName: beatName, - config: config, - migration: migration, - templateType: config.Type, - order: config.Order, - priority: config.Priority, + pattern: pattern, + name: name, + elasticLicensed: elasticLicensed, + beatVersion: *bV, + esVersion: esVersion, + beatName: beatName, + config: config, + migration: migration, + templateType: config.Type, + order: config.Order, + priority: config.Priority, }, nil } @@ -157,7 +160,7 @@ func (t *Template) load(fields mapping.Fields) (common.MapStr, error) { // Start processing at the root properties := common.MapStr{} - processor := Processor{EsVersion: t.esVersion, Migration: t.migration} + processor := Processor{EsVersion: t.esVersion, ElasticLicensed: t.elasticLicensed, Migration: t.migration} if err := processor.Process(fields, nil, properties); err != nil { return nil, err } diff --git a/libbeat/template/template_test.go b/libbeat/template/template_test.go index 52080274dd6..d307d17f1fe 100644 --- a/libbeat/template/template_test.go +++ b/libbeat/template/template_test.go @@ -137,7 +137,7 @@ func createTestTemplate(t *testing.T, beatVersion, esVersion string, config Temp beatVersion = getVersion(beatVersion) esVersion = getVersion(esVersion) ver := common.MustNewVersion(esVersion) - template, err := New(beatVersion, "testbeat", *ver, config, false) + template, err := New(beatVersion, "testbeat", false, *ver, config, false) if err != nil { t.Fatalf("Failed to create the template: %+v", err) } diff --git a/libbeat/tests/system/template/template.go b/libbeat/tests/system/template/template.go index 3005662d195..09c9e0dcb19 100644 --- a/libbeat/tests/system/template/template.go +++ b/libbeat/tests/system/template/template.go @@ -33,13 +33,13 @@ import ( const MaxDefaultFieldLength = 1000 // TestTemplate executes tests on the Beat's index template. -func TestTemplate(t *testing.T, beatName string) { - t.Run("default_field length", testTemplateDefaultFieldLength(beatName)) +func TestTemplate(t *testing.T, beatName string, elasticLicensed bool) { + t.Run("default_field length", testTemplateDefaultFieldLength(beatName, elasticLicensed)) } // testTemplateDefaultFieldLength constructs a template based on the embedded // fields.yml data verifies that the length is less than 1000. -func testTemplateDefaultFieldLength(beatName string) func(*testing.T) { +func testTemplateDefaultFieldLength(beatName string, elasticLicensed bool) func(*testing.T) { return func(t *testing.T) { // 7.0 is when default_field was introduced. esVersion, err := common.NewVersion("7.0.0") @@ -48,7 +48,7 @@ func testTemplateDefaultFieldLength(beatName string) func(*testing.T) { } // Generate a template based on the embedded fields.yml data. - tmpl, err := template.New(version.GetDefaultVersion(), beatName, *esVersion, template.TemplateConfig{}, false) + tmpl, err := template.New(version.GetDefaultVersion(), beatName, elasticLicensed, *esVersion, template.TemplateConfig{}, false) if err != nil { t.Fatal(err) } diff --git a/metricbeat/cmd/root.go b/metricbeat/cmd/root.go index 892d734b8e2..2df515d9625 100644 --- a/metricbeat/cmd/root.go +++ b/metricbeat/cmd/root.go @@ -41,15 +41,25 @@ var Name = "metricbeat" // RootCmd to handle beats cli var RootCmd *cmd.BeatsRootCmd -func init() { +// MetricbeatSettings contains the default settings for metricbeat +func MetricbeatSettings() instance.Settings { var runFlags = pflag.NewFlagSet(Name, pflag.ExitOnError) runFlags.AddGoFlag(flag.CommandLine.Lookup("system.hostfs")) - settings := instance.Settings{ + return instance.Settings{ RunFlags: runFlags, Name: Name, HasDashboards: true, } - RootCmd = cmd.GenRootCmdWithSettings(beater.DefaultCreator(), settings) - RootCmd.AddCommand(cmd.GenModulesCmd(Name, "", BuildModulesManager)) - RootCmd.TestCmd.AddCommand(test.GenTestModulesCmd(Name, "", beater.DefaultTestModulesCreator())) +} + +// Initialize initializes the entrypoint commands for metricbeat +func Initialize(settings instance.Settings) *cmd.BeatsRootCmd { + rootCmd := cmd.GenRootCmdWithSettings(beater.DefaultCreator(), settings) + rootCmd.AddCommand(cmd.GenModulesCmd(Name, "", BuildModulesManager)) + rootCmd.TestCmd.AddCommand(test.GenTestModulesCmd(Name, "", beater.DefaultTestModulesCreator())) + return rootCmd +} + +func init() { + RootCmd = Initialize(MetricbeatSettings()) } diff --git a/metricbeat/main_test.go b/metricbeat/main_test.go index 1c0033a2b2d..445db0353f2 100644 --- a/metricbeat/main_test.go +++ b/metricbeat/main_test.go @@ -44,5 +44,5 @@ func TestSystem(t *testing.T) { } func TestTemplate(t *testing.T) { - template.TestTemplate(t, cmd.Name) + template.TestTemplate(t, cmd.Name, false) } diff --git a/packetbeat/cmd/root.go b/packetbeat/cmd/root.go index 0a22e98a1a0..84c18e70c4d 100644 --- a/packetbeat/cmd/root.go +++ b/packetbeat/cmd/root.go @@ -50,7 +50,8 @@ var withECSVersion = processing.WithFields(common.MapStr{ // RootCmd to handle beats cli var RootCmd *cmd.BeatsRootCmd -func init() { +// PacketbeatSettings contains the default settings for packetbeat +func PacketbeatSettings() instance.Settings { var runFlags = pflag.NewFlagSet(Name, pflag.ExitOnError) runFlags.AddGoFlag(flag.CommandLine.Lookup("I")) runFlags.AddGoFlag(flag.CommandLine.Lookup("t")) @@ -58,12 +59,21 @@ func init() { runFlags.AddGoFlag(flag.CommandLine.Lookup("l")) runFlags.AddGoFlag(flag.CommandLine.Lookup("dump")) - settings := instance.Settings{ + return instance.Settings{ RunFlags: runFlags, Name: Name, HasDashboards: true, Processing: processing.MakeDefaultSupport(true, withECSVersion, processing.WithHost, processing.WithAgentMeta()), } - RootCmd = cmd.GenRootCmdWithSettings(beater.New, settings) - RootCmd.AddCommand(genDevicesCommand()) +} + +// Initialize initializes the entrypoint commands for packetbeat +func Initialize(settings instance.Settings) *cmd.BeatsRootCmd { + rootCmd := cmd.GenRootCmdWithSettings(beater.New, settings) + rootCmd.AddCommand(genDevicesCommand()) + return rootCmd +} + +func init() { + RootCmd = Initialize(PacketbeatSettings()) } diff --git a/packetbeat/main_test.go b/packetbeat/main_test.go index 80c9d76920d..db2244debd3 100644 --- a/packetbeat/main_test.go +++ b/packetbeat/main_test.go @@ -44,5 +44,5 @@ func TestSystem(t *testing.T) { } func TestTemplate(t *testing.T) { - template.TestTemplate(t, cmd.Name) + template.TestTemplate(t, cmd.Name, false) } diff --git a/winlogbeat/cmd/root.go b/winlogbeat/cmd/root.go index 988ac6f9f6c..be30c1886d8 100644 --- a/winlogbeat/cmd/root.go +++ b/winlogbeat/cmd/root.go @@ -47,9 +47,23 @@ var withECSVersion = processing.WithFields(common.MapStr{ }, }) -// RootCmd to handle beats CLI. -var RootCmd = cmd.GenRootCmdWithSettings(beater.New, instance.Settings{ - Name: Name, - HasDashboards: true, - Processing: processing.MakeDefaultSupport(true, withECSVersion, processing.WithAgentMeta()), -}) +// RootCmd to handle beats cli +var RootCmd *cmd.BeatsRootCmd + +// WinlogbeatSettings contains the default settings for winlogbeat +func WinlogbeatSettings() instance.Settings { + return instance.Settings{ + Name: Name, + HasDashboards: true, + Processing: processing.MakeDefaultSupport(true, withECSVersion, processing.WithAgentMeta()), + } +} + +// Initialize initializes the entrypoint commands for packetbeat +func Initialize(settings instance.Settings) *cmd.BeatsRootCmd { + return cmd.GenRootCmdWithSettings(beater.New, settings) +} + +func init() { + RootCmd = Initialize(WinlogbeatSettings()) +} diff --git a/winlogbeat/main_test.go b/winlogbeat/main_test.go index 106da2a768c..c318652b0d5 100644 --- a/winlogbeat/main_test.go +++ b/winlogbeat/main_test.go @@ -44,5 +44,5 @@ func TestSystem(t *testing.T) { } func TestTemplate(t *testing.T) { - template.TestTemplate(t, cmd.Name) + template.TestTemplate(t, cmd.Name, false) } diff --git a/x-pack/auditbeat/cmd/root.go b/x-pack/auditbeat/cmd/root.go index f1e3a2593d6..7e805a92e7d 100644 --- a/x-pack/auditbeat/cmd/root.go +++ b/x-pack/auditbeat/cmd/root.go @@ -5,16 +5,23 @@ package cmd import ( - "github.com/elastic/beats/v7/auditbeat/cmd" + auditbeatcmd "github.com/elastic/beats/v7/auditbeat/cmd" + "github.com/elastic/beats/v7/libbeat/cmd" xpackcmd "github.com/elastic/beats/v7/x-pack/libbeat/cmd" // Register Auditbeat x-pack modules. _ "github.com/elastic/beats/v7/x-pack/auditbeat/include" ) +// Name of the beat +var Name = auditbeatcmd.Name + // RootCmd to handle beats CLI. -var RootCmd = cmd.RootCmd +var RootCmd *cmd.BeatsRootCmd func init() { - xpackcmd.AddXPack(RootCmd, cmd.Name) + settings := auditbeatcmd.AuditbeatSettings() + settings.ElasticLicensed = true + RootCmd = auditbeatcmd.Initialize(settings) + xpackcmd.AddXPack(RootCmd, auditbeatcmd.Name) } diff --git a/x-pack/auditbeat/main_test.go b/x-pack/auditbeat/main_test.go index 7dd24b327fd..c636bc50a04 100644 --- a/x-pack/auditbeat/main_test.go +++ b/x-pack/auditbeat/main_test.go @@ -10,8 +10,8 @@ import ( "flag" "testing" - "github.com/elastic/beats/v7/auditbeat/cmd" "github.com/elastic/beats/v7/libbeat/tests/system/template" + "github.com/elastic/beats/v7/x-pack/auditbeat/cmd" ) var systemTest *bool @@ -32,5 +32,5 @@ func TestSystem(t *testing.T) { } func TestTemplate(t *testing.T) { - template.TestTemplate(t, cmd.Name) + template.TestTemplate(t, cmd.Name, true) } diff --git a/x-pack/filebeat/cmd/root.go b/x-pack/filebeat/cmd/root.go index dc2ebc6fdb6..b95a3cb9e96 100644 --- a/x-pack/filebeat/cmd/root.go +++ b/x-pack/filebeat/cmd/root.go @@ -18,7 +18,9 @@ const Name = fbcmd.Name // Filebeat build the beat root command for executing filebeat and it's subcommands. func Filebeat() *cmd.BeatsRootCmd { - command := fbcmd.Filebeat(inputs.Init) + settings := fbcmd.FilebeatSettings() + settings.ElasticLicensed = true + command := fbcmd.Filebeat(inputs.Init, settings) xpackcmd.AddXPack(command, Name) return command } diff --git a/x-pack/filebeat/main_test.go b/x-pack/filebeat/main_test.go index 6fde6697e3d..6dc51ffe575 100644 --- a/x-pack/filebeat/main_test.go +++ b/x-pack/filebeat/main_test.go @@ -35,5 +35,5 @@ func TestSystem(t *testing.T) { } func TestTemplate(t *testing.T) { - template.TestTemplate(t, fbCommand.Name()) + template.TestTemplate(t, fbCommand.Name(), true) } diff --git a/x-pack/functionbeat/main_test.go b/x-pack/functionbeat/main_test.go index bd62f055ba5..ecb5ac12435 100644 --- a/x-pack/functionbeat/main_test.go +++ b/x-pack/functionbeat/main_test.go @@ -32,5 +32,5 @@ func TestSystem(t *testing.T) { } func TestTemplate(t *testing.T) { - template.TestTemplate(t, cmd.Name) + template.TestTemplate(t, cmd.Name, true) } diff --git a/x-pack/functionbeat/manager/cmd/root.go b/x-pack/functionbeat/manager/cmd/root.go index 98358b40453..ca9e8bda81a 100644 --- a/x-pack/functionbeat/manager/cmd/root.go +++ b/x-pack/functionbeat/manager/cmd/root.go @@ -26,6 +26,7 @@ func init() { Name: Name, HasDashboards: false, ConfigOverrides: config.Overrides, + ElasticLicensed: true, }) RootCmd.RemoveCommand(RootCmd.RunCmd) diff --git a/x-pack/heartbeat/cmd/root.go b/x-pack/heartbeat/cmd/root.go index 8fa75613c0e..8e25993bd87 100644 --- a/x-pack/heartbeat/cmd/root.go +++ b/x-pack/heartbeat/cmd/root.go @@ -5,13 +5,17 @@ package cmd import ( - "github.com/elastic/beats/v7/heartbeat/cmd" + heartbeatCmd "github.com/elastic/beats/v7/heartbeat/cmd" + "github.com/elastic/beats/v7/libbeat/cmd" xpackcmd "github.com/elastic/beats/v7/x-pack/libbeat/cmd" ) // RootCmd to handle beats cli -var RootCmd = cmd.RootCmd +var RootCmd *cmd.BeatsRootCmd func init() { - xpackcmd.AddXPack(RootCmd, cmd.Name) + settings := heartbeatCmd.HeartbeatSettings() + settings.ElasticLicensed = true + RootCmd = heartbeatCmd.Initialize(settings) + xpackcmd.AddXPack(RootCmd, heartbeatCmd.Name) } diff --git a/x-pack/heartbeat/main_test.go b/x-pack/heartbeat/main_test.go index 19d61ffe0f6..e86c240fecc 100644 --- a/x-pack/heartbeat/main_test.go +++ b/x-pack/heartbeat/main_test.go @@ -30,5 +30,5 @@ func TestSystem(t *testing.T) { } func TestTemplate(t *testing.T) { - template.TestTemplate(t, cmd.Name) + template.TestTemplate(t, cmd.Name, true) } diff --git a/x-pack/journalbeat/cmd/root.go b/x-pack/journalbeat/cmd/root.go index 12fed916086..7d27c7e8fef 100644 --- a/x-pack/journalbeat/cmd/root.go +++ b/x-pack/journalbeat/cmd/root.go @@ -5,13 +5,17 @@ package cmd import ( - "github.com/elastic/beats/v7/journalbeat/cmd" + journalbeatCmd "github.com/elastic/beats/v7/journalbeat/cmd" + "github.com/elastic/beats/v7/libbeat/cmd" xpackcmd "github.com/elastic/beats/v7/x-pack/libbeat/cmd" ) // RootCmd to handle beats cli -var RootCmd = cmd.RootCmd +var RootCmd *cmd.BeatsRootCmd func init() { - xpackcmd.AddXPack(RootCmd, cmd.Name) + settings := journalbeatCmd.JournalbeatSettings() + settings.ElasticLicensed = true + RootCmd = journalbeatCmd.Initialize(settings) + xpackcmd.AddXPack(RootCmd, journalbeatCmd.Name) } diff --git a/x-pack/journalbeat/main_test.go b/x-pack/journalbeat/main_test.go index 786c7a57564..fcac188a31c 100644 --- a/x-pack/journalbeat/main_test.go +++ b/x-pack/journalbeat/main_test.go @@ -29,5 +29,5 @@ func TestSystem(t *testing.T) { } func TestTemplate(t *testing.T) { - template.TestTemplate(t, cmd.Name) + template.TestTemplate(t, cmd.Name, true) } diff --git a/x-pack/libbeat/libbeat_test.go b/x-pack/libbeat/libbeat_test.go index 0e7b6854d14..338ebd7e5fb 100644 --- a/x-pack/libbeat/libbeat_test.go +++ b/x-pack/libbeat/libbeat_test.go @@ -29,5 +29,5 @@ func TestSystem(t *testing.T) { } func TestTemplate(t *testing.T) { - template.TestTemplate(t, "mockbeat") + template.TestTemplate(t, "mockbeat", true) } diff --git a/x-pack/metricbeat/cmd/root.go b/x-pack/metricbeat/cmd/root.go index 7979188063d..6cb0977b181 100644 --- a/x-pack/metricbeat/cmd/root.go +++ b/x-pack/metricbeat/cmd/root.go @@ -34,9 +34,10 @@ func init() { var runFlags = pflag.NewFlagSet(Name, pflag.ExitOnError) runFlags.AddGoFlag(flag.CommandLine.Lookup("system.hostfs")) settings := instance.Settings{ - RunFlags: runFlags, - Name: Name, - HasDashboards: true, + RunFlags: runFlags, + Name: Name, + HasDashboards: true, + ElasticLicensed: true, } RootCmd = cmd.GenRootCmdWithSettings(beater.DefaultCreator(), settings) RootCmd.AddCommand(cmd.GenModulesCmd(Name, "", mbcmd.BuildModulesManager)) diff --git a/x-pack/metricbeat/main_test.go b/x-pack/metricbeat/main_test.go index ab3f317d4f5..b092682ccf3 100644 --- a/x-pack/metricbeat/main_test.go +++ b/x-pack/metricbeat/main_test.go @@ -29,5 +29,5 @@ func TestSystem(t *testing.T) { } func TestTemplate(t *testing.T) { - template.TestTemplate(t, cmd.Name) + template.TestTemplate(t, cmd.Name, true) } diff --git a/x-pack/packetbeat/cmd/root.go b/x-pack/packetbeat/cmd/root.go index 1cd9d4a7aa3..e5dd5bb7918 100644 --- a/x-pack/packetbeat/cmd/root.go +++ b/x-pack/packetbeat/cmd/root.go @@ -5,16 +5,20 @@ package cmd import ( - "github.com/elastic/beats/v7/packetbeat/cmd" + "github.com/elastic/beats/v7/libbeat/cmd" + packetbeatCmd "github.com/elastic/beats/v7/packetbeat/cmd" xpackcmd "github.com/elastic/beats/v7/x-pack/libbeat/cmd" ) // Name of this beat. -var Name = cmd.Name +var Name = packetbeatCmd.Name // RootCmd to handle beats cli -var RootCmd = cmd.RootCmd +var RootCmd *cmd.BeatsRootCmd func init() { - xpackcmd.AddXPack(RootCmd, cmd.Name) + settings := packetbeatCmd.PacketbeatSettings() + settings.ElasticLicensed = true + RootCmd = packetbeatCmd.Initialize(settings) + xpackcmd.AddXPack(RootCmd, packetbeatCmd.Name) } diff --git a/x-pack/packetbeat/main_test.go b/x-pack/packetbeat/main_test.go index 5ec2da2d72f..234d68f8169 100644 --- a/x-pack/packetbeat/main_test.go +++ b/x-pack/packetbeat/main_test.go @@ -29,5 +29,5 @@ func TestSystem(t *testing.T) { } func TestTemplate(t *testing.T) { - template.TestTemplate(t, cmd.Name) + template.TestTemplate(t, cmd.Name, true) } diff --git a/x-pack/winlogbeat/cmd/root.go b/x-pack/winlogbeat/cmd/root.go index 684e1660299..bc6cd4b0afd 100644 --- a/x-pack/winlogbeat/cmd/root.go +++ b/x-pack/winlogbeat/cmd/root.go @@ -5,7 +5,8 @@ package cmd import ( - "github.com/elastic/beats/v7/winlogbeat/cmd" + "github.com/elastic/beats/v7/libbeat/cmd" + winlogbeatCmd "github.com/elastic/beats/v7/winlogbeat/cmd" xpackcmd "github.com/elastic/beats/v7/x-pack/libbeat/cmd" // Register fields. @@ -13,11 +14,14 @@ import ( ) // Name of this beat. -var Name = cmd.Name +var Name = winlogbeatCmd.Name // RootCmd to handle beats cli -var RootCmd = cmd.RootCmd +var RootCmd *cmd.BeatsRootCmd func init() { - xpackcmd.AddXPack(RootCmd, cmd.Name) + settings := winlogbeatCmd.WinlogbeatSettings() + settings.ElasticLicensed = true + RootCmd = winlogbeatCmd.Initialize(settings) + xpackcmd.AddXPack(RootCmd, winlogbeatCmd.Name) } diff --git a/x-pack/winlogbeat/main_test.go b/x-pack/winlogbeat/main_test.go index c8c4e35dd34..2e71520f8dc 100644 --- a/x-pack/winlogbeat/main_test.go +++ b/x-pack/winlogbeat/main_test.go @@ -29,5 +29,5 @@ func TestSystem(t *testing.T) { } func TestTemplate(t *testing.T) { - template.TestTemplate(t, cmd.Name) + template.TestTemplate(t, cmd.Name, true) }