Subcommands (elastic#4420)

This change introduces subcommand interface to all beats.

This give us several new features:

 * Isolated subcommands (version, setup, run...)
 * Command line help for all of them
 * Autocompletion and typo hinting
 * Freedom to add beat specific subcommands where needed

Usage examples:

```
$ metricbeat help
$ metricbeat version
$ metricbeat run -h
$ metricbeat run -e -v
```

I've kept root command as an alias for `run`, so old style calls should be still working.

About flags:

Cobra supports persistent (global) and local flags for each command.
I've added config related flags as persistent, common to all
subcommands. While keeping runtime related flags under `run`, see the
full list here:

```
Flags:
  -N, --N                   Disable actual publishing for testing
      --configtest          Test configuration and exit.
      --cpuprofile string   Write cpu profile to file
  -e, --e                   Log to stderr and disable syslog/file output
  -h, --help                help for run
      --httpprof string     Start pprof http server
      --memprofile string   Write memory profile to this file
      --setup               Load the sample Kibana dashboards
  -v, --v                   Log at INFO level
      --version             Print the version and exit

Global Flags:
  -E, --E Flag                      Configuration overwrite (default null)
  -c, --c argList                   Configuration file, relative to path.config (default beat.yml)
  -d, --d string                    Enable certain debug selectors
      --path.config flagOverwrite   Configuration path
      --path.data flagOverwrite     Data path
      --path.home flagOverwrite     Home path
      --path.logs flagOverwrite     Logs path
      --strict.perms                Strict permission checking on config files (default true)
```

In the log term we should aim to deprecate a few of them (setup, version, configtest...) in
favor of subcommands. I'm keeping them for now to maintain backwards compatibilty with 5.X.

CHANGELOG.asciidoc

@@ -46,6 +46,8 @@ https://github.com/elastic/beats/compare/v6.0.0-alpha1...master[Check the HEAD d
 
 *Affecting all Beats*
 
+- New cli subcommands interface. {pull}4420[4420]
+
 *Filebeat*
 
 - Add experimental Redis module. {pull}4441[4441]

NOTICE

@@ -1357,6 +1357,15 @@ Exhibit B - "Incompatible With Secondary Licenses" Notice
       the Mozilla Public License, v. 2.0.
 
 
+--------------------------------------------------------------------
+Dependency: github.com/inconshreveable/mousetrap
+Revision: 76626ae9c91c4f2a10f34cad8ce83ea42c93bb75
+License type (autodetected): Apache License 2.0
+./vendor/github.com/inconshreveable/mousetrap/LICENSE:
+--------------------------------------------------------------------
+Apache License 2.0
+
+
 --------------------------------------------------------------------
 Dependency: github.com/joeshaw/multierror
 Revision: 69b34d4ec901851247ae7e77d33909caf9df99ed
@@ -2867,6 +2876,50 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE.
 
+--------------------------------------------------------------------
+Dependency: github.com/spf13/cobra
+Revision: e606913c4ee45fec232e67e70105fb6c866b95d9
+License type (autodetected): Apache License 2.0
+./vendor/github.com/spf13/cobra/LICENSE.txt:
+--------------------------------------------------------------------
+Apache License 2.0
+
+
+--------------------------------------------------------------------
+Dependency: github.com/spf13/pflag
+Revision: e57e3eeb33f795204c1ca35f56c44f83227c6e66
+License type (autodetected): BSD 3-clause license
+./vendor/github.com/spf13/pflag/LICENSE:
+--------------------------------------------------------------------
+Copyright (c) 2012 Alex Ogier. All rights reserved.
+Copyright (c) 2012 The Go Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+   * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
 --------------------------------------------------------------------
 Dependency: github.com/StackExchange/wmi
 Revision: 9f32b5905fd6ce7384093f9d048437e79f7b4d85

docs/devguide/newbeat.asciidoc

@@ -474,15 +474,17 @@ package main
 
 import (
 	"os"
+	"github.com/spf13/cobra"
 
 	"github.com/elastic/beats/libbeat/beat"
 
 	"github.com/kimjmin/countbeat/beater"
 )
 
+var RootCmd = cmd.GenRootCmd("countbeat", "", beater.New)
+
 func main() {
-	err := beat.Run("countbeat", "", beater.New)
-	if err != nil {
+	if err := RootCmd.Execute(); err != nil {
 		os.Exit(1)
 	}
 }

filebeat/cmd/root.go

@@ -0,0 +1,27 @@
+package cmd
+
+import (
+	"flag"
+
+	"github.com/spf13/pflag"
+
+	"github.com/elastic/beats/filebeat/beater"
+
+	cmd "github.com/elastic/beats/libbeat/cmd"
+)
+
+// Name of this beat
+var Name = "filebeat"
+
+// RootCmd to handle beats cli
+var RootCmd *cmd.BeatsRootCmd
+
+func init() {
+	var runFlags = pflag.NewFlagSet(Name, pflag.ExitOnError)
+	runFlags.AddGoFlag(flag.CommandLine.Lookup("once"))
+	runFlags.AddGoFlag(flag.CommandLine.Lookup("modules"))
+
+	RootCmd = cmd.GenRootCmdWithRunFlags(Name, "", beater.New, runFlags)
+	RootCmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("M"))
+	RootCmd.ConfigTest.Flags().AddGoFlag(flag.CommandLine.Lookup("modules"))
+}

filebeat/main.go

@@ -3,12 +3,9 @@ package main
 import (
 	"os"
 
-	"github.com/elastic/beats/filebeat/beater"
-	"github.com/elastic/beats/libbeat/beat"
+	"github.com/elastic/beats/filebeat/cmd"
 )
 
-var Name = "filebeat"
-
 // The basic model of execution:
 // - prospector: finds files in paths/globs to harvest, starts harvesters
 // - harvester: reads a file, sends events to the spooler
@@ -17,9 +14,8 @@ var Name = "filebeat"
 // - registrar: records positions of files read
 // Finally, prospector uses the registrar information, on restart, to
 // determine where in each file to restart a harvester.
-
 func main() {
-	if err := beat.Run(Name, "", beater.New); err != nil {
+	if err := cmd.RootCmd.Execute(); err != nil {
 		os.Exit(1)
 	}
 }

filebeat/main_test.go

@@ -5,12 +5,16 @@ package main
 import (
 	"flag"
 	"testing"
+
+	"github.com/elastic/beats/filebeat/cmd"
 )
 
 var systemTest *bool
 
 func init() {
 	systemTest = flag.Bool("systemTest", false, "Set to true when running system tests")
+	cmd.RootCmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("systemTest"))
+	cmd.RootCmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("test.coverprofile"))
 }
 
 // Test started when the test binary is started. Only calls main.

generator/beat/{beat}/cmd/root.go.tmpl

@@ -0,0 +1,13 @@
+package cmd
+
+import (
+	"github.com/elastic/beats/metricbeat/beater"
+
+	cmd "github.com/elastic/beats/libbeat/cmd"
+)
+
+// Name of this beat
+var Name = "{beat}"
+
+// RootCmd to handle beats cli
+var RootCmd = cmd.GenRootCmd(Name, "", beater.New)

generator/beat/{beat}/main.go.tmpl

@@ -3,14 +3,11 @@ package main
 import (
 	"os"
 
-	"github.com/elastic/beats/libbeat/beat"
-
-	"{beat_path}/beater"
+	"{beat_path}/cmd"
 )
 
 func main() {
-	err := beat.Run("{beat}", "", beater.New)
-	if err != nil {
+	if err := cmd.RootCmd.Execute(); err != nil {
 		os.Exit(1)
 	}
 }

generator/beat/{beat}/main_test.go.tmpl

@@ -5,12 +5,17 @@ package main
 import (
 	"flag"
 	"testing"
+
+	"{beat_path}/cmd"
 )
 
 var systemTest *bool
 
 func init() {
 	systemTest = flag.Bool("systemTest", false, "Set to true when running system tests")
+
+	cmd.RootCmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("systemTest"))
+	cmd.RootCmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("test.coverprofile"))
 }
 
 // Test started when the test binary is started. Only calls main.

heartbeat/cmd/root.go

@@ -0,0 +1,10 @@
+package cmd
+
+import cmd "github.com/elastic/beats/libbeat/cmd"
+import "github.com/elastic/beats/heartbeat/beater"
+
+// Name of this beat
+var Name = "heartbeat"
+
+// RootCmd to handle beats cli
+var RootCmd = cmd.GenRootCmd(Name, "", beater.New)

heartbeat/main.go

@@ -3,17 +3,14 @@ package main
 import (
 	"os"
 
-	"github.com/elastic/beats/libbeat/beat"
-
-	"github.com/elastic/beats/heartbeat/beater"
+	"github.com/elastic/beats/heartbeat/cmd"
 
 	// register default heartbeat monitors
 	_ "github.com/elastic/beats/heartbeat/monitors/defaults"
 )
 
 func main() {
-	err := beat.Run("heartbeat", "", beater.New)
-	if err != nil {
+	if err := cmd.RootCmd.Execute(); err != nil {
 		os.Exit(1)
 	}
 }

heartbeat/main_test.go

@@ -5,12 +5,16 @@ package main
 import (
 	"flag"
 	"testing"
+
+	"github.com/elastic/beats/heartbeat/cmd"
 )
 
 var systemTest *bool
 
 func init() {
 	systemTest = flag.Bool("systemTest", false, "Set to true when running system tests")
+	cmd.RootCmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("systemTest"))
+	cmd.RootCmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("test.coverprofile"))
 }
 
 // Test started when the test binary is started. Only calls main.