Metricbeat system module updates for Windows

- Added system core metricset for Windows. Per core metrics were implemented
  in elastic/gosigar.
- Added logging of process/system details on Windows to aid in debugging (user,
  arch, cores, sid, privs).
- Fixes #2860 (PPID is zero on Windows).
- Fixes #1704 (Server 2003 - PID’s not recognized). The command line arguments
  for the process will not be reported on XP and 2003.
- Fixes #1897 (OpenProcess access denied on Windows). Added code to enable the
  SeDebugPrivilege when it is available.
- Fixes #2885 (diskio metricset fails on XP and 2003).
- Enabled fsstats by default in Metricbeat config.

(cherry picked from commit eb88c2c)

CHANGELOG.asciidoc

@@ -32,6 +32,10 @@ https://github.com/elastic/beats/compare/v5.0.0...5.0[Check the HEAD diff]
 *Metricbeat*
 
 - Fix `system.process.start_time` on Windows. {pull}2848[2848]
+- Fix `system.process.ppid` on Windows. {issue}2860[2860]
+- Fix system process metricset for Windows XP and 2003. `cmdline` will be unavailable. {issue}1704[1704]
+- Fix access denied issues in system process metricset by enabling SeDebugPrivilege on Windows. {issue}1897[1897]
+- Fix system diskio metricset for Windows XP and 2003. {issue}2885[2885]
 
 *Packetbeat*
 
@@ -49,6 +53,11 @@ https://github.com/elastic/beats/compare/v5.0.0...5.0[Check the HEAD diff]
 *Metricbeat*
 
 - Add username and password config options to the PostgreSQL module. {pull}2889[2890]
+- Add experimental filebeat metricset in the beats module. {pull}2297[2297]
+- Add experimental libbeat metricset in the beats module. {pull}2339[2339]
+- Add experimental docker module. Provided by Ingensi and @douaejeouit based on dockbeat.
+- Add username and password config options to the MongoDB module. {pull}2889[2889]
+- Add system core metricset for Windows. {pull}2883}[2883]
 
 *Packetbeat*
 

metricbeat/docs/modules/system.asciidoc

@@ -83,7 +83,7 @@ metricbeat.modules:
     - filesystem
 
     # File system summary stats
-    #- fsstat
+    - fsstat
 
     # Memory stats
     - memory

metricbeat/etc/beat.full.yml

@@ -29,7 +29,7 @@ metricbeat.modules:
     - filesystem
 
     # File system summary stats
-    #- fsstat
+    - fsstat
 
     # Memory stats
     - memory

metricbeat/etc/beat.yml

@@ -29,7 +29,7 @@ metricbeat.modules:
     - filesystem
 
     # File system summary stats
-    #- fsstat
+    - fsstat
 
     # Memory stats
     - memory

metricbeat/make.bat

@@ -0,0 +1,34 @@
+@echo off
+
+REM
+REM Batch script to build and test on Windows. You can use this in conjunction
+REM with the Vagrant machine.
+REM
+
+go install github.com/elastic/beats/vendor/github.com/pierrre/gotestcover
+if %errorlevel% neq 0 exit /b %errorlevel%
+
+echo Building
+go build
+if %errorlevel% neq 0 exit /b %errorlevel%
+
+echo Testing
+mkdir build\coverage
+gotestcover -race -coverprofile=build/coverage/integration.cov github.com/elastic/beats/metricbeat/...
+if %errorlevel% neq 0 exit /b %errorlevel%
+
+echo System Testing
+go test -c -covermode=atomic -coverpkg ./...
+if %errorlevel% neq 0 exit /b %errorlevel%
+nosetests -v -w tests\system --process-timeout=30
+if %errorlevel% neq 0 exit /b %errorlevel%
+
+echo Aggregating Coverage Reports
+python ..\dev-tools\aggregate_coverage.py -o build\coverage\system.cov .\build\system-tests\run
+if %errorlevel% neq 0 exit /b %errorlevel%
+python ..\dev-tools\aggregate_coverage.py -o build\coverage\full.cov .\build\coverage
+if %errorlevel% neq 0 exit /b %errorlevel%
+go tool cover -html=build\coverage\full.cov -o build\coverage\full.html
+if %errorlevel% neq 0 exit /b %errorlevel%
+
+echo Success

metricbeat/metricbeat.full.yml

@@ -29,7 +29,7 @@ metricbeat.modules:
     - filesystem
 
     # File system summary stats
-    #- fsstat
+    - fsstat
 
     # Memory stats
     - memory

metricbeat/metricbeat.yml

@@ -29,7 +29,7 @@ metricbeat.modules:
     - filesystem
 
     # File system summary stats
-    #- fsstat
+    - fsstat
 
     # Memory stats
     - memory

metricbeat/module/system/_meta/config.full.yml

@@ -16,7 +16,7 @@
     - filesystem
 
     # File system summary stats
-    #- fsstat
+    - fsstat
 
     # Memory stats
     - memory

metricbeat/module/system/_meta/config.yml

@@ -16,7 +16,7 @@
     - filesystem
 
     # File system summary stats
-    #- fsstat
+    - fsstat
 
     # Memory stats
     - memory

metricbeat/module/system/core/_meta/docs.asciidoc

@@ -8,3 +8,4 @@ This metricset is available on:
 - FreeBSD
 - Linux
 - OpenBSD
+- Windows

metricbeat/module/system/core/core.go

@@ -1,4 +1,4 @@
-// +build darwin freebsd linux openbsd
+// +build darwin freebsd linux openbsd windows
 
 package core
 
@@ -24,7 +24,6 @@ type MetricSet struct {
 
 // New is a mb.MetricSetFactory that returns a cores.MetricSet.
 func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
-
 	config := struct {
 		CpuTicks bool `config:"cpu_ticks"` // export CPU usage in ticks
 	}{
@@ -46,16 +45,14 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
 
 // Fetch fetches CPU core metrics from the OS.
 func (m *MetricSet) Fetch() ([]common.MapStr, error) {
-
 	cpuCoreStat, err := cpu.GetCpuTimesList()
 	if err != nil {
 		return nil, errors.Wrap(err, "cpu core times")
 	}
 
 	m.cpu.AddCpuPercentageList(cpuCoreStat)
 
-	cores := []common.MapStr{}
-
+	cores := make([]common.MapStr, 0, len(cpuCoreStat))
 	for core, stat := range cpuCoreStat {
 
 		coreStat := common.MapStr{

metricbeat/module/system/core/core_test.go

@@ -1,4 +1,4 @@
-// +build darwin freebsd linux openbsd
+// +build darwin freebsd linux openbsd windows
 
 package core
 

metricbeat/module/system/filesystem/helper.go

@@ -31,10 +31,8 @@ func GetFileSystemList() ([]sigar.FileSystem, error) {
 }
 
 func GetFileSystemStat(fs sigar.FileSystem) (*FileSystemStat, error) {
-
 	stat := sigar.FileSystemUsage{}
-	err := stat.Get(fs.DirName)
-	if err != nil {
+	if err := stat.Get(fs.DirName); err != nil {
 		return nil, err
 	}
 

metricbeat/module/system/filesystem/helper_test.go

@@ -12,24 +12,27 @@ import (
 )
 
 func TestFileSystemList(t *testing.T) {
-
 	if runtime.GOOS == "darwin" && os.Getenv("TRAVIS") == "true" {
 		t.Skip("FileSystem test fails on Travis/OSX with i/o error")
 	}
 
 	fss, err := GetFileSystemList()
-
-	assert.Nil(t, err)
+	if err != nil {
+		t.Fatal("GetFileSystemList", err)
+	}
 	assert.True(t, (len(fss) > 0))
 
 	for _, fs := range fss {
+		if fs.TypeName == "cdrom" {
+			continue
+		}
 
 		stat, err := GetFileSystemStat(fs)
-		assert.NoError(t, err)
-
-		assert.True(t, (stat.Total >= 0))
-		assert.True(t, (stat.Free >= 0))
-		assert.True(t, (stat.Avail >= 0))
-		assert.True(t, (stat.Used >= 0))
+		if assert.NoError(t, err, "%v", err) {
+			assert.True(t, (stat.Total >= 0))
+			assert.True(t, (stat.Free >= 0))
+			assert.True(t, (stat.Avail >= 0))
+			assert.True(t, (stat.Used >= 0))
+		}
 	}
 }

metricbeat/module/system/process/helper.go

@@ -77,7 +77,7 @@ func (proc *Process) getDetails(cmdline string) error {
 
 	if cmdline == "" {
 		args := sigar.ProcArgs{}
-		if err := args.Get(proc.Pid); err != nil {
+		if err := args.Get(proc.Pid); err != nil && !sigar.IsNotImplemented(err) {
 			return fmt.Errorf("error getting process arguments for pid=%d: %v", proc.Pid, err)
 		}
 		proc.CmdLine = strings.Join(args.List, " ")
@@ -330,14 +330,12 @@ func (procStats *ProcStats) GetProcStats() ([]common.MapStr, error) {
 }
 
 func (procStats *ProcStats) GetProcStatsEvents() ([]common.MapStr, error) {
-
-	events := []common.MapStr{}
-
 	processes, err := procStats.GetProcStats()
 	if err != nil {
 		return nil, err
 	}
 
+	events := make([]common.MapStr, len(processes))
 	for _, proc := range processes {
 		event := common.MapStr{
 			"@timestamp": common.Time(time.Now()),

metricbeat/module/system/system.go

@@ -29,7 +29,7 @@ type Module struct {
 func NewModule(base mb.BaseModule) (mb.Module, error) {
 	// This only needs to be configured once for all system modules.
 	once.Do(func() {
-		configureHostFS()
+		initModule()
 	})
 
 	return &Module{BaseModule: base, HostFS: *HostFS}, nil

metricbeat/module/system/system_linux.go

@@ -7,6 +7,10 @@ import (
 	"github.com/elastic/gosigar"
 )
 
+func initModule() {
+	configureHostFS()
+}
+
 func configureHostFS() {
 	dir := *HostFS
 	if dir == "" {

metricbeat/module/system/system_other.go

@@ -1,7 +1,7 @@
-// +build !linux
+// +build !linux,!windows
 
 package system
 
-func configureHostFS() {
+func initModule() {
 	// Stub method for non-linux.
 }