Metricbeat system module updates for Windows

- Added system core metricset for Windows. Per core metrics were implemented in elastic/gosigar. - Added logging of process/system details on Windows to aid in debugging (user, arch, cores, sid, privs). - Fixes #2860 (PPID is zero on Windows). - Fixes #1704 (Server 2003 - PID’s not recognized). The command line arguments for the process will not be reported on XP and 2003. - Fixes #1897 (OpenProcess access denied on Windows). Added code to enable the SeDebugPrivilege when it is available. - Fixes #2885 (diskio metricset fails on XP and 2003). - Enabled fsstats by default in Metricbeat config.
elastic · Oct 31, 2016 · eb88c2c · eb88c2c
1 parent 2797565
commit eb88c2c
Show file tree

Hide file tree

Showing 21 changed files with 163 additions and 92 deletions.
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
@@ -33,6 +33,10 @@ https://github.com/elastic/beats/compare/v5.0.0...master[Check the HEAD diff]
 
 - Calculate the fsstat values per mounting point, and not filesystem. {pull}2777[2777]
 - Fix `system.process.start_time` on Windows. {pull}2848[2848]
+- Fix `system.process.ppid` on Windows. {issue}2860[2860]
+- Fix system process metricset for Windows XP and 2003. `cmdline` will be unavailable. {issue}1704[1704]
+- Fix access denied issues in system process metricset by enabling SeDebugPrivilege on Windows. {issue}1897[1897]
+- Fix system diskio metricset for Windows XP and 2003. {issue}2885[2885]
 
 *Packetbeat*
 
@@ -54,6 +58,7 @@ https://github.com/elastic/beats/compare/v5.0.0...master[Check the HEAD diff]
 - Add experimental libbeat metricset in the beats module. {pull}2339[2339]
 - Add experimental docker module. Provided by Ingensi and @douaejeouit based on dockbeat.
 - Add username and password config options to the MongoDB module. {pull}2889[2889]
+- Add system core metricset for Windows. {pull}2883}[2883]
 
 *Packetbeat*
 - Define `client_geoip.location` as geo_point in the mappings to be used by the GeoIP processor in the Ingest Node pipeline.

diff --git a/metricbeat/docs/modules/system.asciidoc b/metricbeat/docs/modules/system.asciidoc
@@ -83,7 +83,7 @@ metricbeat.modules:
     - filesystem
 
     # File system summary stats
-    #- fsstat
+    - fsstat
 
     # Memory stats
     - memory

diff --git a/metricbeat/etc/beat.full.yml b/metricbeat/etc/beat.full.yml
@@ -29,7 +29,7 @@ metricbeat.modules:
     - filesystem
 
     # File system summary stats
-    #- fsstat
+    - fsstat
 
     # Memory stats
     - memory

diff --git a/metricbeat/etc/beat.short.yml b/metricbeat/etc/beat.short.yml
diff --git a/metricbeat/etc/beat.yml b/metricbeat/etc/beat.yml
@@ -29,7 +29,7 @@ metricbeat.modules:
     - filesystem
 
     # File system summary stats
-    #- fsstat
+    - fsstat
 
     # Memory stats
     - memory

diff --git a/metricbeat/make.bat b/metricbeat/make.bat
@@ -0,0 +1,34 @@
+@echo off
+
+REM
+REM Batch script to build and test on Windows. You can use this in conjunction
+REM with the Vagrant machine.
+REM
+
+go install github.com/elastic/beats/vendor/github.com/pierrre/gotestcover
+if %errorlevel% neq 0 exit /b %errorlevel%
+
+echo Building
+go build
+if %errorlevel% neq 0 exit /b %errorlevel%
+
+echo Testing
+mkdir build\coverage
+gotestcover -race -coverprofile=build/coverage/integration.cov github.com/elastic/beats/metricbeat/...
+if %errorlevel% neq 0 exit /b %errorlevel%
+
+echo System Testing
+go test -c -covermode=atomic -coverpkg ./...
+if %errorlevel% neq 0 exit /b %errorlevel%
+nosetests -v -w tests\system --process-timeout=30
+if %errorlevel% neq 0 exit /b %errorlevel%
+
+echo Aggregating Coverage Reports
+python ..\dev-tools\aggregate_coverage.py -o build\coverage\system.cov .\build\system-tests\run
+if %errorlevel% neq 0 exit /b %errorlevel%
+python ..\dev-tools\aggregate_coverage.py -o build\coverage\full.cov .\build\coverage
+if %errorlevel% neq 0 exit /b %errorlevel%
+go tool cover -html=build\coverage\full.cov -o build\coverage\full.html
+if %errorlevel% neq 0 exit /b %errorlevel%
+
+echo Success
diff --git a/metricbeat/metricbeat.full.yml b/metricbeat/metricbeat.full.yml
@@ -29,7 +29,7 @@ metricbeat.modules:
     - filesystem
 
     # File system summary stats
-    #- fsstat
+    - fsstat
 
     # Memory stats
     - memory

diff --git a/metricbeat/metricbeat.yml b/metricbeat/metricbeat.yml
@@ -29,7 +29,7 @@ metricbeat.modules:
     - filesystem
 
     # File system summary stats
-    #- fsstat
+    - fsstat
 
     # Memory stats
     - memory

diff --git a/metricbeat/module/system/_meta/config.full.yml b/metricbeat/module/system/_meta/config.full.yml
@@ -16,7 +16,7 @@
     - filesystem
 
     # File system summary stats
-    #- fsstat
+    - fsstat
 
     # Memory stats
     - memory

diff --git a/metricbeat/module/system/_meta/config.yml b/metricbeat/module/system/_meta/config.yml
@@ -16,7 +16,7 @@
     - filesystem
 
     # File system summary stats
-    #- fsstat
+    - fsstat
 
     # Memory stats
     - memory

diff --git a/metricbeat/module/system/core/_meta/docs.asciidoc b/metricbeat/module/system/core/_meta/docs.asciidoc
@@ -8,3 +8,4 @@ This metricset is available on:
 - FreeBSD
 - Linux
 - OpenBSD
+- Windows
diff --git a/metricbeat/module/system/core/core.go b/metricbeat/module/system/core/core.go
@@ -1,4 +1,4 @@
-// +build darwin freebsd linux openbsd
+// +build darwin freebsd linux openbsd windows
 
 package core
 
@@ -24,7 +24,6 @@ type MetricSet struct {
 
 // New is a mb.MetricSetFactory that returns a cores.MetricSet.
 func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
-
 	config := struct {
 		CpuTicks bool `config:"cpu_ticks"` // export CPU usage in ticks
 	}{
@@ -46,16 +45,14 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
 
 // Fetch fetches CPU core metrics from the OS.
 func (m *MetricSet) Fetch() ([]common.MapStr, error) {
-
 	cpuCoreStat, err := cpu.GetCpuTimesList()
 	if err != nil {
 		return nil, errors.Wrap(err, "cpu core times")
 	}
 
 	m.cpu.AddCpuPercentageList(cpuCoreStat)
 
-	cores := []common.MapStr{}
-
+	cores := make([]common.MapStr, 0, len(cpuCoreStat))
 	for core, stat := range cpuCoreStat {
 
 		coreStat := common.MapStr{

diff --git a/metricbeat/module/system/core/core_test.go b/metricbeat/module/system/core/core_test.go
@@ -1,4 +1,4 @@
-// +build darwin freebsd linux openbsd
+// +build darwin freebsd linux openbsd windows
 
 package core
 

diff --git a/metricbeat/module/system/filesystem/helper.go b/metricbeat/module/system/filesystem/helper.go
@@ -31,10 +31,8 @@ func GetFileSystemList() ([]sigar.FileSystem, error) {
 }
 
 func GetFileSystemStat(fs sigar.FileSystem) (*FileSystemStat, error) {
-
 	stat := sigar.FileSystemUsage{}
-	err := stat.Get(fs.DirName)
-	if err != nil {
+	if err := stat.Get(fs.DirName); err != nil {
 		return nil, err
 	}
 

diff --git a/metricbeat/module/system/filesystem/helper_test.go b/metricbeat/module/system/filesystem/helper_test.go
@@ -12,24 +12,27 @@ import (
 )
 
 func TestFileSystemList(t *testing.T) {
-
 	if runtime.GOOS == "darwin" && os.Getenv("TRAVIS") == "true" {
 		t.Skip("FileSystem test fails on Travis/OSX with i/o error")
 	}
 
 	fss, err := GetFileSystemList()
-
-	assert.Nil(t, err)
+	if err != nil {
+		t.Fatal("GetFileSystemList", err)
+	}
 	assert.True(t, (len(fss) > 0))
 
 	for _, fs := range fss {
+		if fs.TypeName == "cdrom" {
+			continue
+		}
 
 		stat, err := GetFileSystemStat(fs)
-		assert.NoError(t, err)
-
-		assert.True(t, (stat.Total >= 0))
-		assert.True(t, (stat.Free >= 0))
-		assert.True(t, (stat.Avail >= 0))
-		assert.True(t, (stat.Used >= 0))
+		if assert.NoError(t, err, "%v", err) {
+			assert.True(t, (stat.Total >= 0))
+			assert.True(t, (stat.Free >= 0))
+			assert.True(t, (stat.Avail >= 0))
+			assert.True(t, (stat.Used >= 0))
+		}
 	}
 }
diff --git a/metricbeat/module/system/process/helper.go b/metricbeat/module/system/process/helper.go
@@ -77,7 +77,7 @@ func (proc *Process) getDetails(cmdline string) error {
 
 	if cmdline == "" {
 		args := sigar.ProcArgs{}
-		if err := args.Get(proc.Pid); err != nil {
+		if err := args.Get(proc.Pid); err != nil && !sigar.IsNotImplemented(err) {
 			return fmt.Errorf("error getting process arguments for pid=%d: %v", proc.Pid, err)
 		}
 		proc.CmdLine = strings.Join(args.List, " ")
@@ -330,14 +330,12 @@ func (procStats *ProcStats) GetProcStats() ([]common.MapStr, error) {
 }
 
 func (procStats *ProcStats) GetProcStatsEvents() ([]common.MapStr, error) {
-
-	events := []common.MapStr{}
-
 	processes, err := procStats.GetProcStats()
 	if err != nil {
 		return nil, err
 	}
 
+	events := make([]common.MapStr, len(processes))
 	for _, proc := range processes {
 		event := common.MapStr{
 			"@timestamp": common.Time(time.Now()),

diff --git a/metricbeat/module/system/system.go b/metricbeat/module/system/system.go
@@ -29,7 +29,7 @@ type Module struct {
 func NewModule(base mb.BaseModule) (mb.Module, error) {
 	// This only needs to be configured once for all system modules.
 	once.Do(func() {
-		configureHostFS()
+		initModule()
 	})
 
 	return &Module{BaseModule: base, HostFS: *HostFS}, nil

diff --git a/metricbeat/module/system/system_linux.go b/metricbeat/module/system/system_linux.go
@@ -7,6 +7,10 @@ import (
 	"github.com/elastic/gosigar"
 )
 
+func initModule() {
+	configureHostFS()
+}
+
 func configureHostFS() {
 	dir := *HostFS
 	if dir == "" {

diff --git a/metricbeat/module/system/system_other.go b/metricbeat/module/system/system_other.go
@@ -1,7 +1,7 @@
-// +build !linux
+// +build !linux,!windows
 
 package system
 
-func configureHostFS() {
+func initModule() {
 	// Stub method for non-linux.
 }