From b47ab14c925a7c0d8f095568310941aecc9d6e07 Mon Sep 17 00:00:00 2001 From: Monica Sarbu Date: Wed, 27 Jul 2016 00:19:47 +0200 Subject: [PATCH] Add the load metricset and change the name of a few exported fields: - system.memory.actual.free is replaced with system.memory.available - system.memory.actual.used is not needed as it can calculated from system.memory.total - [ ] system.memory.available - system.filesystem.avail is replaced with system.filesystem.available - add system.load.norm.* --- CHANGELOG.asciidoc | 4 + metricbeat/docs/fields.asciidoc | 107 ++++++++-------- metricbeat/docs/modules/system.asciidoc | 7 + metricbeat/docs/modules/system/load.asciidoc | 19 +++ metricbeat/etc/beat.full.yml | 3 + metricbeat/etc/beat.yml | 3 + metricbeat/etc/fields.yml | 78 ++++++----- metricbeat/include/list.go | 1 + metricbeat/metricbeat.full.yml | 3 + metricbeat/metricbeat.template-es2x.json | 53 ++++---- metricbeat/metricbeat.template.json | 53 ++++---- metricbeat/metricbeat.yml | 3 + metricbeat/module/doc.go | 1 + metricbeat/module/system/_meta/config.yml | 3 + metricbeat/module/system/cpu/_meta/data.json | 94 ++++++-------- metricbeat/module/system/cpu/_meta/fields.yml | 18 --- metricbeat/module/system/cpu/cpu.go | 11 -- metricbeat/module/system/cpu/helper.go | 121 ------------------ metricbeat/module/system/cpu/helper_test.go | 17 --- .../module/system/filesystem/_meta/data.json | 56 ++++---- .../module/system/filesystem/_meta/fields.yml | 2 +- metricbeat/module/system/filesystem/helper.go | 2 +- metricbeat/module/system/load/_meta/data.json | 25 ++++ .../module/system/load/_meta/docs.asciidoc | 10 ++ .../module/system/load/_meta/fields.yml | 32 +++++ metricbeat/module/system/load/doc.go | 4 + metricbeat/module/system/load/helper.go | 39 ++++++ metricbeat/module/system/load/helper_test.go | 22 ++++ metricbeat/module/system/load/load.go | 68 ++++++++++ .../module/system/memory/_meta/data.json | 67 +++++----- .../module/system/memory/_meta/fields.yml | 26 +--- metricbeat/module/system/memory/memory.go | 10 +- metricbeat/tests/system/test_system.py | 39 +++++- 33 files changed, 530 insertions(+), 471 deletions(-) create mode 100644 metricbeat/docs/modules/system/load.asciidoc create mode 100644 metricbeat/module/system/load/_meta/data.json create mode 100644 metricbeat/module/system/load/_meta/docs.asciidoc create mode 100644 metricbeat/module/system/load/_meta/fields.yml create mode 100644 metricbeat/module/system/load/doc.go create mode 100644 metricbeat/module/system/load/helper.go create mode 100644 metricbeat/module/system/load/helper_test.go create mode 100644 metricbeat/module/system/load/load.go diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index d8ad45b8521..7f285ea6b92 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -21,6 +21,10 @@ https://github.com/elastic/beats/compare/v5.0.0-alpha4...master[Check the HEAD d - The undocumented file output `index` setting was removed. Use `filename` instead. {pull}2077[2077] *Metricbeat* +- Create a separate metricSet for load under the system module and remove load information from CPU stats. {pull}2101[2101] +- Rename `system.memory.actual.free` with `system.memory.available` and remove `system.memory.actual.used`. + {pull}2101[2101] +- Add `system.load.norm.1`, `system.load.norm.5` and `system.load.norm.15`. {pull}2101[2101] *Packetbeat* - Set `enabled` ` in `packetbeat.protocols.icmp` configuration to `true` by default. {pull}1988[1988] diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 73afcb1556f..bf09397e13d 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -2159,37 +2159,6 @@ type: long The amount of CPU time spent in involuntary wait by the virtual CPU while the hypervisor was servicing another processor. Available only on Unix. -[float] -== load Fields - -Load averages. - - - -[float] -=== system.cpu.load.1 - -type: half_float - -Load average for the last minute. - - -[float] -=== system.cpu.load.5 - -type: half_float - -Load average for the last 5 minutes. - - -[float] -=== system.cpu.load.15 - -type: half_float - -Load average for the last 15 minutes. - - [float] == diskio Fields @@ -2279,7 +2248,7 @@ The total number of of milliseconds spent doing I/Os. [float] -=== system.filesystem.avail +=== system.filesystem.available type: long @@ -2402,73 +2371,105 @@ Total space (used plus free). [float] -== memory Fields +== load Fields -`memory` contains local memory stats. +Load averages. [float] -=== system.memory.total +=== system.load.1 -type: long +type: half_float -Total memory. +Load average for the last minute. [float] -=== system.memory.used.bytes +=== system.load.5 -type: long +type: half_float -Used memory. +Load average for the last 5 minutes. [float] -=== system.memory.free +=== system.load.15 -type: long +type: half_float -Available memory. +Load average for the last 15 minutes. [float] -=== system.memory.used.pct +=== system.load.norm.1 type: half_float -The percentage of used memory. +Load divided by the number of cores for the last minute. + + +[float] +=== system.load.norm.5 + +type: half_float + +Load divided by the number of cores for the last 5 minutes. + + +[float] +=== system.load.norm.15 + +type: half_float + +Load divided by the number of cores for the last 15 minutes. + + +[float] +== memory Fields + +`memory` contains local memory stats. + [float] -== actual Fields +=== system.memory.total -Actual memory fields. +type: long +Total memory. [float] -=== system.memory.actual.used.bytes +=== system.memory.used.bytes type: long -Actual used memory. This value is the "used" memory minus the memory used for disk caches and buffers. Available only on Unix. +Used memory. [float] -=== system.memory.actual.free +=== system.memory.free type: long -Actual available memory. This value is the "free" memory plus the memory used for disk caches and buffers. Available only on Unix. +The total amount of free memory in bytes. This value does not include memory consumed by system caches and buffers (see system.memory.available). [float] -=== system.memory.actual.used.pct +=== system.memory.used.pct type: half_float -The percentage of actual used memory. +The percentage of used memory. + + +[float] +=== system.memory.available + +type: long + +Available memory in bytes. It is calculated based on the OS. On Linux it consists of the free memory plus caches and buffers. On OSX it is a sum of free memory and the inactive memory. [float] diff --git a/metricbeat/docs/modules/system.asciidoc b/metricbeat/docs/modules/system.asciidoc index 82966cb63f9..301f5d75da9 100644 --- a/metricbeat/docs/modules/system.asciidoc +++ b/metricbeat/docs/modules/system.asciidoc @@ -58,6 +58,9 @@ metricbeat.modules: # CPU stats - cpu + # System Load stats + - load + # Per CPU core stats #- core @@ -101,6 +104,8 @@ The following metricsets are available: * <> +* <> + * <> * <> @@ -117,6 +122,8 @@ include::system/filesystem.asciidoc[] include::system/fsstat.asciidoc[] +include::system/load.asciidoc[] + include::system/memory.asciidoc[] include::system/network.asciidoc[] diff --git a/metricbeat/docs/modules/system/load.asciidoc b/metricbeat/docs/modules/system/load.asciidoc new file mode 100644 index 00000000000..14c0711a6f5 --- /dev/null +++ b/metricbeat/docs/modules/system/load.asciidoc @@ -0,0 +1,19 @@ +//// +This file is generated! See scripts/docs_collector.py +//// + +[[metricbeat-metricset-system-load]] +include::../../../module/system/load/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/system/load/_meta/data.json[] +---- diff --git a/metricbeat/etc/beat.full.yml b/metricbeat/etc/beat.full.yml index 83d8124b705..48a3629072d 100644 --- a/metricbeat/etc/beat.full.yml +++ b/metricbeat/etc/beat.full.yml @@ -16,6 +16,9 @@ metricbeat.modules: # CPU stats - cpu + # System Load stats + - load + # Per CPU core stats #- core diff --git a/metricbeat/etc/beat.yml b/metricbeat/etc/beat.yml index 15421af9fe4..009f5c4477d 100644 --- a/metricbeat/etc/beat.yml +++ b/metricbeat/etc/beat.yml @@ -16,6 +16,9 @@ metricbeat.modules: # CPU stats - cpu + # System Load stats + - load + # Per CPU core stats #- core diff --git a/metricbeat/etc/fields.yml b/metricbeat/etc/fields.yml index 257290dd5db..2fcb3f380bf 100644 --- a/metricbeat/etc/fields.yml +++ b/metricbeat/etc/fields.yml @@ -1289,24 +1289,6 @@ The amount of CPU time spent in involuntary wait by the virtual CPU while the hypervisor was servicing another processor. Available only on Unix. - - - name: load - type: group - description: > - Load averages. - fields: - - name: "1" - type: half_float - description: > - Load average for the last minute. - - name: "5" - type: half_float - description: > - Load average for the last 5 minutes. - - name: "15" - type: half_float - description: > - Load average for the last 15 minutes. - name: diskio type: group description: > @@ -1366,7 +1348,7 @@ description: > `filesystem` contains local filesystem stats. fields: - - name: avail + - name: available type: long description: > The disk space available to an unprivileged user in bytes. @@ -1432,6 +1414,38 @@ type: long description: > Total space (used plus free). + - name: load + type: group + description: > + Load averages. + fields: + - name: "1" + type: half_float + description: > + Load average for the last minute. + - name: "5" + type: half_float + description: > + Load average for the last 5 minutes. + - name: "15" + type: half_float + description: > + Load average for the last 15 minutes. + + - name: "norm.1" + type: half_float + description: > + Load divided by the number of cores for the last minute. + + - name: "norm.5" + type: half_float + description: > + Load divided by the number of cores for the last 5 minutes. + + - name: "norm.15" + type: half_float + description: > + Load divided by the number of cores for the last 15 minutes. - name: memory type: group description: > @@ -1450,34 +1464,18 @@ - name: free type: long description: > - Available memory. + The total amount of free memory in bytes. This value does not include memory consumed by system caches and buffers (see system.memory.available). - name: used.pct type: half_float description: > The percentage of used memory. - - name: actual - type: group + - name: available + type: long description: > - Actual memory fields. - fields: - - name: used.bytes - type: long - description: > - Actual used memory. This value is the "used" memory minus the memory used for disk caches and buffers. - Available only on Unix. - - - name: free - type: long - description: > - Actual available memory. This value is the "free" memory plus the memory used for disk caches and - buffers. Available only on Unix. - - - name: used.pct - type: half_float - description: > - The percentage of actual used memory. + Available memory in bytes. It is calculated based on the OS. On Linux it consists of the free memory + plus caches and buffers. On OSX it is a sum of free memory and the inactive memory. - name: swap type: group diff --git a/metricbeat/include/list.go b/metricbeat/include/list.go index d785586d17e..b865c01a319 100644 --- a/metricbeat/include/list.go +++ b/metricbeat/include/list.go @@ -25,6 +25,7 @@ import ( _ "github.com/elastic/beats/metricbeat/module/system/diskio" _ "github.com/elastic/beats/metricbeat/module/system/filesystem" _ "github.com/elastic/beats/metricbeat/module/system/fsstat" + _ "github.com/elastic/beats/metricbeat/module/system/load" _ "github.com/elastic/beats/metricbeat/module/system/memory" _ "github.com/elastic/beats/metricbeat/module/system/network" _ "github.com/elastic/beats/metricbeat/module/system/process" diff --git a/metricbeat/metricbeat.full.yml b/metricbeat/metricbeat.full.yml index 9f18daa46fa..bccd7564124 100644 --- a/metricbeat/metricbeat.full.yml +++ b/metricbeat/metricbeat.full.yml @@ -16,6 +16,9 @@ metricbeat.modules: # CPU stats - cpu + # System Load stats + - load + # Per CPU core stats #- core diff --git a/metricbeat/metricbeat.template-es2x.json b/metricbeat/metricbeat.template-es2x.json index 1dc8445a080..61bd5063340 100644 --- a/metricbeat/metricbeat.template-es2x.json +++ b/metricbeat/metricbeat.template-es2x.json @@ -1088,19 +1088,6 @@ } } }, - "load": { - "properties": { - "1": { - "type": "float" - }, - "15": { - "type": "float" - }, - "5": { - "type": "float" - } - } - }, "nice": { "properties": { "pct": { @@ -1202,7 +1189,7 @@ }, "filesystem": { "properties": { - "avail": { + "available": { "type": "long" }, "device_name": { @@ -1262,24 +1249,36 @@ } } }, - "memory": { + "load": { "properties": { - "actual": { + "1": { + "type": "float" + }, + "15": { + "type": "float" + }, + "5": { + "type": "float" + }, + "norm": { "properties": { - "free": { - "type": "long" + "1": { + "type": "float" }, - "used": { - "properties": { - "bytes": { - "type": "long" - }, - "pct": { - "type": "float" - } - } + "15": { + "type": "float" + }, + "5": { + "type": "float" } } + } + } + }, + "memory": { + "properties": { + "available": { + "type": "long" }, "free": { "type": "long" diff --git a/metricbeat/metricbeat.template.json b/metricbeat/metricbeat.template.json index 92c51d71756..6886792e83b 100644 --- a/metricbeat/metricbeat.template.json +++ b/metricbeat/metricbeat.template.json @@ -1059,19 +1059,6 @@ } } }, - "load": { - "properties": { - "1": { - "type": "half_float" - }, - "15": { - "type": "half_float" - }, - "5": { - "type": "half_float" - } - } - }, "nice": { "properties": { "pct": { @@ -1171,7 +1158,7 @@ }, "filesystem": { "properties": { - "avail": { + "available": { "type": "long" }, "device_name": { @@ -1229,24 +1216,36 @@ } } }, - "memory": { + "load": { "properties": { - "actual": { + "1": { + "type": "half_float" + }, + "15": { + "type": "half_float" + }, + "5": { + "type": "half_float" + }, + "norm": { "properties": { - "free": { - "type": "long" + "1": { + "type": "half_float" }, - "used": { - "properties": { - "bytes": { - "type": "long" - }, - "pct": { - "type": "half_float" - } - } + "15": { + "type": "half_float" + }, + "5": { + "type": "half_float" } } + } + } + }, + "memory": { + "properties": { + "available": { + "type": "long" }, "free": { "type": "long" diff --git a/metricbeat/metricbeat.yml b/metricbeat/metricbeat.yml index e2fc374e0e9..9e458e17065 100644 --- a/metricbeat/metricbeat.yml +++ b/metricbeat/metricbeat.yml @@ -16,6 +16,9 @@ metricbeat.modules: # CPU stats - cpu + # System Load stats + - load + # Per CPU core stats #- core diff --git a/metricbeat/module/doc.go b/metricbeat/module/doc.go index 75cca56c481..8b095a72827 100644 --- a/metricbeat/module/doc.go +++ b/metricbeat/module/doc.go @@ -37,6 +37,7 @@ List of standardised words and units across all metricsets. On the left are the * ms: millisecond, millis * mb: megabytes * msg: message +* norm: normalized */ package module diff --git a/metricbeat/module/system/_meta/config.yml b/metricbeat/module/system/_meta/config.yml index 80b1329f781..2a9248dc247 100644 --- a/metricbeat/module/system/_meta/config.yml +++ b/metricbeat/module/system/_meta/config.yml @@ -3,6 +3,9 @@ # CPU stats - cpu + # System Load stats + - load + # Per CPU core stats #- core diff --git a/metricbeat/module/system/cpu/_meta/data.json b/metricbeat/module/system/cpu/_meta/data.json index 8072ea83c0b..1dd9703677e 100644 --- a/metricbeat/module/system/cpu/_meta/data.json +++ b/metricbeat/module/system/cpu/_meta/data.json @@ -1,55 +1,41 @@ { - "@timestamp": "2016-05-23T08:05:34.853Z", - "beat": { - "hostname": "host.example.com", - "name": "host.example.com" - }, - "metricset": { - "host": "localhost", - "module": "system", - "name": "cpu", - "rtt": 115 - }, - "system": { - "cpu": { - "idle": { - "pct": 0.9492, - "ticks": 337165 - }, - "iowait": { - "pct": 0, - "ticks": 1026 - }, - "irq": { - "pct": 0, - "ticks": 0 - }, - "load": { - "1": 1.76, - "15": 0.56, - "5": 0.99 - }, - "nice": { - "pct": 0, - "ticks": 0 - }, - "softirq": { - "pct": 0, - "ticks": 78 - }, - "steal": { - "pct": 0, - "ticks": 0 - }, - "system": { - "pct": 0.0051, - "ticks": 7665 - }, - "user": { - "pct": 0.0457, - "ticks": 37296 - } - } - }, - "type": "metricsets" -} \ No newline at end of file + "@timestamp": "2016-07-27T11:06:46.743Z", + "beat": { + "hostname": "host.example.com", + "name": "host.example.com" + }, + "metricset": { + "module": "system", + "name": "cpu", + "rtt": 29 + }, + "system": { + "cpu": { + "idle": { + "pct": 0.908 + }, + "iowait": { + "pct": 0 + }, + "irq": { + "pct": 0 + }, + "nice": { + "pct": 0 + }, + "softirq": { + "pct": 0 + }, + "steal": { + "pct": 0 + }, + "system": { + "pct": 0.036 + }, + "user": { + "pct": 0.056 + } + } + }, + "type": "metricsets" +} diff --git a/metricbeat/module/system/cpu/_meta/fields.yml b/metricbeat/module/system/cpu/_meta/fields.yml index 97bfffe39ea..32a79b824e3 100644 --- a/metricbeat/module/system/cpu/_meta/fields.yml +++ b/metricbeat/module/system/cpu/_meta/fields.yml @@ -87,21 +87,3 @@ The amount of CPU time spent in involuntary wait by the virtual CPU while the hypervisor was servicing another processor. Available only on Unix. - - - name: load - type: group - description: > - Load averages. - fields: - - name: "1" - type: half_float - description: > - Load average for the last minute. - - name: "5" - type: half_float - description: > - Load average for the last 5 minutes. - - name: "15" - type: half_float - description: > - Load average for the last 15 minutes. diff --git a/metricbeat/module/system/cpu/cpu.go b/metricbeat/module/system/cpu/cpu.go index 4ad161e15ab..ca96a0e80bb 100644 --- a/metricbeat/module/system/cpu/cpu.go +++ b/metricbeat/module/system/cpu/cpu.go @@ -51,11 +51,6 @@ func (m *MetricSet) Fetch() (common.MapStr, error) { } m.cpu.AddCpuPercentage(stat) - loadStat, err := GetSystemLoad() - if err != nil { - return nil, errors.Wrap(err, "load statistics") - } - cpuStat := common.MapStr{ "user": common.MapStr{ "pct": stat.UserPercent, @@ -94,11 +89,5 @@ func (m *MetricSet) Fetch() (common.MapStr, error) { cpuStat["steal"].(common.MapStr)["ticks"] = stat.Stolen } - cpuStat["load"] = common.MapStr{ - "1": loadStat.Load1, - "5": loadStat.Load5, - "15": loadStat.Load15, - } - return cpuStat, nil } diff --git a/metricbeat/module/system/cpu/helper.go b/metricbeat/module/system/cpu/helper.go index 041267957b3..83cad43cb38 100644 --- a/metricbeat/module/system/cpu/helper.go +++ b/metricbeat/module/system/cpu/helper.go @@ -3,12 +3,7 @@ package cpu import ( - "time" - - "github.com/elastic/beats/libbeat/common" - "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/metricbeat/module/system" - "github.com/elastic/beats/metricbeat/module/system/memory" sigar "github.com/elastic/gosigar" ) @@ -120,35 +115,6 @@ func GetCpuPercentageList(last, current []CpuTimes) []CpuTimes { return current } -func (cpu *CPU) GetCpuStatEvent(cpuStat *CpuTimes) common.MapStr { - result := common.MapStr{ - "user_p": cpuStat.UserPercent, - "system_p": cpuStat.SystemPercent, - "idle_p": cpuStat.IdlePercent, - "iowait_p": cpuStat.IOwaitPercent, - "irq_p": cpuStat.IrqPercent, - "nice_p": cpuStat.NicePercent, - "softirq_p": cpuStat.SoftIrqPercent, - "steal_p": cpuStat.StealPercent, - } - - if cpu.CpuTicks { - m := common.MapStr{ - "user": cpuStat.User, - "system": cpuStat.Sys, - "nice": cpuStat.Nice, - "idle": cpuStat.Idle, - "iowait": cpuStat.Wait, - "irq": cpuStat.Irq, - "softirq": cpuStat.SoftIrq, - "steal": cpuStat.Stolen, - } - return common.MapStrUnion(result, m) - } - return result - -} - func (cpu *CPU) AddCpuPercentage(t2 *CpuTimes) { cpu.LastCpuTimes = GetCpuPercentage(cpu.LastCpuTimes, t2) } @@ -156,90 +122,3 @@ func (cpu *CPU) AddCpuPercentage(t2 *CpuTimes) { func (cpu *CPU) AddCpuPercentageList(t2 []CpuTimes) { cpu.LastCpuTimesList = GetCpuPercentageList(cpu.LastCpuTimesList, t2) } - -func (cpu *CPU) GetSystemStats() (common.MapStr, error) { - loadStat, err := GetSystemLoad() - if err != nil { - logp.Warn("Getting load statistics: %v", err) - return nil, err - } - cpuStat, err := GetCpuTimes() - if err != nil { - logp.Warn("Getting cpu times: %v", err) - return nil, err - } - - cpu.AddCpuPercentage(cpuStat) - - memStat, err := memory.GetMemory() - if err != nil { - logp.Warn("Getting memory details: %v", err) - return nil, err - } - memory.AddMemPercentage(memStat) - - swapStat, err := memory.GetSwap() - if err != nil { - logp.Warn("Getting swap details: %v", err) - return nil, err - } - memory.AddSwapPercentage(swapStat) - - event := common.MapStr{ - "@timestamp": common.Time(time.Now()), - "type": "system", - "load": loadStat, - "cpu": cpu.GetCpuStatEvent(cpuStat), - "mem": memory.GetMemoryEvent(memStat), - "swap": memory.GetSwapEvent(swapStat), - } - - return event, nil -} - -func (cpu *CPU) GetCoreStats() ([]common.MapStr, error) { - - events := []common.MapStr{} - - cpuCoreStat, err := GetCpuTimesList() - if err != nil { - logp.Warn("Getting cpu core times: %v", err) - return nil, err - } - cpu.AddCpuPercentageList(cpuCoreStat) - - for coreNumber, stat := range cpuCoreStat { - coreStat := cpu.GetCpuStatEvent(&stat) - coreStat["id"] = coreNumber - - event := common.MapStr{ - "@timestamp": common.Time(time.Now()), - "type": "core", - "core": coreStat, - } - events = append(events, event) - } - - return events, nil -} - -type SystemLoad struct { - Load1 float64 `json:"load1"` - Load5 float64 `json:"load5"` - Load15 float64 `json:"load15"` -} - -func GetSystemLoad() (*SystemLoad, error) { - - concreteSigar := sigar.ConcreteSigar{} - avg, err := concreteSigar.GetLoadAverage() - if err != nil { - return nil, err - } - - return &SystemLoad{ - Load1: avg.One, - Load5: avg.Five, - Load15: avg.Fifteen, - }, nil -} diff --git a/metricbeat/module/system/cpu/helper_test.go b/metricbeat/module/system/cpu/helper_test.go index 22901afaf7e..c9407a8ba7c 100644 --- a/metricbeat/module/system/cpu/helper_test.go +++ b/metricbeat/module/system/cpu/helper_test.go @@ -4,7 +4,6 @@ package cpu import ( - "runtime" "testing" "github.com/elastic/gosigar" @@ -63,19 +62,3 @@ func TestCpuPercentage(t *testing.T) { assert.Equal(t, cpu2.UserPercent, 0.9502) assert.Equal(t, cpu2.SystemPercent, 0.0448) } - -func TestGetSystemLoad(t *testing.T) { - - if runtime.GOOS == "windows" { - return //no load data on windows - } - - load, err := GetSystemLoad() - - assert.NotNil(t, load) - assert.Nil(t, err) - - assert.True(t, (load.Load1 > 0)) - assert.True(t, (load.Load5 > 0)) - assert.True(t, (load.Load15 > 0)) -} diff --git a/metricbeat/module/system/filesystem/_meta/data.json b/metricbeat/module/system/filesystem/_meta/data.json index cfaa5cd30c7..8091fef1598 100644 --- a/metricbeat/module/system/filesystem/_meta/data.json +++ b/metricbeat/module/system/filesystem/_meta/data.json @@ -1,29 +1,29 @@ { - "@timestamp": "2016-05-23T08:05:34.853Z", - "beat": { - "hostname": "host.example.com", - "name": "host.example.com" - }, - "metricset": { - "host": "localhost", - "module": "system", - "name": "filesystem", - "rtt": 115 - }, - "system": { - "filesystem": { - "avail": 44886921216, - "device_name": "none", - "files": 3940352, - "free": 48129642496, - "free_files": 3399857, - "mount_point": "/", - "total": 63371726848, - "used": { - "bytes": 15242084352, - "pct": 0.2405 - } - } - }, - "type": "metricsets" -} \ No newline at end of file + "@timestamp": "2016-05-23T08:05:34.853Z", + "beat": { + "hostname": "host.example.com", + "name": "host.example.com" + }, + "metricset": { + "host": "localhost", + "module": "system", + "name": "filesystem", + "rtt": 115 + }, + "system": { + "filesystem": { + "available": 44886921216, + "device_name": "none", + "files": 3940352, + "free": 48129642496, + "free_files": 3399857, + "mount_point": "/", + "total": 63371726848, + "used": { + "bytes": 15242084352, + "pct": 0.2405 + } + } + }, + "type": "metricsets" + } diff --git a/metricbeat/module/system/filesystem/_meta/fields.yml b/metricbeat/module/system/filesystem/_meta/fields.yml index 7b4dcee9cdd..d24b8722e8c 100644 --- a/metricbeat/module/system/filesystem/_meta/fields.yml +++ b/metricbeat/module/system/filesystem/_meta/fields.yml @@ -3,7 +3,7 @@ description: > `filesystem` contains local filesystem stats. fields: - - name: avail + - name: available type: long description: > The disk space available to an unprivileged user in bytes. diff --git a/metricbeat/module/system/filesystem/helper.go b/metricbeat/module/system/filesystem/helper.go index 6fc18154a27..3d48340c418 100644 --- a/metricbeat/module/system/filesystem/helper.go +++ b/metricbeat/module/system/filesystem/helper.go @@ -82,7 +82,7 @@ func GetFilesystemEvent(fsStat *FileSystemStat) common.MapStr { "mount_point": fsStat.Mount, "total": fsStat.Total, "free": fsStat.Free, - "avail": fsStat.Avail, + "available": fsStat.Avail, "files": fsStat.Files, "free_files": fsStat.FreeFiles, "used": common.MapStr{ diff --git a/metricbeat/module/system/load/_meta/data.json b/metricbeat/module/system/load/_meta/data.json new file mode 100644 index 00000000000..fabbee018c1 --- /dev/null +++ b/metricbeat/module/system/load/_meta/data.json @@ -0,0 +1,25 @@ +{ + "@timestamp": "2016-07-27T11:05:31.362Z", + "beat": { + "hostname": "host.example.com", + "name": "host.example.com" + }, + "metricset": { + "module": "system", + "name": "load", + "rtt": 133 + }, + "system": { + "load": { + "1": 3.9399, + "15": 3.0063, + "5": 3.6045, + "norm": { + "1": 0.985, + "15": 0.7516, + "5": 0.9011 + } + } + }, + "type": "metricsets" +} diff --git a/metricbeat/module/system/load/_meta/docs.asciidoc b/metricbeat/module/system/load/_meta/docs.asciidoc new file mode 100644 index 00000000000..d043fd5f86a --- /dev/null +++ b/metricbeat/module/system/load/_meta/docs.asciidoc @@ -0,0 +1,10 @@ +=== system load MetricSet + +The System `load` metricset provides load statistics. + +This metricset is available on: + + - Darwin + - FreeBSD + - Linux + - OpenBSD diff --git a/metricbeat/module/system/load/_meta/fields.yml b/metricbeat/module/system/load/_meta/fields.yml new file mode 100644 index 00000000000..2aaa0ca6dd4 --- /dev/null +++ b/metricbeat/module/system/load/_meta/fields.yml @@ -0,0 +1,32 @@ +- name: load + type: group + description: > + Load averages. + fields: + - name: "1" + type: half_float + description: > + Load average for the last minute. + - name: "5" + type: half_float + description: > + Load average for the last 5 minutes. + - name: "15" + type: half_float + description: > + Load average for the last 15 minutes. + + - name: "norm.1" + type: half_float + description: > + Load divided by the number of cores for the last minute. + + - name: "norm.5" + type: half_float + description: > + Load divided by the number of cores for the last 5 minutes. + + - name: "norm.15" + type: half_float + description: > + Load divided by the number of cores for the last 15 minutes. diff --git a/metricbeat/module/system/load/doc.go b/metricbeat/module/system/load/doc.go new file mode 100644 index 00000000000..67b1843dc35 --- /dev/null +++ b/metricbeat/module/system/load/doc.go @@ -0,0 +1,4 @@ +/* +Package load collects system load metrics from the host OS. +*/ +package load diff --git a/metricbeat/module/system/load/helper.go b/metricbeat/module/system/load/helper.go new file mode 100644 index 00000000000..23cc9fc6145 --- /dev/null +++ b/metricbeat/module/system/load/helper.go @@ -0,0 +1,39 @@ +// +build darwin freebsd linux openbsd + +package load + +import sigar "github.com/elastic/gosigar" + +type SystemLoad struct { + Load1 float64 `json:"load1"` + Load5 float64 `json:"load5"` + Load15 float64 `json:"load15"` + + /* normalized values load / cores */ + LoadNorm1 float64 `json:"load1_norm"` + LoadNorm5 float64 `json:"load5_norm"` + LoadNorm15 float64 `json:"load15_norm"` +} + +func GetSystemLoad() (*SystemLoad, error) { + + concreteSigar := sigar.ConcreteSigar{} + avg, err := concreteSigar.GetLoadAverage() + if err != nil { + return nil, err + } + + cpuList := sigar.CpuList{} + cpuList.Get() + numCore := len(cpuList.List) + + return &SystemLoad{ + Load1: avg.One, + Load5: avg.Five, + Load15: avg.Fifteen, + + LoadNorm1: avg.One / float64(numCore), + LoadNorm5: avg.Five / float64(numCore), + LoadNorm15: avg.Fifteen / float64(numCore), + }, nil +} diff --git a/metricbeat/module/system/load/helper_test.go b/metricbeat/module/system/load/helper_test.go new file mode 100644 index 00000000000..8b532517ea7 --- /dev/null +++ b/metricbeat/module/system/load/helper_test.go @@ -0,0 +1,22 @@ +// +build !integration +// +build darwin freebsd linux openbsd + +package load + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestGetSystemLoad(t *testing.T) { + + load, err := GetSystemLoad() + + assert.NotNil(t, load) + assert.Nil(t, err) + + assert.True(t, (load.Load1 > 0)) + assert.True(t, (load.Load5 > 0)) + assert.True(t, (load.Load15 > 0)) +} diff --git a/metricbeat/module/system/load/load.go b/metricbeat/module/system/load/load.go new file mode 100644 index 00000000000..3974079d9da --- /dev/null +++ b/metricbeat/module/system/load/load.go @@ -0,0 +1,68 @@ +// +build darwin freebsd linux openbsd + +package load + +import ( + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/metricbeat/mb" + "github.com/elastic/beats/metricbeat/module/system" + "github.com/pkg/errors" +) + +// init registers the MetricSet with the central registry. +// The New method will be called after the setup of the module and before starting to fetch data +func init() { + if err := mb.Registry.AddMetricSet("system", "load", New); err != nil { + panic(err) + } +} + +// MetricSet type defines all fields of the MetricSet +// As a minimum it must inherit the mb.BaseMetricSet fields, but can be extended with +// additional entries. These variables can be used to persist data or configuration between +// multiple fetch calls. +type MetricSet struct { + mb.BaseMetricSet + counter int +} + +// New create a new instance of the MetricSet +// Part of new is also setting up the configuration by processing additional +// configuration entries if needed. +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + + config := struct{}{} + + if err := base.Module().UnpackConfig(&config); err != nil { + return nil, err + } + + return &MetricSet{ + BaseMetricSet: base, + counter: 1, + }, nil +} + +// Fetch methods implements the data gathering and data conversion to the right format +// It returns the event which is then forward to the output. In case of an error, a +// descriptive error must be returned. +func (m *MetricSet) Fetch() (common.MapStr, error) { + + loadStat, err := GetSystemLoad() + if err != nil { + return nil, errors.Wrap(err, "load statistics") + } + + event := common.MapStr{ + "1": system.Round(loadStat.Load1, .5, 4), + "5": system.Round(loadStat.Load5, .5, 4), + "15": system.Round(loadStat.Load15, .5, 4), + "norm": common.MapStr{ + "1": system.Round(loadStat.LoadNorm1, .5, 4), + "5": system.Round(loadStat.LoadNorm5, .5, 4), + "15": system.Round(loadStat.LoadNorm15, .5, 4), + }, + } + + return event, nil +} diff --git a/metricbeat/module/system/memory/_meta/data.json b/metricbeat/module/system/memory/_meta/data.json index 9e4a3b70522..3702fbc67b3 100644 --- a/metricbeat/module/system/memory/_meta/data.json +++ b/metricbeat/module/system/memory/_meta/data.json @@ -1,39 +1,32 @@ { - "@timestamp": "2016-05-23T08:05:34.853Z", - "beat": { - "hostname": "host.example.com", - "name": "host.example.com" - }, - "metricset": { - "host": "localhost", - "module": "system", - "name": "memory", - "rtt": 115 - }, - "system": { - "memory": { - "actual": { - "free": 6274166784, - "used": { - "bytes": 2089246720, - "pct": 0.2498 - } - }, - "free": 5331386368, - "swap": { - "free": 4199542784, - "total": 4199542784, - "used": { - "bytes": 0, - "pct": 0 - } - }, - "total": 8363413504, - "used": { - "bytes": 3032027136, - "pct": 0.3625 - } + "@timestamp": "2016-07-27T11:08:54.166Z", + "beat": { + "hostname": "host.example.com", + "name": "host.example.com" + }, + "metricset": { + "module": "system", + "name": "memory", + "rtt": 135 + }, + "system": { + "memory": { + "available": 5624123392, + "free": 3634081792, + "swap": { + "free": 952631296, + "total": 1073741824, + "used": { + "bytes": 121110528, + "pct": 0.1128 } - }, - "type": "metricsets" -} \ No newline at end of file + }, + "total": 17179869184, + "used": { + "bytes": 13545787392, + "pct": 0.7885 + } + } + }, + "type": "metricsets" +} diff --git a/metricbeat/module/system/memory/_meta/fields.yml b/metricbeat/module/system/memory/_meta/fields.yml index 9756fdc6ab1..5675e90ec14 100644 --- a/metricbeat/module/system/memory/_meta/fields.yml +++ b/metricbeat/module/system/memory/_meta/fields.yml @@ -16,34 +16,18 @@ - name: free type: long description: > - Available memory. + The total amount of free memory in bytes. This value does not include memory consumed by system caches and buffers (see system.memory.available). - name: used.pct type: half_float description: > The percentage of used memory. - - name: actual - type: group + - name: available + type: long description: > - Actual memory fields. - fields: - - name: used.bytes - type: long - description: > - Actual used memory. This value is the "used" memory minus the memory used for disk caches and buffers. - Available only on Unix. - - - name: free - type: long - description: > - Actual available memory. This value is the "free" memory plus the memory used for disk caches and - buffers. Available only on Unix. - - - name: used.pct - type: half_float - description: > - The percentage of actual used memory. + Available memory in bytes. It is calculated based on the OS. On Linux it consists of the free memory + plus caches and buffers. On OSX it is a sum of free memory and the inactive memory. - name: swap type: group diff --git a/metricbeat/module/system/memory/memory.go b/metricbeat/module/system/memory/memory.go index 1c649519ea5..bdc1fd32246 100644 --- a/metricbeat/module/system/memory/memory.go +++ b/metricbeat/module/system/memory/memory.go @@ -47,14 +47,8 @@ func (m *MetricSet) Fetch() (event common.MapStr, err error) { "bytes": memStat.Used, "pct": memStat.UsedPercent, }, - "free": memStat.Free, - "actual": common.MapStr{ - "free": memStat.ActualFree, - "used": common.MapStr{ - "pct": memStat.ActualUsedPercent, - "bytes": memStat.ActualUsed, - }, - }, + "free": memStat.Free, + "available": memStat.ActualFree, } swap := common.MapStr{ diff --git a/metricbeat/tests/system/test_system.py b/metricbeat/tests/system/test_system.py index f28ecfe0265..721f6721bcd 100644 --- a/metricbeat/tests/system/test_system.py +++ b/metricbeat/tests/system/test_system.py @@ -5,12 +5,14 @@ import getpass import os -SYSTEM_CPU_FIELDS = ["idle.pct", "iowait.pct", "irq.pct", "load", "nice.pct", +SYSTEM_CPU_FIELDS = ["idle.pct", "iowait.pct", "irq.pct", "nice.pct", "softirq.pct", "steal.pct", "system.pct", "user.pct"] -SYSTEM_CPU_FIELDS_ALL = ["idle.pct", "idle.ticks", "iowait.pct", "iowait.ticks", "irq.pct", "irq.ticks", "load", "nice.pct", "nice.ticks", +SYSTEM_CPU_FIELDS_ALL = ["idle.pct", "idle.ticks", "iowait.pct", "iowait.ticks", "irq.pct", "irq.ticks", "nice.pct", "nice.ticks", "softirq.pct", "softirq.ticks", "steal.pct", "steal.ticks", "system.pct", "system.ticks", "user.pct", "user.ticks"] +SYSTEM_LOAD_FIELDS = ["1", "5", "15", "norm.1", "norm.5", "norm.15"] + SYSTEM_CORE_FIELDS = ["id", "idle.pct", "iowait.pct", "irq.pct", "nice.pct", "softirq.pct", "steal.pct", "system.pct", "user.pct"] @@ -20,13 +22,13 @@ SYSTEM_DISKIO_FIELDS = ["name", "read.count", "write.count", "read.bytes", "write.bytes", "read.time", "write.time", "io.time"] -SYSTEM_FILESYSTEM_FIELDS = ["avail", "device_name", "files", "free", +SYSTEM_FILESYSTEM_FIELDS = ["available", "device_name", "files", "free", "free_files", "mount_point", "total", "used.bytes", "used.pct"] SYSTEM_FSSTAT_FIELDS = ["count", "total_files", "total_size"] -SYSTEM_MEMORY_FIELDS = ["swap", "actual", "free", "total", "used.bytes", "used.pct"] +SYSTEM_MEMORY_FIELDS = ["swap", "available", "free", "total", "used.bytes", "used.pct"] SYSTEM_NETWORK_FIELDS = ["name", "out.bytes", "in.bytes", "out.packets", "in.packets", "in.error", "out.error", "in.dropeed", "out.dropped"] @@ -148,6 +150,32 @@ def test_core_with_cpu_ticks(self): core = evt["system"]["core"] self.assertItemsEqual(self.de_dot(SYSTEM_CORE_FIELDS_ALL), core.keys()) + @unittest.skipUnless(re.match("(?i)linux|darwin|freebsd|openbsd", sys.platform), "os") + def test_load(self): + """ + Test system load. + """ + self.render_config_template(modules=[{ + "name": "system", + "metricsets": ["load"], + "period": "5s" + }]) + proc = self.start_beat() + self.wait_until(lambda: self.output_lines() > 0) + proc.check_kill_and_wait() + + # Ensure no errors or warnings exist in the log. + log = self.get_log() + self.assertNotRegexpMatches(log, "ERR|WARN") + + output = self.read_output_json() + self.assertEqual(len(output), 1) + evt = output[0] + self.assert_fields_are_documented(evt) + + cpu = evt["system"]["load"] + self.assertItemsEqual(self.de_dot(SYSTEM_LOAD_FIELDS), cpu.keys()) + @unittest.skipUnless(re.match("(?i)win|linux|freebsd", sys.platform), "os") def test_diskio(self): """ @@ -258,9 +286,6 @@ def test_memory(self): used_p = float(mem["used"]["bytes"]) / mem["total"] self.assertAlmostEqual(mem["used"]["pct"], used_p, places=4) - used_p = float(mem["actual"]["used"]["bytes"]) / mem["total"] - self.assertAlmostEqual(mem["actual"]["used"]["pct"], used_p, places=4) - swap = memory["swap"] if swap["total"] != 0: used_p = float(swap["used"]["bytes"]) / swap["total"]