[Metricbeat] Expand docker/blkio data (#16638)

* initial commit of blkio enhancements * add test coverage * add changelog
elastic · Mar 6, 2020 · ee2d287 · ee2d287
1 parent 62f2971
commit ee2d287
Show file tree

Hide file tree

Showing 9 changed files with 217 additions and 47 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -214,6 +214,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add `cloudfoundry` module to send events from Cloud Foundry. {pull}16671[16671]
 - Add `redisenterprise` module. {pull}16482[16482] {issue}15269[15269]
 - Align fields to ECS and add more tests for the azure module. {issue}16024[16024] {pull}16754[16754]
+- Add additional cgroup fields to docker/diskio{pull}16638[16638]
 - Add PubSub metricset to Google Cloud Platform module {pull}15536[15536]
 
 *Packetbeat*

diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc
@@ -6509,6 +6509,36 @@ format: bytes
 Number of current reads per second
 
 
+type: long
+
+--
+
+*`docker.diskio.read.service_time`*::
++
+--
+Total time to service IO requests, in nanoseconds
+
+
+type: long
+
+--
+
+*`docker.diskio.read.wait_time`*::
++
+--
+Total time requests spent waiting in queues for service, in nanoseconds
+
+
+type: long
+
+--
+
+*`docker.diskio.read.queued`*::
++
+--
+Total number of queued requests
+
+
 type: long
 
 --
@@ -6561,6 +6591,36 @@ format: bytes
 Number of current writes per second
 
 
+type: long
+
+--
+
+*`docker.diskio.write.service_time`*::
++
+--
+Total time to service IO requests, in nanoseconds
+
+
+type: long
+
+--
+
+*`docker.diskio.write.wait_time`*::
++
+--
+Total time requests spent waiting in queues for service, in nanoseconds
+
+
+type: long
+
+--
+
+*`docker.diskio.write.queued`*::
++
+--
+Total number of queued requests
+
+
 type: long
 
 --
@@ -6613,6 +6673,36 @@ format: bytes
 Number of current operations per second
 
 
+type: long
+
+--
+
+*`docker.diskio.summary.service_time`*::
++
+--
+Total time to service IO requests, in nanoseconds
+
+
+type: long
+
+--
+
+*`docker.diskio.summary.wait_time`*::
++
+--
+Total time requests spent waiting in queues for service, in nanoseconds
+
+
+type: long
+
+--
+
+*`docker.diskio.summary.queued`*::
++
+--
+Total number of queued requests
+
+
 type: long
 
 --

diff --git a/metricbeat/module/docker/diskio/_meta/data.json b/metricbeat/module/docker/diskio/_meta/data.json
@@ -1,55 +1,40 @@
 {
     "@timestamp": "2017-10-12T08:05:34.853Z",
-    "agent": {
-        "hostname": "host.example.com",
-        "name": "host.example.com"
-    },
     "container": {
-        "id": "cc78e58acfda4501105dc4de8e3ae218f2da616213e6e3af168c40103829302a",
+        "id": "8abaa1f3514d3554503034a1df6ee09457f328757bbc9555245244ee853c0b44",
         "image": {
-            "name": "metricbeat_elasticsearch"
+            "name": "zookeeper"
         },
-        "name": "metricbeat_elasticsearch_1_df866b3a7b3d",
+        "name": "some-zookeeper",
         "runtime": "docker"
     },
     "docker": {
-        "container": {
-            "labels": {
-                "com_docker_compose_config-hash": "e3e0a2c6e5d1afb741bc8b1ecb09cda0395886b7a3e5084a9fd110be46d70f78",
-                "com_docker_compose_container-number": "1",
-                "com_docker_compose_oneoff": "False",
-                "com_docker_compose_project": "metricbeat",
-                "com_docker_compose_service": "elasticsearch",
-                "com_docker_compose_slug": "df866b3a7b3d50c0802350cbe58ee5b34fa32b7f6ba7fe9e48cde2c12dd0201d",
-                "com_docker_compose_version": "1.23.1",
-                "license": "Elastic License",
-                "org_label-schema_build-date": "20181006",
-                "org_label-schema_license": "GPLv2",
-                "org_label-schema_name": "elasticsearch",
-                "org_label-schema_schema-version": "1.0",
-                "org_label-schema_url": "https://www.elastic.co/products/elasticsearch",
-                "org_label-schema_vcs-url": "https://github.com/elastic/elasticsearch-docker",
-                "org_label-schema_vendor": "Elastic",
-                "org_label-schema_version": "6.5.1"
-            }
-        },
         "diskio": {
             "read": {
-                "bytes": 998932480,
-                "ops": 8473,
-                "rate": 0
+                "bytes": 42409984,
+                "ops": 1823,
+                "queued": 0,
+                "rate": 0,
+                "service_time": 0,
+                "wait_time": 0
             },
             "reads": 0,
             "summary": {
-                "bytes": 1090650112,
-                "ops": 9585,
-                "rate": 0
+                "bytes": 42414080,
+                "ops": 1824,
+                "queued": 0,
+                "rate": 0,
+                "service_time": 0,
+                "wait_time": 0
             },
             "total": 0,
             "write": {
-                "bytes": 91717632,
-                "ops": 1112,
-                "rate": 0
+                "bytes": 4096,
+                "ops": 1,
+                "queued": 0,
+                "rate": 0,
+                "service_time": 0,
+                "wait_time": 0
             },
             "writes": 0
         }
@@ -60,7 +45,8 @@
         "module": "docker"
     },
     "metricset": {
-        "name": "diskio"
+        "name": "diskio",
+        "period": 10000
     },
     "service": {
         "address": "/var/run/docker.sock",

diff --git a/metricbeat/module/docker/diskio/_meta/fields.yml b/metricbeat/module/docker/diskio/_meta/fields.yml
@@ -22,6 +22,18 @@
           type: long
           description: >
             Number of current reads per second
+        - name: service_time
+          type: long
+          description: >
+            Total time to service IO requests, in nanoseconds
+        - name: wait_time
+          type: long
+          description: > 
+            Total time requests spent waiting in queues for service, in nanoseconds
+        - name: queued
+          type: long
+          description: >
+            Total number of queued requests
     - name: reads
       type: scaled_float
       deprecated: 6.4
@@ -45,6 +57,18 @@
           type: long
           description: >
             Number of current writes per second
+        - name: service_time
+          type: long
+          description: >
+            Total time to service IO requests, in nanoseconds
+        - name: wait_time
+          type: long
+          description: > 
+            Total time requests spent waiting in queues for service, in nanoseconds
+        - name: queued
+          type: long
+          description: >
+            Total number of queued requests
     - name: writes
       type: scaled_float
       deprecated: 6.4
@@ -68,6 +92,18 @@
           type: long
           description: >
             Number of current operations per second
+        - name: service_time
+          type: long
+          description: >
+            Total time to service IO requests, in nanoseconds
+        - name: wait_time
+          type: long
+          description: > 
+            Total time requests spent waiting in queues for service, in nanoseconds
+        - name: queued
+          type: long
+          description: >
+            Total number of queued requests
     - name: total
       type: scaled_float
       deprecated: 6.4

diff --git a/metricbeat/module/docker/diskio/data.go b/metricbeat/module/docker/diskio/data.go
@@ -34,19 +34,28 @@ func eventMapping(r mb.ReporterV2, stats *BlkioStats) {
 		"writes": stats.writes,
 		"total":  stats.totals,
 		"read": common.MapStr{
-			"ops":   stats.serviced.reads,
-			"bytes": stats.servicedBytes.reads,
-			"rate":  stats.reads,
+			"ops":          stats.serviced.reads,
+			"bytes":        stats.servicedBytes.reads,
+			"rate":         stats.reads,
+			"service_time": stats.servicedTime.reads,
+			"wait_time":    stats.waitTime.reads,
+			"queued":       stats.queued.reads,
 		},
 		"write": common.MapStr{
-			"ops":   stats.serviced.writes,
-			"bytes": stats.servicedBytes.writes,
-			"rate":  stats.writes,
+			"ops":          stats.serviced.writes,
+			"bytes":        stats.servicedBytes.writes,
+			"rate":         stats.writes,
+			"service_time": stats.servicedTime.writes,
+			"wait_time":    stats.waitTime.writes,
+			"queued":       stats.queued.writes,
 		},
 		"summary": common.MapStr{
-			"ops":   stats.serviced.totals,
-			"bytes": stats.servicedBytes.totals,
-			"rate":  stats.totals,
+			"ops":          stats.serviced.totals,
+			"bytes":        stats.servicedBytes.totals,
+			"rate":         stats.totals,
+			"service_time": stats.servicedTime.totals,
+			"wait_time":    stats.waitTime.totals,
+			"queued":       stats.queued.totals,
 		},
 	}
 

diff --git a/metricbeat/module/docker/diskio/diskio.go b/metricbeat/module/docker/diskio/diskio.go
@@ -34,6 +34,7 @@ func init() {
 	)
 }
 
+// MetricSet type defines all fields of the MetricSet
 type MetricSet struct {
 	mb.BaseMetricSet
 	blkioService *BlkioService

diff --git a/metricbeat/module/docker/diskio/diskio_test.go b/metricbeat/module/docker/diskio/diskio_test.go
@@ -256,6 +256,30 @@ func TestGetBlkioStatsList(t *testing.T) {
 					{Major: 1, Minor: 2, Op: "Write", Value: 1000},
 					{Major: 1, Minor: 2, Op: "Total", Value: 1500},
 				},
+				IoServiceTimeRecursive: []types.BlkioStatEntry{
+					{Major: 1, Minor: 1, Op: "Read", Value: 10000},
+					{Major: 1, Minor: 1, Op: "Write", Value: 20000},
+					{Major: 1, Minor: 1, Op: "Total", Value: 30000},
+					{Major: 1, Minor: 2, Op: "Read", Value: 500},
+					{Major: 1, Minor: 2, Op: "Write", Value: 1500},
+					{Major: 1, Minor: 2, Op: "Total", Value: 2000},
+				},
+				IoWaitTimeRecursive: []types.BlkioStatEntry{
+					{Major: 1, Minor: 1, Op: "Read", Value: 1000000},
+					{Major: 1, Minor: 1, Op: "Write", Value: 25604332},
+					{Major: 1, Minor: 1, Op: "Total", Value: 26604332},
+					{Major: 1, Minor: 2, Op: "Read", Value: 500},
+					{Major: 1, Minor: 2, Op: "Write", Value: 1500},
+					{Major: 1, Minor: 2, Op: "Total", Value: 2000},
+				},
+				IoQueuedRecursive: []types.BlkioStatEntry{
+					{Major: 1, Minor: 1, Op: "Read", Value: 100},
+					{Major: 1, Minor: 1, Op: "Write", Value: 200},
+					{Major: 1, Minor: 1, Op: "Total", Value: 300},
+					{Major: 1, Minor: 2, Op: "Read", Value: 50},
+					{Major: 1, Minor: 2, Op: "Write", Value: 100},
+					{Major: 1, Minor: 2, Op: "Total", Value: 150},
+				},
 			},
 		}},
 	}}
@@ -271,6 +295,15 @@ func TestGetBlkioStatsList(t *testing.T) {
 	assert.Equal(t,
 		BlkioRaw{Time: later, reads: 1500, writes: 3000, totals: 4500},
 		stats.servicedBytes)
+	assert.Equal(t,
+		BlkioRaw{Time: later, reads: 10500, writes: 21500, totals: 32000},
+		stats.servicedTime)
+	assert.Equal(t,
+		BlkioRaw{Time: later, reads: 1000500, writes: 25605832, totals: 26606332},
+		stats.waitTime)
+	assert.Equal(t,
+		BlkioRaw{Time: later, reads: 150, writes: 300, totals: 450},
+		stats.queued)
 }
 
 func TestGetBlkioStatsListWindows(t *testing.T) {

diff --git a/metricbeat/module/docker/diskio/helper.go b/metricbeat/module/docker/diskio/helper.go
@@ -25,6 +25,7 @@ import (
 	"github.com/elastic/beats/v7/metricbeat/module/docker"
 )
 
+// BlkioStats contains all formatted blkio stats
 type BlkioStats struct {
 	Time      time.Time
 	Container *docker.Container
@@ -34,6 +35,9 @@ type BlkioStats struct {
 
 	serviced      BlkioRaw
 	servicedBytes BlkioRaw
+	servicedTime  BlkioRaw
+	waitTime      BlkioRaw
+	queued        BlkioRaw
 }
 
 // Add adds blkio stats
@@ -46,6 +50,7 @@ func (s *BlkioStats) Add(o *BlkioStats) {
 	s.servicedBytes.Add(&o.servicedBytes)
 }
 
+// BlkioRaw sums raw Blkio stats
 type BlkioRaw struct {
 	Time   time.Time
 	reads  uint64
@@ -130,6 +135,15 @@ func (io *BlkioService) getBlkioStats(myRawStat *docker.Stat, dedot bool) BlkioS
 		servicedBytes: io.getNewStats(
 			myRawStat.Stats.Read,
 			myRawStat.Stats.BlkioStats.IoServiceBytesRecursive),
+		servicedTime: io.getNewStats(
+			myRawStat.Stats.Read,
+			myRawStat.Stats.BlkioStats.IoServiceTimeRecursive),
+		waitTime: io.getNewStats(
+			myRawStat.Stats.Read,
+			myRawStat.Stats.BlkioStats.IoWaitTimeRecursive),
+		queued: io.getNewStats(
+			myRawStat.Stats.Read,
+			myRawStat.Stats.BlkioStats.IoQueuedRecursive),
 	}
 }