Added write and read disk IO times on Metricbeat system module (#8508)

Added write and read disk io times on Metricbeat system module

CHANGELOG.asciidoc

@@ -79,6 +79,7 @@ https://github.com/elastic/beats/compare/v6.4.0...master[Check the HEAD diff]
 - Add docker diskio stats on Windows. {issue}6815[6815] {pull}8126[8126]
 - Fix incorrect type conversion of average response time in Haproxy dashboards {pull}8404[8404]
 - Fix dropwizard module parsing of metric names. {issue}8365[8365] {pull}6385[8385]
+- Added io disk read and write times to system module {issue}8473[8473] {pull}8508[8508]
 - Avoid mapping issues in kubernetes module. {pull}8487[8487]
 
 *Packetbeat*

metricbeat/docs/fields.asciidoc

@@ -19918,6 +19918,16 @@ format: bytes
 The number of Bytes read from the device per second.
 
 
+--
+
+*`system.diskio.iostat.read.await`*::
++
+--
+type: float
+
+The average time spent for read requests issued to the device to be served.
+
+
 --
 
 *`system.diskio.iostat.write.per_sec.bytes`*::
@@ -19930,6 +19940,16 @@ format: bytes
 The number of Bytes write from the device per second.
 
 
+--
+
+*`system.diskio.iostat.write.await`*::
++
+--
+type: float
+
+The average time spent for write requests issued to the device to be served.
+
+
 --
 
 *`system.diskio.iostat.request.avg_size`*::

metricbeat/module/system/diskio/_meta/data.json

@@ -27,7 +27,8 @@
                     "request": {
                         "merges_per_sec": 0,
                         "per_sec": 0
-                    }
+                    },
+                    "await": 0
                 },
                 "request": {
                     "avg_size": 0
@@ -40,7 +41,8 @@
                     "request": {
                         "merges_per_sec": 0,
                         "per_sec": 0
-                    }
+                    },
+                    "await": 0
                 }
             },
             "name": "sda",

metricbeat/module/system/diskio/_meta/fields.yml

@@ -82,12 +82,22 @@
           The number of Bytes read from the device per second.
       format: bytes
 
+    - name: iostat.read.await
+      type: float
+      description: >
+          The average time spent for read requests issued to the device to be served.
+
     - name: iostat.write.per_sec.bytes
       type: float
       description: >
           The number of Bytes write from the device per second.
       format: bytes
 
+    - name: iostat.write.await
+      type: float
+      description: >
+          The average time spent for write requests issued to the device to be served.
+
     - name: iostat.request.avg_size
       type: float
       description: >

metricbeat/module/system/diskio/diskio.go

@@ -99,6 +99,7 @@ func (m *MetricSet) Fetch() ([]common.MapStr, error) {
 					"per_sec": common.MapStr{
 						"bytes": extraMetrics.ReadBytesPerSec,
 					},
+					"await": extraMetrics.AvgReadAwaitTime,
 				},
 				"write": common.MapStr{
 					"request": common.MapStr{
@@ -108,6 +109,7 @@ func (m *MetricSet) Fetch() ([]common.MapStr, error) {
 					"per_sec": common.MapStr{
 						"bytes": extraMetrics.WriteBytesPerSec,
 					},
+					"await": extraMetrics.AvgWriteAwaitTime,
 				},
 				"queue": common.MapStr{
 					"avg_size": extraMetrics.AvgQueueSize,

metricbeat/module/system/diskio/diskstat.go

@@ -35,13 +35,15 @@ type DiskIOMetric struct {
 	ReadRequestCountPerSec       float64 `json:"rrqCps"`
 	WriteRequestCountPerSec      float64 `json:"wrqCps"`
 	// using bytes instead of sector
-	ReadBytesPerSec  float64 `json:"rBps"`
-	WriteBytesPerSec float64 `json:"wBps"`
-	AvgRequestSize   float64 `json:"avgrqSz"`
-	AvgQueueSize     float64 `json:"avgquSz"`
-	AvgAwaitTime     float64 `json:"await"`
-	AvgServiceTime   float64 `json:"svctm"`
-	BusyPct          float64 `json:"busy"`
+	ReadBytesPerSec   float64 `json:"rBps"`
+	WriteBytesPerSec  float64 `json:"wBps"`
+	AvgRequestSize    float64 `json:"avgrqSz"`
+	AvgQueueSize      float64 `json:"avgquSz"`
+	AvgAwaitTime      float64 `json:"await"`
+	AvgReadAwaitTime  float64 `json:"r_await"`
+	AvgWriteAwaitTime float64 `json:"w_await"`
+	AvgServiceTime    float64 `json:"svctm"`
+	BusyPct           float64 `json:"busy"`
 }
 
 type DiskIOStat struct {

metricbeat/module/system/diskio/diskstat_linux.go

@@ -98,6 +98,8 @@ func (stat *DiskIOStat) CalIOStatistics(counter disk.IOCountersStat) (DiskIOMetr
 	result.AvgRequestSize = size
 	result.AvgQueueSize = queue
 	result.AvgAwaitTime = wait
+	result.AvgReadAwaitTime = float64(rd_ticks) / float64(rd_ios)
+	result.AvgWriteAwaitTime = float64(wr_ticks) / float64(wr_ios)
 	result.AvgServiceTime = svct
 	result.BusyPct = 100.0 * float64(ticks) / deltams
 	if result.BusyPct > 100.0 {

metricbeat/module/system/diskio/diskstat_linux_test.go

@@ -22,6 +22,9 @@ package diskio
 import (
 	"testing"
 
+	sigar "github.com/elastic/gosigar"
+
+	"github.com/shirou/gopsutil/disk"
 	"github.com/stretchr/testify/assert"
 
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
@@ -81,3 +84,42 @@ func TestDataEmptyFilter(t *testing.T) {
 	assert.NoError(t, err)
 	assert.Equal(t, 10, len(data))
 }
+
+func TestDiskIOStat_CalIOStatistics(t *testing.T) {
+	counter := disk.IOCountersStat{
+		ReadCount:  13,
+		WriteCount: 17,
+		ReadTime:   19,
+		WriteTime:  23,
+		Name:       "iostat",
+	}
+
+	stat := &DiskIOStat{
+		lastDiskIOCounters: map[string]disk.IOCountersStat{
+			"iostat": disk.IOCountersStat{
+				ReadCount:  3,
+				WriteCount: 5,
+				ReadTime:   7,
+				WriteTime:  11,
+				Name:       "iostat",
+			},
+		},
+		lastCpu: sigar.Cpu{Idle: 100},
+		curCpu:  sigar.Cpu{Idle: 1},
+	}
+
+	expected := DiskIOMetric{
+		AvgAwaitTime:      24.0 / 22.0,
+		AvgReadAwaitTime:  1.2,
+		AvgWriteAwaitTime: 1,
+	}
+
+	got, err := stat.CalIOStatistics(counter)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	assert.Equal(t, expected.AvgAwaitTime, got.AvgAwaitTime)
+	assert.Equal(t, expected.AvgReadAwaitTime, got.AvgReadAwaitTime)
+	assert.Equal(t, expected.AvgWriteAwaitTime, got.AvgWriteAwaitTime)
+}