Docker module cleanup

* Make default period 10s, same as for all other beats * Remove unnecessary code comments * Make convertContainerPort local to container MetricSet * Only add labels to container MetricSet if not empty * Apply some code conventions * Convert local @timestamp to timestamp to prevent confusion * Replace rx / tx with in / out in network metricset to be consistent with system network metricset * Add interface name as field instead of having it as a dynamic element. This makes it easier to filter. * If not containers are running, just no events are returned * Add data.json for all metricsets * Introduce ToMapStr for Container data * Move socket to Container MapStr. Is Socket info really needed? * Remove unnecessary nesting levels as namespace already given by metricset * Have one docker client per metricset instead of having one global client
elastic · Oct 4, 2016 · 83e7c61 · 83e7c61
1 parent 5842aee
commit 83e7c61
Show file tree

Hide file tree

Showing 33 changed files with 592 additions and 456 deletions.
diff --git a/metricbeat/docs/modules/docker.asciidoc b/metricbeat/docs/modules/docker.asciidoc
@@ -23,7 +23,7 @@ metricbeat.modules:
 #- module: docker
   #metricsets: ["cpu","memory","network","diskio","container"]
   #enabled: true
-  #period: 5s
+  #period: 10s
   #hosts: ["localhost"]
   #socket: unix:///var/run/docker.sock
 

diff --git a/metricbeat/etc/beat.full.yml b/metricbeat/etc/beat.full.yml
@@ -82,7 +82,7 @@ metricbeat.modules:
 #- module: docker
   #metricsets: ["cpu","memory","network","diskio","container"]
   #enabled: true
-  #period: 5s
+  #period: 10s
   #hosts: ["localhost"]
   #socket: unix:///var/run/docker.sock
 

diff --git a/metricbeat/metricbeat.full.yml b/metricbeat/metricbeat.full.yml
@@ -82,7 +82,7 @@ metricbeat.modules:
 #- module: docker
   #metricsets: ["cpu","memory","network","diskio","container"]
   #enabled: true
-  #period: 5s
+  #period: 10s
   #hosts: ["localhost"]
   #socket: unix:///var/run/docker.sock
 

diff --git a/metricbeat/module/docker/_meta/config.yml b/metricbeat/module/docker/_meta/config.yml
@@ -1,7 +1,7 @@
 #- module: docker
   #metricsets: ["cpu","memory","network","diskio","container"]
   #enabled: true
-  #period: 5s
+  #period: 10s
   #hosts: ["localhost"]
   #socket: unix:///var/run/docker.sock
 

diff --git a/metricbeat/module/docker/config.go b/metricbeat/module/docker/config.go
@@ -1,7 +1,8 @@
 package docker
 
 type TlsConfig struct {
-	Enabled  bool   `config:"enabled"`
+	Enabled bool `config:"enabled"`
+	// TODO: Naming should be standardised with output cert configs
 	CaPath   string `config:"ca_path"`
 	CertPath string `config:"cert_path"`
 	KeyPath  string `config:"key_path"`
@@ -12,8 +13,8 @@ type Config struct {
 	Tls    TlsConfig
 }
 
-func GetDefaultConf() *Config {
-	return &Config{
+func GetDefaultConf() Config {
+	return Config{
 		Socket: "unix:///var/run/docker.sock",
 		Tls: TlsConfig{
 			Enabled: false,

diff --git a/metricbeat/module/docker/container/_meta/data.json b/metricbeat/module/docker/container/_meta/data.json
@@ -1,19 +1,30 @@
 {
-    "@timestamp":"2016-05-23T08:05:34.853Z",
-    "beat":{
-        "hostname":"beathost",
-        "name":"beathost"
+    "@timestamp": "2016-05-23T08:05:34.853Z",
+    "beat": {
+        "hostname": "host.example.com",
+        "name": "host.example.com"
     },
-    "metricset":{
-        "host":"localhost",
-        "module":"mysql",
-        "name":"status",
-        "rtt":44269
-    },
-    "docker":{
-        "container":{
-            "example": "container"
+    "docker": {
+        "container": {
+            "command": "bash",
+            "created": "2016-10-03T19:45:33.000Z",
+            "id": "42d21fbdb065b223b8d10802dd3f8d7030e42719fbaba373f0250862835e2501",
+            "image": "debian",
+            "labels": [],
+            "name": "sleepy_montalcini",
+            "ports": [],
+            "size": {
+                "root_fs": 0,
+                "rw": 0
+            },
+            "status": "Up 10 hours"
         }
     },
-    "type":"metricsets"
-}
+    "metricset": {
+        "host": "localhost",
+        "module": "docker",
+        "name": "container",
+        "rtt": 115
+    },
+    "type": "metricsets"
+}
diff --git a/metricbeat/module/docker/container/container.go b/metricbeat/module/docker/container/container.go
@@ -10,50 +10,46 @@ import (
 	"github.com/elastic/beats/metricbeat/module/docker"
 )
 
-// init registers the MetricSet with the central registry.
-// The New method will be called after the setup of the module and before starting to fetch data
 func init() {
 	if err := mb.Registry.AddMetricSet("docker", "container", New); err != nil {
 		panic(err)
 	}
 }
 
-// MetricSet type defines all fields of the MetricSet
-// As a minimum it must inherit the mb.BaseMetricSet fields, but can be extended with
-// additional entries. These variables can be used to persist data or configuration between
-// multiple fetch calls.
 type MetricSet struct {
 	mb.BaseMetricSet
 	dockerClient *dc.Client
 }
 
-// New create a new instance of the MetricSet
-// Part of new is also setting up the configuration by processing additional
-// configuration entries if needed.
+// New create a new instance of the container MetricSet
 func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
 
 	logp.Warn("EXPERIMENTAL: The container metricset is experimental")
 
 	config := docker.GetDefaultConf()
-
 	if err := base.Module().UnpackConfig(&config); err != nil {
 		return nil, err
 	}
 
+	client, err := docker.NewDockerClient(&config)
+	if err != nil {
+		return nil, err
+	}
+
 	return &MetricSet{
 		BaseMetricSet: base,
-		dockerClient:  docker.CreateDockerCLient(config),
+		dockerClient:  client,
 	}, nil
 }
 
-// Fetch methods implements the data gathering and data conversion to the right format
-// It returns the event which is then forward to the output. In case of an error, a
-// descriptive error must be returned.
+// Fetch returns a list of all containers as events
+// This is based on https://docs.docker.com/engine/reference/api/docker_remote_api_v1.24/#/list-containers
 func (m *MetricSet) Fetch() ([]common.MapStr, error) {
 
-	containersList, err := m.dockerClient.ListContainers(dc.ListContainersOptions{})
+	// Fetch list of all containers
+	containers, err := m.dockerClient.ListContainers(dc.ListContainersOptions{})
 	if err != nil {
 		return nil, err
 	}
-	return eventsMapping(containersList), nil
+	return eventsMapping(containers), nil
 }
diff --git a/metricbeat/module/docker/container/container_integration_test.go b/metricbeat/module/docker/container/container_integration_test.go
@@ -0,0 +1,26 @@
+// +build integration
+
+package container
+
+import (
+	"testing"
+
+	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
+)
+
+func TestData(t *testing.T) {
+	f := mbtest.NewEventsFetcher(t, getConfig())
+	err := mbtest.WriteEvents(f, t)
+	if err != nil {
+		t.Fatal("write", err)
+	}
+}
+
+func getConfig() map[string]interface{} {
+	return map[string]interface{}{
+		"module":     "docker",
+		"metricsets": []string{"container"},
+		"hosts":      []string{"localhost"},
+		"socket":     "unix:///var/run/docker.sock",
+	}
+}
diff --git a/metricbeat/module/docker/container/data.go b/metricbeat/module/docker/container/data.go
@@ -6,7 +6,6 @@ import (
 	dc "github.com/fsouza/go-dockerclient"
 
 	"github.com/elastic/beats/libbeat/common"
-
 	"github.com/elastic/beats/metricbeat/module/docker"
 )
 
@@ -17,24 +16,40 @@ func eventsMapping(containersList []dc.APIContainers) []common.MapStr {
 	}
 	return myEvents
 }
-func eventMapping(mycontainer *dc.APIContainers) common.MapStr {
+
+func eventMapping(cont *dc.APIContainers) common.MapStr {
 
 	event := common.MapStr{
-		"@timestamp": time.Now(),
-		"container": common.MapStr{
-			"created":      common.Time(time.Unix(mycontainer.Created, 0)),
-			"id":           mycontainer.ID,
-			"name":         docker.ExtractContainerName(mycontainer.Names),
-			"labels":       docker.BuildLabelArray(mycontainer.Labels),
-			"command":      mycontainer.Command,
-			"image":        mycontainer.Image,
-			"ports":        docker.ConvertContainerPorts(&mycontainer.Ports),
-			"size_root_fs": mycontainer.SizeRootFs,
-			"size_rw":      mycontainer.SizeRw,
-			"status":       mycontainer.Status,
+		"created": common.Time(time.Unix(cont.Created, 0)),
+		"id":      cont.ID,
+		"name":    docker.ExtractContainerName(cont.Names),
+		"command": cont.Command,
+		"image":   cont.Image,
+		"ports":   convertContainerPorts(cont.Ports),
+		"labels":  docker.BuildLabelArray(cont.Labels),
+		"size": common.MapStr{
+			"root_fs": cont.SizeRootFs,
+			"rw":      cont.SizeRw,
 		},
-		"socket": docker.GetSocket(),
+		"status": cont.Status,
 	}
 
 	return event
 }
+
+func convertContainerPorts(ports []dc.APIPort) []map[string]interface{} {
+	var outputPorts = []map[string]interface{}{}
+	for _, port := range ports {
+		outputPort := common.MapStr{
+			"ip": port.IP,
+			"port": common.MapStr{
+				"private": port.PrivatePort,
+				"public":  port.PublicPort,
+			},
+			"type": port.Type,
+		}
+		outputPorts = append(outputPorts, outputPort)
+	}
+
+	return outputPorts
+}
diff --git a/metricbeat/module/docker/cpu/_meta/data.json b/metricbeat/module/docker/cpu/_meta/data.json
@@ -1,19 +1,32 @@
 {
-    "@timestamp":"2016-05-23T08:05:34.853Z",
-    "beat":{
-        "hostname":"beathost",
-        "name":"beathost"
+    "@timestamp": "2016-05-23T08:05:34.853Z",
+    "beat": {
+        "hostname": "host.example.com",
+        "name": "host.example.com"
     },
-    "metricset":{
-        "host":"localhost",
-        "module":"mysql",
-        "name":"status",
-        "rtt":44269
-    },
-    "docker":{
-        "cpu":{
-            "example": "cpu"
+    "docker": {
+        "cpu": {
+            "container": {
+                "id": "42d21fbdb065b223b8d10802dd3f8d7030e42719fbaba373f0250862835e2501",
+                "name": "sleepy_montalcini",
+                "socket": "unix:///var/run/docker.sock"
+            },
+            "usage": {
+                "kernel_mode": 0,
+                "per_cpu": {
+                    "0": 0,
+                    "1": 0
+                },
+                "total": 0,
+                "user_mode": 0
+            }
         }
     },
-    "type":"metricsets"
-}
+    "metricset": {
+        "host": "localhost",
+        "module": "docker",
+        "name": "cpu",
+        "rtt": 115
+    },
+    "type": "metricsets"
+}
diff --git a/metricbeat/module/docker/cpu/cpu.go b/metricbeat/module/docker/cpu/cpu.go
@@ -9,27 +9,19 @@ import (
 	"github.com/elastic/beats/metricbeat/module/docker"
 )
 
-// init registers the MetricSet with the central registry.
-// The New method will be called after the setup of the module and before starting to fetch data
 func init() {
 	if err := mb.Registry.AddMetricSet("docker", "cpu", New); err != nil {
 		panic(err)
 	}
 }
 
-// MetricSet type defines all fields of the MetricSet
-// As a minimum it must inherit the mb.BaseMetricSet fields, but can be extended with
-// additional entries. These variables can be used to persist data or configuration between
-// multiple fetch calls.
 type MetricSet struct {
 	mb.BaseMetricSet
 	cpuService   *CPUService
 	dockerClient *dc.Client
 }
 
-// New create a new instance of the MetricSet
-// Part of new is also setting up the configuration by processing additional
-// configuration entries if needed.
+// New create a new instance of the docker cpu MetricSet
 func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
 
 	logp.Warn("EXPERIMENTAL: The cpu metricset is experimental")
@@ -38,24 +30,28 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
 	if err := base.Module().UnpackConfig(&config); err != nil {
 		return nil, err
 	}
+
+	client, err := docker.NewDockerClient(&config)
+	if err != nil {
+		return nil, err
+	}
+
 	return &MetricSet{
 		BaseMetricSet: base,
-		dockerClient:  docker.CreateDockerCLient(config),
+		dockerClient:  client,
 		cpuService:    &CPUService{},
 	}, nil
 }
 
-// Fetch methods implements the data gathering and data conversion to the right format
-// It returns the event which is then forward to the output. In case of an error, a
-// descriptive error must be returned.
+// Fetch returns a list of docker cpu stats
 func (m *MetricSet) Fetch() ([]common.MapStr, error) {
 
-	rawStats, err := docker.FetchDockerStats(m.dockerClient)
-
+	stats, err := docker.FetchStats(m.dockerClient)
 	if err != nil {
 		return nil, err
 	}
-	formatedStats := m.cpuService.GetCPUStatsList(rawStats)
+
+	formatedStats := m.cpuService.getCPUStatsList(stats)
 	return eventsMapping(formatedStats), nil
 
 }
diff --git a/metricbeat/module/docker/cpu/cpu_integration_test.go b/metricbeat/module/docker/cpu/cpu_integration_test.go
@@ -0,0 +1,26 @@
+// +build integration
+
+package cpu
+
+import (
+	"testing"
+
+	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
+)
+
+func TestData(t *testing.T) {
+	f := mbtest.NewEventsFetcher(t, getConfig())
+	err := mbtest.WriteEvents(f, t)
+	if err != nil {
+		t.Fatal("write", err)
+	}
+}
+
+func getConfig() map[string]interface{} {
+	return map[string]interface{}{
+		"module":     "docker",
+		"metricsets": []string{"cpu"},
+		"hosts":      []string{"localhost"},
+		"socket":     "unix:///var/run/docker.sock",
+	}
+}