Metricbeat: Add munin module (#6473) (#6517)

Add munin module, with node metricset that obtain metrics
from a running munin node using the same protocol used by
munin masters.

CHANGELOG.asciidoc

@@ -251,6 +251,7 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di
 - Making the jolokia/jmx module GA. {pull}6143[6143]
 - Making the MongoDB module GA. {pull}6554[6554]
 - Allow to disable labels `dedot` in Docker module, in favor of a safe way to keep dots. {pull}6490[6490]
+- Add experimental module to collect metrics from munin nodes. {pull}6517[6517]
 
 *Packetbeat*
 

metricbeat/docker-compose.yml

@@ -27,6 +27,7 @@ services:
       - ./module/logstash/_meta/env
       - ./module/memcached/_meta/env
       - ./module/mongodb/_meta/env
+      - ./module/munin/_meta/env
       - ./module/mysql/_meta/env
       - ./module/nginx/_meta/env
       - ./module/php_fpm/_meta/env
@@ -107,6 +108,9 @@ services:
   mongodb:
     build: ./module/mongodb/_meta
 
+  munin:
+    build: ./module/munin/_meta
+
   mysql:
     build: ./module/mysql/_meta
 

metricbeat/docs/fields.asciidoc

@@ -37,6 +37,7 @@ grouped in the following categories:
 * <<exported-fields-logstash>>
 * <<exported-fields-memcached>>
 * <<exported-fields-mongodb>>
+* <<exported-fields-munin>>
 * <<exported-fields-mysql>>
 * <<exported-fields-nginx>>
 * <<exported-fields-php_fpm>>
@@ -7656,6 +7657,21 @@ type: long
 Number of sync operations.
 
 
+[[exported-fields-munin]]
+== Munin fields
+
+experimental[]
+Munin node metrics exporter
+
+
+
+[float]
+== munin fields
+
+munin contains metrics exposed by a munin node agent
+
+
+
 [[exported-fields-mysql]]
 == MySQL fields
 

metricbeat/docs/modules/munin.asciidoc

@@ -0,0 +1,41 @@
+////
+This file is generated! See scripts/docs_collector.py
+////
+
+[[metricbeat-module-munin]]
+== Munin module
+
+experimental[]
+
+== munin module
+
+This is the munin module.
+
+
+
+[float]
+=== Example configuration
+
+The Munin module supports the standard configuration options that are described
+in <<configuration-metricbeat>>. Here is an example configuration:
+
+[source,yaml]
+----
+metricbeat.modules:
+- module: munin
+  metricsets: ["node"]
+  enabled: false
+  period: 10s
+  hosts: ["localhost:4949"]
+  node.namespace: node
+----
+
+[float]
+=== Metricsets
+
+The following metricsets are available:
+
+* <<metricbeat-metricset-munin-node,node>>
+
+include::munin/node.asciidoc[]
+

metricbeat/docs/modules/munin/node.asciidoc

@@ -0,0 +1,17 @@
+////
+This file is generated! See scripts/docs_collector.py
+////
+
+[[metricbeat-metricset-munin-node]]
+=== Munin node metricset
+
+experimental[]
+
+include::../../../module/munin/node/_meta/docs.asciidoc[]
+
+
+==== Fields
+
+For a description of each field in the metricset, see the
+<<exported-fields-munin,exported fields>> section.
+

metricbeat/docs/modules_list.asciidoc

@@ -79,6 +79,8 @@ This file is generated! See scripts/docs_collector.py
 .3+|   |<<metricbeat-metricset-mongodb-collstats,collstats>>   
 |<<metricbeat-metricset-mongodb-dbstats,dbstats>>   
 |<<metricbeat-metricset-mongodb-status,status>>   
+|<<metricbeat-module-munin,Munin>>  experimental[]   |    
+.1+|   |<<metricbeat-metricset-munin-node,node>> experimental[]  
 |<<metricbeat-module-mysql,MySQL>>     |    
 .1+|   |<<metricbeat-metricset-mysql-status,status>>   
 |<<metricbeat-module-nginx,Nginx>>     |    
@@ -147,6 +149,7 @@ include::modules/kubernetes.asciidoc[]
 include::modules/logstash.asciidoc[]
 include::modules/memcached.asciidoc[]
 include::modules/mongodb.asciidoc[]
+include::modules/munin.asciidoc[]
 include::modules/mysql.asciidoc[]
 include::modules/nginx.asciidoc[]
 include::modules/php_fpm.asciidoc[]

metricbeat/include/list.go

@@ -83,6 +83,8 @@ import (
 	_ "github.com/elastic/beats/metricbeat/module/mongodb/collstats"
 	_ "github.com/elastic/beats/metricbeat/module/mongodb/dbstats"
 	_ "github.com/elastic/beats/metricbeat/module/mongodb/status"
+	_ "github.com/elastic/beats/metricbeat/module/munin"
+	_ "github.com/elastic/beats/metricbeat/module/munin/node"
 	_ "github.com/elastic/beats/metricbeat/module/mysql"
 	_ "github.com/elastic/beats/metricbeat/module/mysql/status"
 	_ "github.com/elastic/beats/metricbeat/module/nginx"

metricbeat/metricbeat.reference.yml

@@ -349,6 +349,14 @@ metricbeat.modules:
   # Password to use when connecting to MongoDB. Empty by default.
   #password: pass
 
+#-------------------------------- Munin Module -------------------------------
+- module: munin
+  metricsets: ["node"]
+  enabled: false
+  period: 10s
+  hosts: ["localhost:4949"]
+  node.namespace: node
+
 #-------------------------------- MySQL Module -------------------------------
 - module: mysql
   metricsets: ["status"]

metricbeat/module/munin/_meta/Dockerfile

@@ -0,0 +1,13 @@
+FROM ubuntu:16.04
+
+RUN apt-get update && \
+    apt-get install -y munin-node netcat && \
+    apt-get clean && rm rm -rf /var/lib/apt/lists/*
+
+EXPOSE 4949
+
+COPY munin-node.conf /etc/munin/munin-node.conf
+
+HEALTHCHECK --interval=1s --retries=90 CMD nc -z 127.0.0.1 4949
+
+CMD munin-node

metricbeat/module/munin/_meta/config.yml

@@ -0,0 +1,6 @@
+- module: munin
+  metricsets: ["node"]
+  enabled: false
+  period: 10s
+  hosts: ["localhost:4949"]
+  node.namespace: node

metricbeat/module/munin/_meta/docs.asciidoc

@@ -0,0 +1,4 @@
+== munin module
+
+This is the munin module.
+

metricbeat/module/munin/_meta/env

@@ -0,0 +1,2 @@
+MUNIN_HOST=munin
+MUNIN_PORT=4949

metricbeat/module/munin/_meta/fields.yml

@@ -0,0 +1,13 @@
+- key: munin
+  title: "Munin"
+  description: >
+    experimental[]
+
+    Munin node metrics exporter
+  release: experimental
+  fields:
+    - name: munin
+      type: group
+      description: >
+          munin contains metrics exposed by a munin node agent
+      fields:

metricbeat/module/munin/_meta/munin-node.conf

@@ -0,0 +1,15 @@
+setsid 0
+
+ignore_file [\#~]$
+ignore_file DEADJOE$
+ignore_file \.bak$
+ignore_file %$
+ignore_file \.dpkg-(tmp|new|old|dist)$
+ignore_file \.rpm(save|new)$
+ignore_file \.pod$
+
+allow .*
+
+host 0.0.0.0
+
+port 4949

metricbeat/module/munin/doc.go

@@ -0,0 +1,2 @@
+// Package munin is a Metricbeat module that contains MetricSets.
+package munin

metricbeat/module/munin/munin.go

@@ -0,0 +1,115 @@
+package munin
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"net"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/joeshaw/multierror"
+	"github.com/pkg/errors"
+
+	"github.com/elastic/beats/libbeat/common"
+)
+
+const (
+	unknownValue = "U"
+)
+
+// Node connection
+type Node struct {
+	conn net.Conn
+
+	writer io.Writer
+	reader *bufio.Reader
+}
+
+// Connect with a munin node
+func Connect(address string, timeout time.Duration) (*Node, error) {
+	conn, err := net.DialTimeout("tcp", address, timeout)
+	if err != nil {
+		return nil, err
+	}
+	n := &Node{conn: conn,
+		writer: conn,
+		reader: bufio.NewReader(conn),
+	}
+	// Cosume and ignore first line returned by munin, it is a comment
+	// about the node
+	scanner := bufio.NewScanner(n.reader)
+	scanner.Scan()
+	return n, scanner.Err()
+}
+
+// Close node connection relasing its resources
+func (n *Node) Close() error {
+	return n.conn.Close()
+}
+
+// List of items exposed by the node
+func (n *Node) List() ([]string, error) {
+	_, err := io.WriteString(n.writer, "list\n")
+	if err != nil {
+		return nil, err
+	}
+
+	scanner := bufio.NewScanner(n.reader)
+	scanner.Scan()
+	return strings.Fields(scanner.Text()), scanner.Err()
+}
+
+// Fetch metrics from munin node
+func (n *Node) Fetch(items ...string) (common.MapStr, error) {
+	var errs multierror.Errors
+	event := common.MapStr{}
+
+	for _, item := range items {
+		_, err := io.WriteString(n.writer, "fetch "+item+"\n")
+		if err != nil {
+			errs = append(errs, err)
+			continue
+		}
+
+		scanner := bufio.NewScanner(n.reader)
+		scanner.Split(bufio.ScanWords)
+		for scanner.Scan() {
+			name := strings.TrimSpace(scanner.Text())
+
+			// Munin delimites metrics with a dot
+			if name == "." {
+				break
+			}
+
+			name = strings.TrimSuffix(name, ".value")
+
+			if !scanner.Scan() {
+				if scanner.Err() == nil {
+					errs = append(errs, errors.New("unexpected EOF when expecting value"))
+				}
+				break
+			}
+			value := scanner.Text()
+
+			key := fmt.Sprintf("%s.%s", item, name)
+
+			if value == unknownValue {
+				errs = append(errs, errors.Errorf("unknown value for %s", key))
+				continue
+			}
+			if f, err := strconv.ParseFloat(value, 64); err == nil {
+				event.Put(key, f)
+				continue
+			}
+			event.Put(key, value)
+		}
+
+		if scanner.Err() != nil {
+			errs = append(errs, scanner.Err())
+		}
+	}
+
+	return event, errs.Err()
+}