Add Elasticsearch 5.x output

Godeps

@@ -63,4 +63,5 @@ golang.org/x/text 506f9d5c962f284575e88337e7d9296d27e729d3
 gopkg.in/dancannon/gorethink.v1 edc7a6a68e2d8015f5ffe1b2560eed989f8a45be
 gopkg.in/fatih/pool.v2 6e328e67893eb46323ad06f0e92cb9536babbabc
 gopkg.in/mgo.v2 3f83fa5005286a7fe593b055f0d7771a7dce4655
+gopkg.in/olivere/elastic.v5 ee3ebceab960cf68ab9a89ee6d78c031ef5b4a4e
 gopkg.in/yaml.v2 4c78c975fe7c825c6d1466c42be594d1d6f3aba6

Makefile

@@ -51,6 +51,7 @@ docker-run:
 		-e ADVERTISED_PORT=9092 \
 		-p "2181:2181" -p "9092:9092" \
 		-d spotify/kafka
+	docker run --name elasticsearch -p "9200:9200" -p "9300:9300" -d elasticsearch:5
 	docker run --name mysql -p "3306:3306" -e MYSQL_ALLOW_EMPTY_PASSWORD=yes -d mysql
 	docker run --name memcached -p "11211:11211" -d memcached
 	docker run --name postgres -p "5432:5432" -d postgres
@@ -69,15 +70,16 @@ docker-run-circle:
 		-e ADVERTISED_PORT=9092 \
 		-p "2181:2181" -p "9092:9092" \
 		-d spotify/kafka
+	docker run --name elasticsearch -p "9200:9200" -p "9300:9300" -d elasticsearch:5
 	docker run --name nsq -p "4150:4150" -d nsqio/nsq /nsqd
 	docker run --name mqtt -p "1883:1883" -d ncarlier/mqtt
 	docker run --name riemann -p "5555:5555" -d stealthly/docker-riemann
 	docker run --name nats -p "4222:4222" -d nats
 
 # Kill all docker containers, ignore errors
 docker-kill:
-	-docker kill nsq aerospike redis rabbitmq postgres memcached mysql kafka mqtt riemann nats
-	-docker rm nsq aerospike redis rabbitmq postgres memcached mysql kafka mqtt riemann nats
+	-docker kill nsq aerospike redis rabbitmq postgres memcached mysql kafka mqtt riemann nats elasticsearch
+	-docker rm nsq aerospike redis rabbitmq postgres memcached mysql kafka mqtt riemann nats elasticsearch
 
 # Run full unit tests using docker containers (includes setup and teardown)
 test: vet docker-kill docker-run

README.md

@@ -209,6 +209,7 @@ Telegraf can also collect metrics via the following service plugins:
 * [aws cloudwatch](./plugins/outputs/cloudwatch)
 * [datadog](./plugins/outputs/datadog)
 * [discard](./plugins/outputs/discard)
+* [elasticsearch](./plugins/outputs/elasticsearch)
 * [file](./plugins/outputs/file)
 * [graphite](./plugins/outputs/graphite)
 * [graylog](./plugins/outputs/graylog)

plugins/outputs/all/all.go

@@ -6,6 +6,7 @@ import (
 	_ "github.com/influxdata/telegraf/plugins/outputs/cloudwatch"
 	_ "github.com/influxdata/telegraf/plugins/outputs/datadog"
 	_ "github.com/influxdata/telegraf/plugins/outputs/discard"
+	_ "github.com/influxdata/telegraf/plugins/outputs/elasticsearch"
 	_ "github.com/influxdata/telegraf/plugins/outputs/file"
 	_ "github.com/influxdata/telegraf/plugins/outputs/graphite"
 	_ "github.com/influxdata/telegraf/plugins/outputs/graylog"

plugins/outputs/elasticsearch/README.md

@@ -0,0 +1,178 @@
+## Elasticsearch Output Plugin for Telegraf
+
+This plugin writes to [Elasticsearch](https://www.elastic.co) via HTTP using Elastic (http://olivere.github.io/elastic/).
+
+It only supports Elasticsearch 5.x series currently.
+
+## Elasticsearch indexes and templates
+
+### Indexes per time-frame
+
+This plugin can manage indexes per time-frame, as commonly done in other tools with Elasticsearch.
+
+The timestamp of the metric collected will be used to decide the index destination.
+
+For more information about this usage on Elasticsearch, check https://www.elastic.co/guide/en/elasticsearch/guide/master/time-based.html#index-per-timeframe
+
+### Template management
+
+Index templates are used in Elasticsearch to define settings and mappings for the indexes and how the fields should be analyzed.
+For more information on how this works, see https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-templates.html
+
+This plugin can create a working template for use with telegraf metrics. It uses Elasticsearch dynamic templates feature to set proper types for the tags and metrics fields.
+If the template specified already exists, it will not overwrite unless you configure this plugin to do so.
+
+Example of an index template created by telegraf:
+
+```json
+{
+  "order": 0,
+  "template": "telegraf-*",
+  "mappings": {
+    "_default_": {
+      "dynamic_templates": [
+        {
+          "tags": {
+            "path_match": "tag.*",
+            "mapping": {
+              "ignore_above": 512,
+              "type": "keyword"
+            },
+            "match_mapping_type": "string"
+          }
+        },
+        {
+          "metrics": {
+            "mapping": {
+              "index": false,
+              "type": "float"
+            },
+            "match_mapping_type": "long"
+          }
+        }
+      ],
+      "_all": {
+        "enabled": false
+      },
+      "properties": {
+        "input_plugin": {
+          "type": "keyword"
+        },
+        "@timestamp": {
+          "type": "date"
+        }
+      }
+    }
+  }
+}
+
+```
+
+### Example events:
+
+This plugin will format the events in the following way:
+
+```json
+{
+  "@timestamp": "2017-01-01T00:00:00+00:00",
+  "input_plugin": "cpu",
+  "cpu": {
+    "usage_guest": 0,
+    "usage_guest_nice": 0,
+    "usage_idle": 71.85413456197966,
+    "usage_iowait": 0.256805341656516,
+    "usage_irq": 0,
+    "usage_nice": 0,
+    "usage_softirq": 0.2054442732579466,
+    "usage_steal": 0,
+    "usage_system": 15.04879301548127,
+    "usage_user": 12.634822807288275
+  },
+  "tag": {
+    "cpu": "cpu-total",
+    "host": "elastichost",
+    "dc": "datacenter1"
+  }
+}
+```
+
+```json
+{
+  "@timestamp": "2017-01-01T00:00:00+00:00",
+  "input_plugin": "system",
+  "system": {
+    "load1": 0.78,
+    "load15": 0.8,
+    "load5": 0.8,
+    "n_cpus": 2,
+    "n_users": 2
+  },
+  "tag": {
+    "host": "elastichost",
+    "dc": "datacenter1"
+  }
+}
+```
+
+### Configuration:
+
+```toml
+# Configuration for Elasticsearch to send metrics to.
+[[outputs.elasticsearch]]
+  ## The full HTTP endpoint URL for your Elasticsearch instance
+  ## Multiple urls can be specified as part of the same cluster,
+  ## this means that only ONE of the urls will be written to each interval.
+  urls = [ "http://node1.es.example.com:9200" ] # required.
+  ## Set to true to ask Elasticsearch a list of all cluster nodes,
+  ## thus it is not necessary to list all nodes in the urls config option
+  enable_sniffer = true
+  ## Set the interval to check if the nodes are available, in seconds.
+  ## Setting to 0 will disable the health check (not recommended in production)
+  health_check_interval = 10
+  ## HTTP basic authentication details (eg. when using Shield)
+  # username = "telegraf"
+  # password = "mypassword"
+
+  # Index Config
+  ## The target index for metrics (Elasticsearch will create if it not exists).
+  ## You can use the date specifiers below to create indexes per time frame.
+  ## The metric timestamp will be used to decide the destination index name
+  # %Y - year (2016)
+  # %y - last two digits of year (00..99)
+  # %m - month (01..12)
+  # %d - day of month (e.g., 01)
+  # %H - hour (00..23)
+  index_name = "telegraf-%Y.%m.%d" # required.
+
+  ## Template Config
+  ## Set to true if you want telegraf to manage its index template.
+  ## If enabled it will create a recommended index template for telegraf indexes
+  manage_template = true
+  ## The template name used for telegraf indexes
+  template_name = "telegraf"
+  ## Set to true if you want to overwrite an existing template
+  overwrite_template = false
+```
+
+### Required parameters:
+
+* `urls`: A list containing the full HTTP URL of one or more nodes from your Elasticsearch instance.
+* `index_name`: The target index for metrics. You can use the date specifiers below to create indexes per time frame.
+
+``` 
+  %Y - year (2017)
+  %y - last two digits of year (00..99)
+  %m - month (01..12)
+  %d - day of month (e.g., 01)
+  %H - hour (00..23)
+```
+
+### Optional parameters:
+
+* `enable_sniffer`: Set to true to ask Elasticsearch a list of all cluster nodes, thus it is not necessary to list all nodes in the urls config option.
+* `health_check_interval`: Set the interval to check if the nodes are available, in seconds. Setting to 0 will disable the health check (not recommended in production).
+* `username`: The username for HTTP basic authentication details (eg. when using Shield).
+* `password`: The password for HTTP basic authentication details (eg. when using Shield).
+* `manage_template`: Set to true if you want telegraf to manage its index template. If enabled it will create a recommended index template for telegraf indexes.
+* `template_name`: The template name used for telegraf indexes.
+* `overwrite_template`: Set to true if you want to overwrite an existing template.