Generate Kibana index pattern from etc/kibana/fields.yml (#2122)

The format of each field can be specified in fields.yml using the `format` option.

Generate Kibana index pattern for all the elastic beats by running `make update`

CHANGELOG.asciidoc

@@ -44,6 +44,7 @@ https://github.com/elastic/beats/compare/v5.0.0-alpha5...master[Check the HEAD d
 ==== Added
 
 *Affecting all Beats*
+- Add script to generate the Kibana index-pattern from fields.yml. {pull}2122[2122]
 
 *Metricbeat*
 

filebeat/etc/kibana/index-pattern/filebeat.json

@@ -1,5 +1 @@
-{
-  "fields": "[{\"name\":\"offset\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"_index\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":false,\"analyzed\":false,\"doc_values\":false},{\"name\":\"line\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":false},{\"name\":\"message\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"_source\",\"type\":\"_source\",\"count\":0,\"scripted\":false,\"indexed\":false,\"analyzed\":false,\"doc_values\":false},{\"name\":\"_id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":false,\"analyzed\":false,\"doc_values\":false},{\"name\":\"@timestamp\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":false},{\"name\":\"beat.name\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"count\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"source\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true}]", 
-  "timeFieldName": "@timestamp", 
-  "title": "filebeat-*"
-}
+{"fields": "[{\"count\": 0, \"name\": \"beat.name\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"beat.hostname\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"@timestamp\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"date\", \"scripted\": false}, {\"count\": 0, \"name\": \"tags\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"fields\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"dict\", \"scripted\": false}, {\"count\": 0, \"name\": \"source\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"keyword\", \"scripted\": false}, {\"count\": 0, \"name\": \"offset\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"name\": \"message\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"text\", \"scripted\": false}, {\"count\": 0, \"name\": \"type\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"input_type\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}]", "fieldFormatMap": "{\"@timestamp\": {\"id\": \"YYYY-MM-DDTHH:MM:SS.milliZ\"}}", "timeFieldName": "@timestamp", "title": "filebeat-*"}

libbeat/scripts/Makefile

@@ -239,6 +239,10 @@ update: python-env
 	# Update docs version
 	cp ${ES_BEATS}/libbeat/docs/version.asciidoc docs/version.asciidoc
 
+	# Generate index-pattern
+	. ${PYTHON_ENV}/bin/activate && python ${ES_BEATS}/libbeat/scripts/generate_index_pattern.py --index ${BEATNAME}-* --libbeat ${ES_BEATS}/libbeat --beat $(PWD)
+
+
 # Builds the documents for the beat
 .PHONY: docs
 docs:

libbeat/scripts/generate_index_pattern.py

@@ -0,0 +1,115 @@
+#!/usr/bin/env python
+
+"""
+This script generates the index-pattern for Kibana from
+the fields.yml file.
+"""
+
+import yaml
+import argparse
+import string
+import re
+import json
+
+
+def fields_to_json(section, path, output):
+
+    for field in section["fields"]:
+        if path == "":
+            newpath = field["name"]
+        else:
+            newpath = path + "." + field["name"]
+
+        if "type" in field and field["type"] == "group":
+            fields_to_json(field, newpath, output)
+        else:
+            field_to_json(field, newpath, output)
+
+
+def field_to_json(desc, path, output):
+
+    field = {
+        "name": path,
+        "count": 0,
+        "scripted": False,
+        "indexed": True,
+        "analyzed": False,
+        "doc_values": True,
+    }
+    if "type" in desc:
+        if desc["type"] in ["half_float", "float", "integer", "long"]:
+            field["type"] = "number"
+        else:
+            field["type"] = desc["type"]
+    else:
+        field["type"] = "string"
+
+    output["fields"].append(field)
+
+    if "format" in desc:
+        output["fieldFormatMap"][path] = {
+            "id": desc["format"],
+        }
+
+
+def fields_to_index_pattern(args, input):
+
+    docs = yaml.load(input)
+
+    if docs is None:
+        print("fields.yml is empty. Cannot generate index-pattern")
+        return
+
+    output = {
+        "fields": [],
+        "fieldFormatMap": {},
+        "timeFieldName": "@timestamp",
+        "title": args.index,
+
+    }
+
+    for k, section in enumerate(docs["fields"]):
+        fields_to_json(section, "", output)
+
+    output["fields"] = json.dumps(output["fields"])
+    output["fieldFormatMap"] = json.dumps(output["fieldFormatMap"])
+    return output
+
+
+def get_index_pattern_name(index):
+
+    allow = string.letters + string.digits + "_"
+    return re.sub('[^%s]' % allow, '', index)
+
+
+if __name__ == "__main__":
+
+    parser = argparse.ArgumentParser(
+        description="Generates the index-pattern for a Beat.")
+    parser.add_argument("--index", help="The name of the index-pattern")
+    parser.add_argument("--beat", help="Local Beat directory")
+    parser.add_argument("--libbeat", help="Libbeat local directory")
+
+    args = parser.parse_args()
+
+    # generate the index-pattern content
+    with open(args.beat + "/etc/fields.yml", 'r') as f:
+        fields = f.read()
+
+        # Prepend beat fields from libbeat
+        with open(args.libbeat + "/_meta/fields.yml") as f:
+            fields = f.read() + fields
+
+        # with open(target, 'w') as output:
+        output = fields_to_index_pattern(args, fields)
+
+    # dump output to a json file
+    fileName = get_index_pattern_name(args.index)
+    target = args.beat + "/etc/kibana/index-pattern/" + fileName + ".json"
+
+    output = json.dumps(output)
+
+    with open(target, 'w') as f:
+        f.write(output)
+
+    print "The index pattern was created under {}".format(target)