Remove defaults from fields.common.yml for more flexibility

Having the defaults in code instead of the libbeat fields.yml will allow to reuse the same script also for fields.yml files from a metricbeat module for example without having to merge in the libbeat file. This is a prerequisite for dynamic index template generation in Golang which can be only for a module.

The top level `fields` entry was removed as without the defaults, it is not needed anymore.

All scripts were adjusted for the new format.

libbeat/_meta/fields.common.yml

@@ -1,11 +1,4 @@
-defaults:
-  type: keyword
-  required: false
-  index: true
-  doc_values: true
-  ignore_above: 1024
 
-fields:
 - key: beat
   title: Beat
   description: >

libbeat/scripts/generate_fields_docs.py

@@ -84,15 +84,15 @@ def fields_to_asciidoc(input, output, beat):
 
     # Create sections from available fields
     sections = {}
-    for v in docs["fields"]:
+    for v in docs:
         sections[v["key"]] = v["title"]
 
     for key in sorted(sections):
         output.write("* <<exported-fields-{}>>\n".format(key))
     output.write("\n--\n")
 
     # Sort alphabetically by key
-    for section in sorted(docs["fields"], key=lambda field: field["key"]):
+    for section in sorted(docs, key=lambda field: field["key"]):
         section["name"] = section["title"]
         section["anchor"] = section["key"]
         document_fields(output, section, sections, "")

libbeat/scripts/generate_index_pattern.py

@@ -97,7 +97,7 @@ def fields_to_index_pattern(args, input):
 
     }
 
-    for k, section in enumerate(docs["fields"]):
+    for k, section in enumerate(docs):
         fields_to_json(section, "", output)
 
     # add meta fields
@@ -149,9 +149,10 @@ def get_index_pattern_name(index):
     with open(fields_yml, 'r') as f:
         fields = f.read()
 
-        # Prepend beat fields from libbeat
-        with open(args.libbeat + "/_meta/fields.generated.yml") as f:
-            fields = f.read() + fields
+        if os.path.basename(args.beat) != "libbeat":
+            # Prepend beat fields from libbeat
+            with open(args.libbeat + "/_meta/fields.generated.yml") as f:
+                fields = f.read() + fields
 
         # with open(target, 'w') as output:
         output = fields_to_index_pattern(args, fields)

libbeat/scripts/generate_template.py

@@ -30,16 +30,16 @@ def fields_to_es_template(args, input, output, index, version):
         print("fields.yml is empty. Cannot generate template.")
         return
 
-    # Each template needs defaults
-    if "defaults" not in docs.keys():
-        print("No defaults are defined. Each template needs at" +
-              " least defaults defined.")
-        return
-
-    defaults = docs["defaults"]
+    defaults = {
+        "type": "keyword",
+        "required": False,
+        "index": True,
+        "doc_values": True,
+        "ignore_above": 1024,
+    }
 
-    for k, section in enumerate(docs["fields"]):
-        docs["fields"][k] = dedot(section)
+    for k, section in enumerate(docs):
+        docs[k] = dedot(section)
 
     # skeleton
     template = {
@@ -105,9 +105,8 @@ def fields_to_es_template(args, input, output, index, version):
             }
         })
 
-    for section in docs["fields"]:
-        prop, dynamic = fill_section_properties(args, section,
-                                                defaults, "")
+    for section in docs:
+        prop, dynamic = fill_section_properties(args, section, defaults, "")
         properties.update(prop)
         dynamic_templates.extend(dynamic)
 

libbeat/tests/system/beat/beat.py

@@ -429,7 +429,7 @@ def extract_fields(doc_list, name):
             fields = []
             dictfields = []
 
-            for item in doc["fields"]:
+            for item in doc:
                 subfields, subdictfields = extract_fields(item["fields"], "")
                 fields.extend(subfields)
                 dictfields.extend(subdictfields)