Add --oss flag to the ECS generator script

CHANGELOG.next.md

@@ -42,6 +42,7 @@ Thanks, you're awesome :-) -->
 * Introduced `--strict` flag to perform stricter schema validation when running the generator script. #937
 * Added check under `--strict` that ensures composite types in example fields are quoted. #966
 * Added `ignore_above` and `normalizer` support for keyword multi-fields. #971
+* Added `--oss` flag for users who want to generate ECS templates for use on OSS clusters. #991
 
 #### Improvements
 

USAGE.md

@@ -29,6 +29,7 @@ relevant artifacts for their unique set of data sources.
     + [Subset](#subset)
     + [Ref](#ref)
     + [Mapping & Template Settings](#mapping--template-settings)
+    + [OSS](#oss)
     + [Strict Mode](#strict-mode)
     + [Intermediate-Only](#intermediate-only)
 
@@ -295,14 +296,33 @@ The `--template-settings` argument defines [index level settings](https://www.el
 
 For `template.json`, the `mappings` object is left empty: `{}`. Likewise the `properties` object remains empty in the `mapping.json` example. This will be filled in automatically by the script.
 
+#### OSS
+
+ECS now uses basic licensed types that are not available on OSS Elasticsearch clusters. Some of these types have an OSS replacement that can be used instead, without too much loss of functionality.
+
+This flag performs a best effort fallback, replacing basic data types with their OSS replacement.
+
+Indices using purely OSS types will benefit from the normalization of ECS, but may be missing on some of the added functionality of basic types.
+
+Current fallbacks applied by this flag are:
+
+- `wildcard` => `keyword`
+- `version` => `keyword`
+
+Usage:
+
+```
+$ python scripts/generator.py --oss
+```
+
 #### Strict Mode
 
 The `--strict` argument enables "strict mode". Strict mode performs a stricter validation step against the schema's contents.
 
 Basic usage:
 
 ```
-$ python/generator.py --strict
+$ python scripts/generator.py --strict
 ```
 
 Strict mode requires the following conditions, else the script exits on an exception:

scripts/generator.py

@@ -12,6 +12,7 @@
 from generators import intermediate_files
 
 from schema import loader
+from schema import oss
 from schema import cleaner
 from schema import finalizer
 from schema import subset_filter
@@ -41,6 +42,8 @@ def main():
     # ecs_helpers.yaml_dump('ecs.yml', fields)
 
     fields = loader.load_schemas(ref=args.ref, included_files=args.include)
+    if args.oss:
+        oss.fallback(fields)
     cleaner.clean(fields, strict=args.strict)
     finalizer.finalize(fields)
     fields = subset_filter.filter(fields, args.subset, out_dir)
@@ -60,20 +63,21 @@ def main():
 
 def argument_parser():
     parser = argparse.ArgumentParser()
-    parser.add_argument('--intermediate-only', action='store_true',
-                        help='generate intermediary files only')
+    parser.add_argument('--ref', action='store', help='git reference to use when building schemas')
     parser.add_argument('--include', nargs='+',
                         help='include user specified directory of custom field definitions')
     parser.add_argument('--subset', nargs='+',
                         help='render a subset of the schema')
-    parser.add_argument('--out', action='store', help='directory to store the generated files')
-    parser.add_argument('--ref', action='store', help='git reference to use when building schemas')
+    parser.add_argument('--out', action='store', help='directory to output the generated files')
     parser.add_argument('--template-settings', action='store',
                         help='index template settings to use when generating elasticsearch template')
     parser.add_argument('--mapping-settings', action='store',
                         help='mapping settings to use when generating elasticsearch template')
+    parser.add_argument('--oss', action='store_true', help='replace basic data types with oss ones where possible')
     parser.add_argument('--strict', action='store_true',
-                        help='enforce stricter checking at schema cleanup')
+                        help='enforce strict checking at schema cleanup')
+    parser.add_argument('--intermediate-only', action='store_true',
+                        help='generate intermediary files only')
     args = parser.parse_args()
     # Clean up empty include of the Makefile
     if args.include and [''] == args.include:

scripts/schema/oss.py

@@ -0,0 +1,29 @@
+# This script performs a best effort fallback of basic data types to equivalent
+# OSS data types.
+# Note however that not all basic data types have an OSS replacement.
+#
+# The way this script is currently written, it has to be run on the fields *before*
+# the cleaner script applies defaults, as there's no concept of defaults here.
+# But since it navigates using the visitor script, it can easily be moved around
+# in the chain, provided we add support for defaults as well.
+#
+# For now, no warning is output on basic fields that don't have a fallback.
+# This could be improved if ECS starts using such types.
+
+from schema import visitor
+
+TYPE_FALLBACKS = {
+    'wildcard': 'keyword',
+    'version': 'keyword'
+}
+
+
+def fallback(fields):
+    """Verify all fields for basic data type usage, and fallback to an OSS equivalent if appropriate."""
+    visitor.visit_fields(fields, field_func=perform_fallback)
+
+
+def perform_fallback(field):
+    """Performs a best effort fallback of basic data types to equivalent OSS data types."""
+    if field['field_details']['type'] in TYPE_FALLBACKS.keys():
+        field['field_details']['type'] = TYPE_FALLBACKS[field['field_details']['type']]

scripts/tests/unit/test_schema_oss.py

@@ -0,0 +1,35 @@
+import os
+import pprint
+import sys
+import unittest
+
+sys.path.append(os.path.join(os.path.dirname(__file__), '../..'))
+
+from schema import oss
+from schema import visitor
+
+
+class TestSchemaOss(unittest.TestCase):
+
+    def setUp(self):
+        self.maxDiff = None
+
+    def test_wildcard_fallback(self):
+        field = {'field_details': {'name': 'myfield', 'type': 'wildcard'}}
+        oss.perform_fallback(field)
+        self.assertEqual('keyword', field['field_details']['type'])
+
+    def test_version_fallback(self):
+        field = {'field_details': {'name': 'myfield', 'type': 'version'}}
+        oss.perform_fallback(field)
+        self.assertEqual('keyword', field['field_details']['type'])
+
+    def test_basic_without_fallback(self):
+        field = {'field_details': {'name': 'myfield', 'type': 'histogram'}}
+        oss.perform_fallback(field)
+        self.assertEqual('histogram', field['field_details']['type'])
+
+    def test_oss_no_fallback(self):
+        field = {'field_details': {'name': 'myfield', 'type': 'keyword'}}
+        oss.perform_fallback(field)
+        self.assertEqual('keyword', field['field_details']['type'])