From 74e4ee426e2babbf84ea2b3da0b944af7505d198 Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Wed, 30 Sep 2020 16:25:43 -0400 Subject: [PATCH 1/9] Reorder flags in docs to start with the more user focused ones --- scripts/generator.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/generator.py b/scripts/generator.py index 733f4155fe..eb4b8a2894 100644 --- a/scripts/generator.py +++ b/scripts/generator.py @@ -60,20 +60,20 @@ def main(): def argument_parser(): parser = argparse.ArgumentParser() - parser.add_argument('--intermediate-only', action='store_true', - help='generate intermediary files only') + parser.add_argument('--ref', action='store', help='git reference to use when building schemas') parser.add_argument('--include', nargs='+', help='include user specified directory of custom field definitions') parser.add_argument('--subset', nargs='+', help='render a subset of the schema') - parser.add_argument('--out', action='store', help='directory to store the generated files') - parser.add_argument('--ref', action='store', help='git reference to use when building schemas') + parser.add_argument('--out', action='store', help='directory to output the generated files') parser.add_argument('--template-settings', action='store', help='index template settings to use when generating elasticsearch template') parser.add_argument('--mapping-settings', action='store', help='mapping settings to use when generating elasticsearch template') parser.add_argument('--strict', action='store_true', - help='enforce stricter checking at schema cleanup') + help='enforce strict checking at schema cleanup') + parser.add_argument('--intermediate-only', action='store_true', + help='generate intermediary files only') args = parser.parse_args() # Clean up empty include of the Makefile if args.include and [''] == args.include: From 525609311d9cde6146884fd4348f5784363b9e7a Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Wed, 30 Sep 2020 16:32:35 -0400 Subject: [PATCH 2/9] Add the --oss flag --- scripts/generator.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/generator.py b/scripts/generator.py index eb4b8a2894..d32e263a44 100644 --- a/scripts/generator.py +++ b/scripts/generator.py @@ -70,6 +70,7 @@ def argument_parser(): help='index template settings to use when generating elasticsearch template') parser.add_argument('--mapping-settings', action='store', help='mapping settings to use when generating elasticsearch template') + parser.add_argument('--oss', action='store_true', help='replace basic data types with oss ones where possible') parser.add_argument('--strict', action='store_true', help='enforce strict checking at schema cleanup') parser.add_argument('--intermediate-only', action='store_true', From a1398ff41c0f5b7d8edc8d40777819324347e576 Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Thu, 1 Oct 2020 15:27:14 -0400 Subject: [PATCH 3/9] Implement simple OSS type fallback --- scripts/generator.py | 3 +++ scripts/schema/oss.py | 28 +++++++++++++++++++++ scripts/tests/unit/test_schema_oss.py | 35 +++++++++++++++++++++++++++ 3 files changed, 66 insertions(+) create mode 100644 scripts/schema/oss.py create mode 100644 scripts/tests/unit/test_schema_oss.py diff --git a/scripts/generator.py b/scripts/generator.py index d32e263a44..b6dcf05db9 100644 --- a/scripts/generator.py +++ b/scripts/generator.py @@ -12,6 +12,7 @@ from generators import intermediate_files from schema import loader +from schema import oss from schema import cleaner from schema import finalizer from schema import subset_filter @@ -41,6 +42,8 @@ def main(): # ecs_helpers.yaml_dump('ecs.yml', fields) fields = loader.load_schemas(ref=args.ref, included_files=args.include) + if args.oss: + oss.fallback(fields) cleaner.clean(fields, strict=args.strict) finalizer.finalize(fields) fields = subset_filter.filter(fields, args.subset, out_dir) diff --git a/scripts/schema/oss.py b/scripts/schema/oss.py new file mode 100644 index 0000000000..55985ad4e0 --- /dev/null +++ b/scripts/schema/oss.py @@ -0,0 +1,28 @@ +# This script performs a best effort fallback of basic data types to equivalent +# OSS data types. +# Note however that not all basic data types have an OSS replacement. +# +# The way this script is currently written, it has to be run on the fields *before* +# the cleaner script applies defaults, as there's no concept of defaults here. +# But since it navigates using the visitor script, it can easily be moved around +# in the chain, provided we add support for defaults as well. +# +# For now, no warning is output on basic fields that don't have a fallback. +# This could be improved when ECS starts using such types. + +from schema import visitor + +TYPE_FALLBACKS = { + 'wildcard': 'keyword', + 'version': 'keyword' +} + +def fallback(fields): + """Verify all fields for basic data type usage, and fallback to an OSS equivalent if appropriate.""" + visitor.visit_fields(fields, field_func=perform_fallback) + + +def perform_fallback(field): + """Performs a best effort fallback of basic data types to equivalent OSS data types.""" + if field['field_details']['type'] in TYPE_FALLBACKS.keys(): + field['field_details']['type'] = TYPE_FALLBACKS[field['field_details']['type']] diff --git a/scripts/tests/unit/test_schema_oss.py b/scripts/tests/unit/test_schema_oss.py new file mode 100644 index 0000000000..354d1656c5 --- /dev/null +++ b/scripts/tests/unit/test_schema_oss.py @@ -0,0 +1,35 @@ +import os +import pprint +import sys +import unittest + +sys.path.append(os.path.join(os.path.dirname(__file__), '../..')) + +from schema import oss +from schema import visitor + + +class TestSchemaOss(unittest.TestCase): + + def setUp(self): + self.maxDiff = None + + def test_wildcard_fallback(self): + field = { 'field_details': { 'name': 'myfield', 'type': 'wildcard' } } + oss.perform_fallback(field) + self.assertEqual('keyword', field['field_details']['type']) + + def test_version_fallback(self): + field = { 'field_details': { 'name': 'myfield', 'type': 'version' } } + oss.perform_fallback(field) + self.assertEqual('keyword', field['field_details']['type']) + + def test_basic_without_fallback(self): + field = { 'field_details': { 'name': 'myfield', 'type': 'histogram' } } + oss.perform_fallback(field) + self.assertEqual('histogram', field['field_details']['type']) + + def test_oss_no_fallback(self): + field = { 'field_details': { 'name': 'myfield', 'type': 'keyword' } } + oss.perform_fallback(field) + self.assertEqual('keyword', field['field_details']['type']) From d689b53591b6925be1a99ebb645c2cde4f9ce5b7 Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Thu, 1 Oct 2020 15:36:33 -0400 Subject: [PATCH 4/9] Changelog --- CHANGELOG.next.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.md b/CHANGELOG.next.md index b3b73b8bda..2572e441bf 100644 --- a/CHANGELOG.next.md +++ b/CHANGELOG.next.md @@ -42,6 +42,7 @@ Thanks, you're awesome :-) --> * Introduced `--strict` flag to perform stricter schema validation when running the generator script. #937 * Added check under `--strict` that ensures composite types in example fields are quoted. #966 * Added `ignore_above` and `normalizer` support for keyword multi-fields. #971 +* Added `--oss` flag for users who want to generate ECS templates for use on OSS clusters. #991 #### Improvements From 0fa7554661b385c2785f090f4a744598be91ed48 Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Thu, 1 Oct 2020 15:47:26 -0400 Subject: [PATCH 5/9] Add docs in the USAGE.md file --- USAGE.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/USAGE.md b/USAGE.md index e70da6b14f..a98a875802 100644 --- a/USAGE.md +++ b/USAGE.md @@ -29,6 +29,7 @@ relevant artifacts for their unique set of data sources. + [Subset](#subset) + [Ref](#ref) + [Mapping & Template Settings](#mapping--template-settings) + + [OSS](#oss) + [Strict Mode](#strict-mode) + [Intermediate-Only](#intermediate-only) @@ -295,6 +296,25 @@ The `--template-settings` argument defines [index level settings](https://www.el For `template.json`, the `mappings` object is left empty: `{}`. Likewise the `properties` object remains empty in the `mapping.json` example. This will be filled in automatically by the script. +#### OSS + +ECS now uses basic licensed types that are not available on OSS Elasticsearch clusters. Some of these types have an OSS replacement that can be used instead, without too much loss of functionality. + +This flag performs a best effort fallback, replacing basic data types with their OSS replacement. + +Indices using purely OSS types will benefit from the normalization of ECS, but may be missing on some of the added functionality of basic types. + +Current fallbacks applied by this flag are: + +- `wildcard` => `keyword` +- `version` => `keyword` + +Usage: + +``` +$ python scripts/generator.py --oss +``` + #### Strict Mode The `--strict` argument enables "strict mode". Strict mode performs a stricter validation step against the schema's contents. From a5030061550ed7090a5d46dbae7e01ba9b84b70f Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Thu, 1 Oct 2020 15:49:40 -0400 Subject: [PATCH 6/9] Fix small mistake in another part of the docs --- USAGE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/USAGE.md b/USAGE.md index a98a875802..abd257781b 100644 --- a/USAGE.md +++ b/USAGE.md @@ -322,7 +322,7 @@ The `--strict` argument enables "strict mode". Strict mode performs a stricter v Basic usage: ``` -$ python/generator.py --strict +$ python scripts/generator.py --strict ``` Strict mode requires the following conditions, else the script exits on an exception: From cd16747c61275c5099f87f081b4fcacea1f8d1e4 Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Thu, 1 Oct 2020 15:51:33 -0400 Subject: [PATCH 7/9] If, not when --- scripts/schema/oss.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/schema/oss.py b/scripts/schema/oss.py index 55985ad4e0..21c4c80eae 100644 --- a/scripts/schema/oss.py +++ b/scripts/schema/oss.py @@ -8,7 +8,7 @@ # in the chain, provided we add support for defaults as well. # # For now, no warning is output on basic fields that don't have a fallback. -# This could be improved when ECS starts using such types. +# This could be improved if ECS starts using such types. from schema import visitor From 1bb60b8d9eb0b5c68c47131b81dc44de47353d9f Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Thu, 1 Oct 2020 15:53:50 -0400 Subject: [PATCH 8/9] Code formatting --- scripts/schema/oss.py | 1 + scripts/tests/unit/test_schema_oss.py | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/scripts/schema/oss.py b/scripts/schema/oss.py index 21c4c80eae..ba38a254b1 100644 --- a/scripts/schema/oss.py +++ b/scripts/schema/oss.py @@ -17,6 +17,7 @@ 'version': 'keyword' } + def fallback(fields): """Verify all fields for basic data type usage, and fallback to an OSS equivalent if appropriate.""" visitor.visit_fields(fields, field_func=perform_fallback) diff --git a/scripts/tests/unit/test_schema_oss.py b/scripts/tests/unit/test_schema_oss.py index 354d1656c5..4ac08d9d08 100644 --- a/scripts/tests/unit/test_schema_oss.py +++ b/scripts/tests/unit/test_schema_oss.py @@ -15,21 +15,21 @@ def setUp(self): self.maxDiff = None def test_wildcard_fallback(self): - field = { 'field_details': { 'name': 'myfield', 'type': 'wildcard' } } + field = {'field_details': {'name': 'myfield', 'type': 'wildcard'}} oss.perform_fallback(field) self.assertEqual('keyword', field['field_details']['type']) def test_version_fallback(self): - field = { 'field_details': { 'name': 'myfield', 'type': 'version' } } + field = {'field_details': {'name': 'myfield', 'type': 'version'}} oss.perform_fallback(field) self.assertEqual('keyword', field['field_details']['type']) def test_basic_without_fallback(self): - field = { 'field_details': { 'name': 'myfield', 'type': 'histogram' } } + field = {'field_details': {'name': 'myfield', 'type': 'histogram'}} oss.perform_fallback(field) self.assertEqual('histogram', field['field_details']['type']) def test_oss_no_fallback(self): - field = { 'field_details': { 'name': 'myfield', 'type': 'keyword' } } + field = {'field_details': {'name': 'myfield', 'type': 'keyword'}} oss.perform_fallback(field) self.assertEqual('keyword', field['field_details']['type']) From c947c4f5bb1c55ad8555ee36e2d81b9d2543eceb Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Fri, 2 Oct 2020 09:36:02 -0400 Subject: [PATCH 9/9] Try to make it clear that the --oss flag will be unnecessary for most users. I don't think this was clear enough before. --- USAGE.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/USAGE.md b/USAGE.md index abd257781b..cb0c49bf27 100644 --- a/USAGE.md +++ b/USAGE.md @@ -298,11 +298,16 @@ For `template.json`, the `mappings` object is left empty: `{}`. Likewise the `pr #### OSS -ECS now uses basic licensed types that are not available on OSS Elasticsearch clusters. Some of these types have an OSS replacement that can be used instead, without too much loss of functionality. +**IMPORTANT**: This feature is unnecessary for most users. Our default free distribution +comes with the Elastic Basic license, and supports all data types used by ECS. +Learn more about our licenses [here](https://www.elastic.co/subscriptions). + +Users that want to use the open source version of Elasticsearch do not have access to the basic data types. +However some of these types have an OSS replacement that can be used instead, without too much loss of functionality. This flag performs a best effort fallback, replacing basic data types with their OSS replacement. -Indices using purely OSS types will benefit from the normalization of ECS, but may be missing on some of the added functionality of basic types. +Indices using purely OSS types will benefit from the normalization of ECS, but may be missing on some of the added functionality of these basic types. Current fallbacks applied by this flag are: