From 3dfc5a8dd6d0c8a527d93f0149184ac592aeb206 Mon Sep 17 00:00:00 2001 From: Eric Beahan Date: Wed, 22 Jul 2020 16:50:12 -0500 Subject: [PATCH 1/8] Consolidating field-details into single template removing unnessary func param final tweaks of consolidate template cleanup j2 filter declarations more housekeeping remove debugging statement removing unneeded formatting code linting final newline removing unneeded templates --- docs/field-details.asciidoc | 1 + scripts/generators/asciidoc_fields.py | 225 +++--------------- scripts/templates/field_details.j2 | 97 ++++++++ .../field_details/acceptable_value_names.j2 | 8 - .../field_details/field_reuse_section.j2 | 6 - .../templates/field_details/nestings_row.j2 | 7 - .../field_details/nestings_table_header.j2 | 11 - scripts/templates/field_details/row.j2 | 14 -- .../templates/field_details/table_header.j2 | 14 -- 9 files changed, 133 insertions(+), 250 deletions(-) create mode 100644 scripts/templates/field_details.j2 delete mode 100644 scripts/templates/field_details/acceptable_value_names.j2 delete mode 100644 scripts/templates/field_details/field_reuse_section.j2 delete mode 100644 scripts/templates/field_details/nestings_row.j2 delete mode 100644 scripts/templates/field_details/nestings_table_header.j2 delete mode 100644 scripts/templates/field_details/row.j2 delete mode 100644 scripts/templates/field_details/table_header.j2 diff --git a/docs/field-details.asciidoc b/docs/field-details.asciidoc index d774947a62..d1fc97823a 100644 --- a/docs/field-details.asciidoc +++ b/docs/field-details.asciidoc @@ -7038,3 +7038,4 @@ Note also that the `x509` fields are not expected to be used directly at the roo + diff --git a/scripts/generators/asciidoc_fields.py b/scripts/generators/asciidoc_fields.py index c25d7b8162..2fc466e497 100644 --- a/scripts/generators/asciidoc_fields.py +++ b/scripts/generators/asciidoc_fields.py @@ -5,11 +5,6 @@ from generators import ecs_helpers -# jinja2 setup -TEMPLATE_DIR = path.join(path.dirname(path.abspath(__file__)), '../templates') -template_loader = jinja2.FileSystemLoader(searchpath=TEMPLATE_DIR) -template_env = jinja2.Environment(loader=template_loader) - def generate(nested, ecs_version, out_dir): save_asciidoc(path.join(out_dir, 'fields.asciidoc'), page_field_index(nested, ecs_version)) @@ -19,6 +14,27 @@ def generate(nested, ecs_version, out_dir): # Helpers +def render_fieldset_reuse_text(fields): + """Renders the expected nesting locations. + + :param fields: The reusable, expected fields + """ + sorted_fields = sorted(fields, key=lambda k: k['full']) + return map(lambda f: f['full'], sorted_fields) + + +def render_nestings_reuse_section(fieldset): + rows = [] + for reused_here_entry in fieldset['reused_here']: + rows.append({ + 'flat_nesting': "{}.*".format(reused_here_entry['full']), + 'name': reused_here_entry['schema_name'], + 'short': reused_here_entry['short'] + }) + + return sorted(rows, key=lambda x: x['flat_nesting']) + + def templated(template_name): """Decorator function to simplify rendering a template. @@ -53,11 +69,22 @@ def save_asciidoc(f, text): with open(f, "w") as outfile: outfile.write(text) +# jinja2 setup + + +TEMPLATE_DIR = path.join(path.dirname(path.abspath(__file__)), '../templates') +template_loader = jinja2.FileSystemLoader(searchpath=TEMPLATE_DIR) +template_env = jinja2.Environment(loader=template_loader) +template_env.filters.update({ + 'list_extract_keys': ecs_helpers.list_extract_keys, + 'render_fieldset_reuse_text': render_fieldset_reuse_text, + 'render_nestings_reuse_section': render_nestings_reuse_section}) # Rendering schemas # Field Index + @templated('fields_template.j2') def page_field_index(nested, ecs_version): fieldsets = ecs_helpers.dict_sorted_by_keys(nested, ['group', 'name']) @@ -66,192 +93,10 @@ def page_field_index(nested, ecs_version): # Field Details Page - +@templated('field_details.j2') def page_field_details(nested): - page_text = '' - for fieldset in ecs_helpers.dict_sorted_by_keys(nested, ['group', 'name']): - page_text += render_fieldset(fieldset, nested) - return page_text - - -def render_fieldset(fieldset, nested): - text = field_details_table_header( - title=fieldset['title'], - name=fieldset['name'], - description=fieldset['description'] - ) - - text += render_fields(fieldset['fields']) - - text += table_footer() - - text += render_fieldset_reuse_section(fieldset, nested) - - return text - - -def render_fields(fields): - text = '' - for _, field in sorted(fields.items()): - # Skip fields nested in this field set - if 'original_fieldset' not in field: - text += render_field_details_row(field) - return text - - -def render_field_allowed_values(field): - if not 'allowed_values' in field: - return '' - allowed_values = ', '.join(ecs_helpers.list_extract_keys(field['allowed_values'], 'name')) - - return field_acceptable_value_names( - allowed_values=allowed_values, - flat_name=field['flat_name'], - dashed_name=field['dashed_name'] - ) - - -def render_field_details_row(field): - example = '' - if 'allowed_values' in field: - example = render_field_allowed_values(field) - elif 'example' in field: - example = "example: `{}`".format(str(field['example'])) - - field_type_with_mf = field['type'] - if 'multi_fields' in field: - field_type_with_mf += "\n\nMulti-fields:\n\n" - for mf in field['multi_fields']: - field_type_with_mf += "* {} (type: {})\n\n".format(mf['flat_name'], mf['type']) - - field_normalization = '' - if 'array' in field['normalize']: - field_normalization = "\nNote: this field should contain an array of values.\n\n" - - text = field_details_row( - flat_name=field['flat_name'], - description=field['description'], - field_type=field_type_with_mf, - example=example, - normalization=field_normalization, - level=field['level'] - ) - - return text - - -def render_fieldset_reuse_section(fieldset, nested): - '''Render the section on where field set can be nested, and which field sets can be nested here''' - if not ('nestings' in fieldset or 'reusable' in fieldset): - return '' - - text = field_reuse_section( - reuse_of_fieldset=render_fieldset_reuses_text(fieldset) - ) - - if 'nestings' in fieldset: - text += nestings_table_header( - name=fieldset['name'], - title=fieldset['title'] - ) - rows = [] - for reused_here_entry in fieldset['reused_here']: - rows.append({ - 'flat_nesting': "{}.*".format(reused_here_entry['full']), - 'name': reused_here_entry['schema_name'], - 'short': reused_here_entry['short'] - }) - - for row in sorted(rows, key=lambda x: x['flat_nesting']): - text += nestings_row( - nesting_name=row['name'], - flat_nesting=row['flat_nesting'], - nesting_short=row['short'] - ) - - text += table_footer() - return text - - -def render_fieldset_reuses_text(fieldset): - '''Render where a given field set is expected to be reused''' - if 'reusable' not in fieldset: - return '' - - section_name = fieldset['name'] - sorted_fields = sorted(fieldset['reusable']['expected'], key=lambda k: k['full']) - rendered_fields = map(lambda f: "`{}`".format(f['full']), sorted_fields) - text = "The `{}` fields are expected to be nested at: {}.\n\n".format( - section_name, ', '.join(rendered_fields)) - - if 'top_level' in fieldset['reusable'] and fieldset['reusable']['top_level']: - template = "Note also that the `{}` fields may be used directly at the root of the events.\n\n" - else: - template = "Note also that the `{}` fields are not expected to " + \ - "be used directly at the root of the events.\n\n" - text += template.format(section_name) - return text - - -# Templates - -def table_footer(): - return ''' -|===== -''' - -# Field Details Page - -# Main Fields Table - - -@templated('field_details/table_header.j2') -def field_details_table_header(title, name, description): - return dict(name=name, title=title, description=description) - - -@templated('field_details/row.j2') -def field_details_row(flat_name, description, field_type, normalization, example, level): - return dict( - flat_name=flat_name, - description=description, - field_type=field_type, - normalization=normalization, - example=example, - level=level - ) - - -@templated('field_details/acceptable_value_names.j2') -def field_acceptable_value_names(allowed_values, dashed_name, flat_name): - return dict( - allowed_values=allowed_values, - dashed_name=dashed_name, - flat_name=flat_name - ) - - -# Field reuse - -@templated('field_details/field_reuse_section.j2') -def field_reuse_section(reuse_of_fieldset): - return dict(reuse_of_fieldset=reuse_of_fieldset) - - -# Nestings table - -@templated('field_details/nestings_table_header.j2') -def nestings_table_header(name, title): - return dict(name=name, title=title) - - -@templated('field_details/nestings_row.j2') -def nestings_row(nesting_name, flat_nesting, nesting_short): - return dict( - nesting_name=nesting_name, - flat_nesting=flat_nesting, - nesting_short=nesting_short - ) + fieldsets = ecs_helpers.dict_sorted_by_keys(nested, ['group', 'name']) + return dict(fieldsets=fieldsets) # Allowed values section diff --git a/scripts/templates/field_details.j2 b/scripts/templates/field_details.j2 new file mode 100644 index 0000000000..dffa6bf087 --- /dev/null +++ b/scripts/templates/field_details.j2 @@ -0,0 +1,97 @@ + +{% for fieldset in fieldsets -%} +[[ecs-{{ fieldset['name'] }}]] +=== {{ fieldset['title'] }} Fields + +{{ fieldset['description']|replace("\n", "\n\n") }} + +==== {{ fieldset['title'] }} Field Details + +[options="header"] +|===== +| Field | Description | Level + +// =============================================================== + +{% for _, field in fieldset['fields'].items()|sort -%} +{% if 'original_fieldset' not in field -%} +| {{ field['flat_name'] }} +| {{ field['description']|replace("\n", "\n\n") }} + +type: {{ field['type'] }} + +{% if 'multi_fields' in field -%} +Multi-fields: + +{% for mf in field['multi_fields'] -%} +* {{ mf['flat_name'] }} (type: {{ mf ['type'] }}) + + +{% endfor %} +{% endif %} +{% if 'array' in field['normalize'] -%} +Note: this field should contain an array of values. + + +{% endif %} +{% if 'allowed_values' in field -%} +{% set allowed_values = field['allowed_values']|list_extract_keys('name')|join(', ') %} +*Important*: The field value must be one of the following: + +{{ allowed_values }} + +To learn more about when to use which value, visit the page +<> +{% elif 'example' in field -%} +example: `{{ field['example'] }}` +{%- endif %} + +| {{ field['level'] }} + +// =============================================================== + +{% endif -%} +{% endfor -%} +|===== + +{% if 'nestings' in fieldset or 'reusable' in fieldset -%} +==== Field Reuse + +{% if 'reusable' in fieldset -%} +{% set sorted_reuse_fields = fieldset['reusable']['expected']|render_fieldset_reuse_text -%} +The `{{ fieldset['name'] }}` fields are expected to be nested at: `{{ sorted_reuse_fields|join("`, `") }}`. + +{% if 'top_level' in fieldset['reusable'] and fieldset['reusable']['top_level'] -%} +Note also that the `{{ fieldset['name'] }}` fields may be used directly at the root of the events. + +{% else -%} +Note also that the `{{ fieldset['name'] }}` fields are not expected to be used directly at the root of the events. + +{% endif %} +{% else %} +{% endif %} + +{% if 'nestings' in fieldset -%} +[[ecs-{{ fieldset['name'] }}-nestings]] +===== Field sets that can be nested under {{ fieldset['title'] }} + +[options="header"] +|===== +| Nested fields | Description + +// =============================================================== + + +{% for entry in fieldset|render_nestings_reuse_section -%} +| <> +| {{ entry['short'] }} + +// =============================================================== + + +{% endfor -%} +|===== + +{% endif %} +{%- endif %} +{%- endfor %} diff --git a/scripts/templates/field_details/acceptable_value_names.j2 b/scripts/templates/field_details/acceptable_value_names.j2 deleted file mode 100644 index 6080445742..0000000000 --- a/scripts/templates/field_details/acceptable_value_names.j2 +++ /dev/null @@ -1,8 +0,0 @@ - -*Important*: The field value must be one of the following: - -{{ allowed_values }} - -To learn more about when to use which value, visit the page -<> - diff --git a/scripts/templates/field_details/field_reuse_section.j2 b/scripts/templates/field_details/field_reuse_section.j2 deleted file mode 100644 index 37aa7ded45..0000000000 --- a/scripts/templates/field_details/field_reuse_section.j2 +++ /dev/null @@ -1,6 +0,0 @@ - -==== Field Reuse - -{{ reuse_of_fieldset }} - - diff --git a/scripts/templates/field_details/nestings_row.j2 b/scripts/templates/field_details/nestings_row.j2 deleted file mode 100644 index 826af848bb..0000000000 --- a/scripts/templates/field_details/nestings_row.j2 +++ /dev/null @@ -1,7 +0,0 @@ - -| <> -| {{ nesting_short }} - -// =============================================================== - - diff --git a/scripts/templates/field_details/nestings_table_header.j2 b/scripts/templates/field_details/nestings_table_header.j2 deleted file mode 100644 index 2ef25791d9..0000000000 --- a/scripts/templates/field_details/nestings_table_header.j2 +++ /dev/null @@ -1,11 +0,0 @@ - -[[ecs-{{ name }}-nestings]] -===== Field sets that can be nested under {{ title }} - -[options="header"] -|===== -| Nested fields | Description - -// =============================================================== - - diff --git a/scripts/templates/field_details/row.j2 b/scripts/templates/field_details/row.j2 deleted file mode 100644 index 90e2c8877f..0000000000 --- a/scripts/templates/field_details/row.j2 +++ /dev/null @@ -1,14 +0,0 @@ - -| {{ flat_name }} -| {{ description|replace("\n", "\n\n") }} - -type: {{ field_type }} - -{{ normalization }} - -{{ example }} - -| {{ level }} - -// =============================================================== - diff --git a/scripts/templates/field_details/table_header.j2 b/scripts/templates/field_details/table_header.j2 deleted file mode 100644 index 4496e8e768..0000000000 --- a/scripts/templates/field_details/table_header.j2 +++ /dev/null @@ -1,14 +0,0 @@ - -[[ecs-{{ name }}]] -=== {{ title }} Fields - -{{ description|replace("\n", "\n\n") }} - -==== {{ title }} Field Details - -[options="header"] -|===== -| Field | Description | Level - -// =============================================================== - From 04b8446b3898477ebd69199dfcf21d87af5b22c4 Mon Sep 17 00:00:00 2001 From: Eric Beahan Date: Mon, 3 Aug 2020 17:23:01 -0500 Subject: [PATCH 2/8] Improve readability of template improve whitespace for readability adding comments --- scripts/templates/field_details.j2 | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/scripts/templates/field_details.j2 b/scripts/templates/field_details.j2 index dffa6bf087..79da1d8813 100644 --- a/scripts/templates/field_details.j2 +++ b/scripts/templates/field_details.j2 @@ -1,10 +1,13 @@ {% for fieldset in fieldsets -%} + +{# Title & Description -#} [[ecs-{{ fieldset['name'] }}]] === {{ fieldset['title'] }} Fields {{ fieldset['description']|replace("\n", "\n\n") }} +{# Field Details Table Header -#} ==== {{ fieldset['title'] }} Field Details [options="header"] @@ -13,29 +16,37 @@ // =============================================================== +{# Iterate through each field in the set -#} {% for _, field in fieldset['fields'].items()|sort -%} {% if 'original_fieldset' not in field -%} + +{# `Field` column -#} | {{ field['flat_name'] }} +{# `Description` column -#} | {{ field['description']|replace("\n", "\n\n") }} type: {{ field['type'] }} {% if 'multi_fields' in field -%} + Multi-fields: {% for mf in field['multi_fields'] -%} + * {{ mf['flat_name'] }} (type: {{ mf ['type'] }}) {% endfor %} {% endif %} {% if 'array' in field['normalize'] -%} + Note: this field should contain an array of values. {% endif %} -{% if 'allowed_values' in field -%} -{% set allowed_values = field['allowed_values']|list_extract_keys('name')|join(', ') %} +{% if 'allowed_values' in field %} +{% set allowed_values = field['allowed_values']|list_extract_keys('name')|join(', ') -%} + *Important*: The field value must be one of the following: {{ allowed_values }} @@ -43,28 +54,37 @@ Note: this field should contain an array of values. To learn more about when to use which value, visit the page <> {% elif 'example' in field -%} + example: `{{ field['example'] }}` + {%- endif %} +{# `Level` column -#} | {{ field['level'] }} // =============================================================== {% endif -%} {% endfor -%} + |===== +{# do we have `nestings` or `reusable` sections to worry about? -#} {% if 'nestings' in fieldset or 'reusable' in fieldset -%} + ==== Field Reuse {% if 'reusable' in fieldset -%} {% set sorted_reuse_fields = fieldset['reusable']['expected']|render_fieldset_reuse_text -%} + The `{{ fieldset['name'] }}` fields are expected to be nested at: `{{ sorted_reuse_fields|join("`, `") }}`. {% if 'top_level' in fieldset['reusable'] and fieldset['reusable']['top_level'] -%} + Note also that the `{{ fieldset['name'] }}` fields may be used directly at the root of the events. {% else -%} + Note also that the `{{ fieldset['name'] }}` fields are not expected to be used directly at the root of the events. {% endif %} @@ -72,6 +92,7 @@ Note also that the `{{ fieldset['name'] }}` fields are not expected to be used d {% endif %} {% if 'nestings' in fieldset -%} + [[ecs-{{ fieldset['name'] }}-nestings]] ===== Field sets that can be nested under {{ fieldset['title'] }} @@ -83,6 +104,7 @@ Note also that the `{{ fieldset['name'] }}` fields are not expected to be used d {% for entry in fieldset|render_nestings_reuse_section -%} + | <> | {{ entry['short'] }} @@ -90,6 +112,7 @@ Note also that the `{{ fieldset['name'] }}` fields are not expected to be used d {% endfor -%} + |===== {% endif %} From 2f026af19dde358e5a4d7b142cd4fd90ba116d31 Mon Sep 17 00:00:00 2001 From: Eric Beahan Date: Fri, 7 Aug 2020 18:16:18 -0500 Subject: [PATCH 3/8] refactoring to simplify field-details template logic remove one level of iteration improve var name remove render_fieldset_reuse_text filter from template remove render_nestings_reuse_section filter remove sorting from j2 on fieldset fields list refactor logic for extracting allowed_values names linting --- docs/field-details.asciidoc | 1 - scripts/generators/asciidoc_fields.py | 65 ++++++++++++++++++++++----- scripts/templates/field_details.j2 | 13 ++---- 3 files changed, 58 insertions(+), 21 deletions(-) diff --git a/docs/field-details.asciidoc b/docs/field-details.asciidoc index d1fc97823a..6ff1d8645c 100644 --- a/docs/field-details.asciidoc +++ b/docs/field-details.asciidoc @@ -1,4 +1,3 @@ - [[ecs-base]] === Base Fields diff --git a/scripts/generators/asciidoc_fields.py b/scripts/generators/asciidoc_fields.py index 2fc466e497..908bddaad4 100644 --- a/scripts/generators/asciidoc_fields.py +++ b/scripts/generators/asciidoc_fields.py @@ -14,16 +14,26 @@ def generate(nested, ecs_version, out_dir): # Helpers -def render_fieldset_reuse_text(fields): - """Renders the expected nesting locations. +def render_fieldset_reuse_text(fieldset): + """Renders the expected nesting locations + if the the `reusable` object is present. - :param fields: The reusable, expected fields + :param fieldset: The fieldset to evaluate """ - sorted_fields = sorted(fields, key=lambda k: k['full']) + if not fieldset.get('reusable'): + return None + reusable_fields = fieldset['reusable']['expected'] + sorted_fields = sorted(reusable_fields, key=lambda k: k['full']) return map(lambda f: f['full'], sorted_fields) def render_nestings_reuse_section(fieldset): + """Renders the reuse section entries. + + :param fieldset: The target fieldset + """ + if not fieldset.get('reused_here'): + return None rows = [] for reused_here_entry in fieldset['reused_here']: rows.append({ @@ -35,6 +45,33 @@ def render_nestings_reuse_section(fieldset): return sorted(rows, key=lambda x: x['flat_nesting']) +def extract_allowed_values_key_names(field): + """Extracts the `name` keys from the field's + allowed_values if present in the field + object. + + :param field: The target field + """ + if not field.get('allowed_values'): + return [] + return ecs_helpers.list_extract_keys(field['allowed_values'], 'name') + + +def sort_fields(fieldset): + """Prepares a fieldset's fields for being + passed into the j2 template for rendering. This + includes sorting them into a list of objects and + adding a field for the names of any allowed values + for the field, if present. + + :param fieldset: The target fieldset + """ + fields_list = list(fieldset['fields'].values()) + for field in fields_list: + field['allowed_value_names'] = extract_allowed_values_key_names(field) + return sorted(fields_list, key=lambda field: field['name']) + + def templated(template_name): """Decorator function to simplify rendering a template. @@ -75,10 +112,6 @@ def save_asciidoc(f, text): TEMPLATE_DIR = path.join(path.dirname(path.abspath(__file__)), '../templates') template_loader = jinja2.FileSystemLoader(searchpath=TEMPLATE_DIR) template_env = jinja2.Environment(loader=template_loader) -template_env.filters.update({ - 'list_extract_keys': ecs_helpers.list_extract_keys, - 'render_fieldset_reuse_text': render_fieldset_reuse_text, - 'render_nestings_reuse_section': render_nestings_reuse_section}) # Rendering schemas @@ -93,10 +126,22 @@ def page_field_index(nested, ecs_version): # Field Details Page -@templated('field_details.j2') def page_field_details(nested): fieldsets = ecs_helpers.dict_sorted_by_keys(nested, ['group', 'name']) - return dict(fieldsets=fieldsets) + results = (generate_field_details_page(fieldset) for fieldset in fieldsets) + return ''.join(results) + + +@templated('field_details.j2') +def generate_field_details_page(fieldset): + # render field reuse text section + sorted_reuse_fields = render_fieldset_reuse_text(fieldset) + render_nestings_reuse_fields = render_nestings_reuse_section(fieldset) + sorted_fields = sort_fields(fieldset) + return dict(fieldset=fieldset, + sorted_reuse_fields=sorted_reuse_fields, + render_nestings_reuse_section=render_nestings_reuse_fields, + sorted_fields=sorted_fields) # Allowed values section diff --git a/scripts/templates/field_details.j2 b/scripts/templates/field_details.j2 index 79da1d8813..4e9346f400 100644 --- a/scripts/templates/field_details.j2 +++ b/scripts/templates/field_details.j2 @@ -1,6 +1,3 @@ - -{% for fieldset in fieldsets -%} - {# Title & Description -#} [[ecs-{{ fieldset['name'] }}]] === {{ fieldset['title'] }} Fields @@ -17,7 +14,7 @@ // =============================================================== {# Iterate through each field in the set -#} -{% for _, field in fieldset['fields'].items()|sort -%} +{% for field in sorted_fields -%} {% if 'original_fieldset' not in field -%} {# `Field` column -#} @@ -45,11 +42,9 @@ Note: this field should contain an array of values. {% endif %} {% if 'allowed_values' in field %} -{% set allowed_values = field['allowed_values']|list_extract_keys('name')|join(', ') -%} - *Important*: The field value must be one of the following: -{{ allowed_values }} +{{ field['allowed_value_names']|join(', ') }} To learn more about when to use which value, visit the page <> @@ -75,7 +70,6 @@ example: `{{ field['example'] }}` ==== Field Reuse {% if 'reusable' in fieldset -%} -{% set sorted_reuse_fields = fieldset['reusable']['expected']|render_fieldset_reuse_text -%} The `{{ fieldset['name'] }}` fields are expected to be nested at: `{{ sorted_reuse_fields|join("`, `") }}`. @@ -103,7 +97,7 @@ Note also that the `{{ fieldset['name'] }}` fields are not expected to be used d // =============================================================== -{% for entry in fieldset|render_nestings_reuse_section -%} +{% for entry in render_nestings_reuse_section -%} | <> | {{ entry['short'] }} @@ -117,4 +111,3 @@ Note also that the `{{ fieldset['name'] }}` fields are not expected to be used d {% endif %} {%- endif %} -{%- endfor %} From 65d34cd70bf356bb082bf43684bfdac5be5d8395 Mon Sep 17 00:00:00 2001 From: Eric Beahan Date: Mon, 10 Aug 2020 13:20:21 -0500 Subject: [PATCH 4/8] dropping redundant _template naming --- scripts/generators/asciidoc_fields.py | 4 ++-- .../templates/{field_values_template.j2 => field_values.j2} | 0 scripts/templates/{fields_template.j2 => fields.j2} | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename scripts/templates/{field_values_template.j2 => field_values.j2} (100%) rename scripts/templates/{fields_template.j2 => fields.j2} (100%) diff --git a/scripts/generators/asciidoc_fields.py b/scripts/generators/asciidoc_fields.py index 908bddaad4..2aa6f4a8cd 100644 --- a/scripts/generators/asciidoc_fields.py +++ b/scripts/generators/asciidoc_fields.py @@ -118,7 +118,7 @@ def save_asciidoc(f, text): # Field Index -@templated('fields_template.j2') +@templated('fields.j2') def page_field_index(nested, ecs_version): fieldsets = ecs_helpers.dict_sorted_by_keys(nested, ['group', 'name']) return dict(ecs_version=ecs_version, fieldsets=fieldsets) @@ -146,7 +146,7 @@ def generate_field_details_page(fieldset): # Allowed values section -@templated('field_values_template.j2') +@templated('field_values.j2') def page_field_values(nested, template_name='field_values_template.j2'): category_fields = ['event.kind', 'event.category', 'event.type', 'event.outcome'] nested_fields = [] diff --git a/scripts/templates/field_values_template.j2 b/scripts/templates/field_values.j2 similarity index 100% rename from scripts/templates/field_values_template.j2 rename to scripts/templates/field_values.j2 diff --git a/scripts/templates/fields_template.j2 b/scripts/templates/fields.j2 similarity index 100% rename from scripts/templates/fields_template.j2 rename to scripts/templates/fields.j2 From 65246d220cc34445536137aed2497e37ced14e7b Mon Sep 17 00:00:00 2001 From: Eric Beahan Date: Mon, 10 Aug 2020 15:57:18 -0500 Subject: [PATCH 5/8] adding asciidoc generator tests simplify setup and expand coverage linting --- scripts/tests/test_asciidoc_fields.py | 132 ++++++++++++++++++++++++++ 1 file changed, 132 insertions(+) create mode 100644 scripts/tests/test_asciidoc_fields.py diff --git a/scripts/tests/test_asciidoc_fields.py b/scripts/tests/test_asciidoc_fields.py new file mode 100644 index 0000000000..1a099a9958 --- /dev/null +++ b/scripts/tests/test_asciidoc_fields.py @@ -0,0 +1,132 @@ +import os +import sys +import unittest + +sys.path.append(os.path.join(os.path.dirname(__file__), '..')) + +from scripts.generators import asciidoc_fields +from scripts.generators import intermediate_files +from scripts.schema import cleaner +from scripts.schema import loader +from scripts.schema import finalizer + + +class TestGeneratorsAsciiFields(unittest.TestCase): + + def setUp(self): + self.foo_fieldset = self.dummy_fieldset() + + def dummy_fieldset(self): + return { + 'description': 'foo', + 'fields': { + 'foo.type': { + 'dashed_name': 'foo-type', + 'description': 'describes the foo', + 'example': '2016-05-23T08:05:34.853Z', + 'flat_name': 'foo.type', + 'level': 'core', + 'name': 'type', + 'normalize': ['array'], + 'short': 'describes the foo', + 'ignore_above': 1024, + 'type': 'keyword', + 'allowed_values': [{ + 'description': 'fluffy foo', + 'name': 'fluffy', + }, + { + 'description': 'coarse foo', + 'name': 'coarse', + } + ] + }, + 'foo.id': { + 'dashed_name': 'foo-id', + 'description': 'Unique ID of the foo.', + 'example': 'foo123', + 'flat_name': 'foo.id', + 'ignore_above': 1024, + 'level': 'core', + 'name': 'id', + 'normalize': [], + 'short': 'Unique ID of the foo.', + 'type': 'keyword' + } + }, + 'reusable': { + 'expected': [ + { + 'as': 'foo', + 'at': 'server', + 'full': 'server.foo' + }, + { + 'as': 'foo', + 'at': 'source', + 'full': 'source.foo' + }, + { + 'as': 'foo', + 'at': 'client', + 'full': 'client.foo', + }, + { + 'as': 'foo', + 'at': 'destination', + 'full': 'destination.foo' + } + ], + 'top_level': False, + }, + 'reused_here': [ + { + 'full': 'foo.as', + 'schema_name': 'as', + 'short': 'Fields describing an AS' + } + ], + 'group': 2, + 'name': 'foo', + 'prefix': 'foo.', + 'short': 'Foo fields', + 'title': 'Foo', + 'type': 'group' + } + + def test_validate_sort_fieldset(self): + sorted_foo_fields = asciidoc_fields.sort_fields(self.foo_fieldset) + #import pdb;pdb.set_trace() + self.assertIsInstance(sorted_foo_fields, list) + + # `allowed_value_names` always present + for field in sorted_foo_fields: + self.assertIsInstance(field.get('allowed_value_names'), list) + + self.assertFalse(sorted_foo_fields[0]['allowed_value_names']) + self.assertEqual('id', sorted_foo_fields[0]['name']) + self.assertEqual('type', sorted_foo_fields[1]['name']) + self.assertIn('fluffy', sorted_foo_fields[1]['allowed_value_names']) + self.assertIn('coarse', sorted_foo_fields[1]['allowed_value_names']) + + def test_rendering_fieldset_reuse(self): + foo_reuse_fields = asciidoc_fields.render_fieldset_reuse_text(self.foo_fieldset) + expected_sorted_reuse_fields = ( + 'client.foo', + 'destination.foo', + 'server.foo', + 'source.foo' + ) + + self.assertEqual(expected_sorted_reuse_fields, tuple(foo_reuse_fields)) + + def test_rendering_fieldset_nesting(self): + foo_nesting_fields = asciidoc_fields.render_nestings_reuse_section(self.foo_fieldset) + self.assertIsInstance(foo_nesting_fields, list) + self.assertEqual('foo.as.*', foo_nesting_fields[0]['flat_nesting']) + self.assertEqual('as', foo_nesting_fields[0]['name']) + self.assertEqual('Fields describing an AS', foo_nesting_fields[0]['short']) + + +if __name__ == '__main__': + unittest.main() From ba1a17831f3c719d576c06870d0fdbc10eb50372 Mon Sep 17 00:00:00 2001 From: Eric Beahan Date: Wed, 19 Aug 2020 15:30:45 -0500 Subject: [PATCH 6/8] remove unneeded else statement --- scripts/templates/field_details.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/templates/field_details.j2 b/scripts/templates/field_details.j2 index 4e9346f400..2b18177603 100644 --- a/scripts/templates/field_details.j2 +++ b/scripts/templates/field_details.j2 @@ -81,9 +81,9 @@ Note also that the `{{ fieldset['name'] }}` fields may be used directly at the r Note also that the `{{ fieldset['name'] }}` fields are not expected to be used directly at the root of the events. +{% endif -%} {% endif %} -{% else %} -{% endif %} + {% if 'nestings' in fieldset -%} From 23e06d36f21eb10414ac2e2b02cd86fae2b2b82d Mon Sep 17 00:00:00 2001 From: Eric Beahan Date: Wed, 19 Aug 2020 15:57:51 -0500 Subject: [PATCH 7/8] add comments at the end of conditional clauses --- scripts/templates/field_details.j2 | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/scripts/templates/field_details.j2 b/scripts/templates/field_details.j2 index 2b18177603..0b1bb6e224 100644 --- a/scripts/templates/field_details.j2 +++ b/scripts/templates/field_details.j2 @@ -33,8 +33,8 @@ Multi-fields: * {{ mf['flat_name'] }} (type: {{ mf ['type'] }}) -{% endfor %} -{% endif %} +{% endfor %}{# for mf #} +{% endif %}{# if 'multi_fields' #} {% if 'array' in field['normalize'] -%} Note: this field should contain an array of values. @@ -52,15 +52,15 @@ To learn more about when to use which value, visit the page example: `{{ field['example'] }}` -{%- endif %} +{%- endif %}{# if 'allowed_values' elif 'example' #} {# `Level` column -#} | {{ field['level'] }} // =============================================================== -{% endif -%} -{% endfor -%} +{% endif %}{# if 'original_fieldset' -#} +{% endfor %}{# for 'field' -#} |===== @@ -81,8 +81,8 @@ Note also that the `{{ fieldset['name'] }}` fields may be used directly at the r Note also that the `{{ fieldset['name'] }}` fields are not expected to be used directly at the root of the events. -{% endif -%} -{% endif %} +{% endif %}{# if 'top_level' -#} +{% endif %}{# if 'reusable' #} {% if 'nestings' in fieldset -%} @@ -109,5 +109,5 @@ Note also that the `{{ fieldset['name'] }}` fields are not expected to be used d |===== -{% endif %} -{%- endif %} +{% endif %}{# if 'nestings' #} +{%- endif %}{# if 'nestings' or 'reusable' in fieldset #} From d73ebb31436aab21369684302acac0c803e24e3b Mon Sep 17 00:00:00 2001 From: Eric Beahan Date: Thu, 20 Aug 2020 12:16:08 -0500 Subject: [PATCH 8/8] changelog --- CHANGELOG.next.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.md b/CHANGELOG.next.md index a29a639b38..e1ce40c4ab 100644 --- a/CHANGELOG.next.md +++ b/CHANGELOG.next.md @@ -101,6 +101,7 @@ Thanks, you're awesome :-) --> * Jinja2 templates now define the doc structure for the AsciiDoc generator. #865 * Intermediate `ecs_flat.yml` and `ecs_nested.yml` files are now generated for each individual subset, in addition to the intermediate files generated for the combined subset. #873 +* Field details Jinja2 template components have been consolidated into one template #897 #### Deprecated