diff --git a/CHANGELOG.next.md b/CHANGELOG.next.md index 61494ad0b8..f60d15de27 100644 --- a/CHANGELOG.next.md +++ b/CHANGELOG.next.md @@ -63,7 +63,7 @@ Thanks, you're awesome :-) --> #### Improvements -* Add support for reusing offical fieldsets in custom schemas. #751 +* Add support for reusing official fieldsets in custom schemas. #751 * Add full path names to reused fieldsets in `nestings` array in `ecs_nested.yml`. #803 * Allow shorthand notation for including all subfields in subsets. #805 * Add support for Elasticsearch `enabled` field parameter. #824 @@ -83,6 +83,7 @@ Thanks, you're awesome :-) --> * There's a new representation of ECS at `generated/ecs/ecs.yml`, which is a deeply nested representation of the fields. This file is not in git, as it's only meant for developers working on the ECS tools. #864 +* Jinja2 templates now define the doc structure for the AsciiDoc generator. #865 #### Deprecated diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a9176d06db..6831d55c19 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -104,6 +104,51 @@ To build the docs and open them in your browser: make docs ``` +### Generated Documentation Files + +The following files are generated based on the current schema using [Jinja](https://jinja.palletsprojects.com/) templates: + +| File | Template | +| ------------------ | -------- | +| [fields.asciidoc](docs/fields.asciidoc) | [fields_template.j2](scripts/templates/fields_template.j2) | +| [fields-values.asciidoc](docs/field-values.asciidoc) | [field_values_template.j2](scripts/templates/field_values_template.j2) | +| [field-details.asciidoc](docs/field-details.asciidoc) | [field_details directory](scripts/templates/field_details) | + +Running `make` will update these files using the [scripts/generators/asciidoc_fields.py](scripts/generators/asciidoc_fields.py) generator. These doc files should *not* be modified directly. Any changes as a result of a schema update and subsequent run of `make` *should* be committed. + +### Jinja Templates + +Jinja templates allow for formatting or styling changes to templates without needing to modify the generator script directly. Some details about the Jinja template engine and our implementation are covered below as a primer; the full syntax and semantics of Jinja is covered in the [project's documentation](https://jinja.palletsprojects.com/en/2.11.x/templates/). + +#### Delimiters + +* Statements: `{% ... %}` +* Expressions: `{{ ... }}` +* Comments: `{{# ... #}}` + +#### Variables + +Templates variables are passed to the template by the application. Typically these will either be used in an expression or within a control structure statement (e.g. a `for` loop). In the below example, `users` is passed into the template and is iterated over with a `for` loop. + +```python + +``` + +#### Implementation + +The `@templated('template_file_name')` decorator is used to inject the additional functionality that renders and returns the template's content to the generator. Decorated functions should return a dict used to generate the template. When the decorated function returns, the dictionary is passed to the template renderer. + +```python +@templated('fields_template.j2') +def page_field_index(intermediate_nested, ecs_version): + fieldsets = ecs_helpers.dict_sorted_by_keys(intermediate_nested, ['group', 'name']) + return dict(ecs_version=ecs_version, fieldsets=fieldsets) +``` + ## Schema Files The [schemas](schemas) directory contains the files which define the Elastic Common Schema data model. The file structure is documented in [schemas/README.md](schemas). Field additions and modifications will be made to the `schemas/*.yml` files. diff --git a/scripts/generators/asciidoc_fields.py b/scripts/generators/asciidoc_fields.py index 59703fc79a..706cda8b72 100644 --- a/scripts/generators/asciidoc_fields.py +++ b/scripts/generators/asciidoc_fields.py @@ -1,15 +1,54 @@ -from os.path import join +from functools import wraps +import os.path as path + +import jinja2 + from generators import ecs_helpers +# jinja2 setup +TEMPLATE_DIR = path.join(path.dirname(path.abspath(__file__)), '../templates') +template_loader = jinja2.FileSystemLoader(searchpath=TEMPLATE_DIR) +template_env = jinja2.Environment(loader=template_loader) + def generate(nested, ecs_version, out_dir): - save_asciidoc(join(out_dir, 'fields.asciidoc'), page_field_index(nested, ecs_version)) - save_asciidoc(join(out_dir, 'field-details.asciidoc'), page_field_details(nested)) - save_asciidoc(join(out_dir, 'field-values.asciidoc'), page_field_values(nested)) + save_asciidoc(path.join(out_dir, 'fields.asciidoc'), page_field_index(nested, ecs_version)) + save_asciidoc(path.join(out_dir, 'field-details.asciidoc'), page_field_details(nested)) + save_asciidoc(path.join(out_dir, 'field-values.asciidoc'), page_field_values(nested)) # Helpers +def templated(template_name): + """Decorator function to simplify rendering a template. + + :param template_name: the name of the template to be rendered + """ + def decorator(func): + @wraps(func) + def decorated_function(*args, **kwargs): + ctx = func(*args, **kwargs) + if ctx is None: + ctx = {} + elif not isinstance(ctx, dict): + return ctx + return render_template(template_name, **ctx) + return decorated_function + return decorator + + +def render_template(template_name, **context): + """Renders a template from the template folder with the given + context. + + :param template_name: the name of the template to be rendered + :param context: the variables that should be available in the + context of the template. + """ + template = template_env.get_template(template_name) + return template.render(**context) + + def save_asciidoc(file, text): with open(file, "w") as outfile: outfile.write(text) @@ -19,22 +58,10 @@ def save_asciidoc(file, text): # Field Index - +@templated('fields_template.j2') def page_field_index(nested, ecs_version): - page_text = index_header(ecs_version) - for fieldset in ecs_helpers.dict_sorted_by_keys(nested, ['group', 'name']): - page_text += render_field_index_row(fieldset) - page_text += table_footer() - page_text += index_footer() - return page_text - - -def render_field_index_row(fieldset): - return index_row().format( - fieldset_id='ecs-' + fieldset['name'], - fieldset_title=fieldset['title'], - fieldset_short=fieldset.get('short', fieldset['description']) - ) + fieldsets = ecs_helpers.dict_sorted_by_keys(nested, ['group', 'name']) + return dict(ecs_version=ecs_version, fieldsets=fieldsets) # Field Details Page @@ -47,10 +74,10 @@ def page_field_details(nested): def render_fieldset(fieldset, nested): - text = field_details_table_header().format( - fieldset_title=fieldset['title'], - fieldset_name=fieldset['name'], - fieldset_description=render_asciidoc_paragraphs(fieldset['description']) + text = field_details_table_header( + title=fieldset['title'], + name=fieldset['name'], + description=fieldset['description'] ) text += render_fields(fieldset['fields']) @@ -64,26 +91,22 @@ def render_fieldset(fieldset, nested): def render_fields(fields): text = '' - for field_name, field in sorted(fields.items()): + for _, field in sorted(fields.items()): # Skip fields nested in this field set if 'original_fieldset' not in field: text += render_field_details_row(field) return text -def render_asciidoc_paragraphs(string): - '''Simply double the \n''' - return string.replace("\n", "\n\n") - - def render_field_allowed_values(field): if not 'allowed_values' in field: return '' allowed_values = ', '.join(ecs_helpers.list_extract_keys(field['allowed_values'], 'name')) - return field_acceptable_value_names().format( + + return field_acceptable_value_names( allowed_values=allowed_values, - field_flat_name=field['flat_name'], - field_dashed_name=field['dashed_name'], + flat_name=field['flat_name'], + dashed_name=field['dashed_name'] ) @@ -104,14 +127,15 @@ def render_field_details_row(field): if 'array' in field['normalize']: field_normalization = "\nNote: this field should contain an array of values.\n\n" - text = field_details_row().format( - field_flat_name=field['flat_name'], - field_description=render_asciidoc_paragraphs(field['description']), - field_example=example, - field_normalization=field_normalization, - field_level=field['level'], + text = field_details_row( + flat_name=field['flat_name'], + description=field['description'], field_type=field_type_with_mf, + example=example, + normalization=field_normalization, + level=field['level'] ) + return text @@ -120,13 +144,14 @@ def render_fieldset_reuse_section(fieldset, nested): if not ('nestings' in fieldset or 'reusable' in fieldset): return '' - text = field_reuse_section().format( + text = field_reuse_section( reuse_of_fieldset=render_fieldset_reuses_text(fieldset) ) + if 'nestings' in fieldset: - text += nestings_table_header().format( - fieldset_name=fieldset['name'], - fieldset_title=fieldset['title'] + text += nestings_table_header( + name=fieldset['name'], + title=fieldset['title'] ) rows = [] for reused_here_entry in fieldset['reused_here']: @@ -135,8 +160,14 @@ def render_fieldset_reuse_section(fieldset, nested): 'name': reused_here_entry['schema_name'], 'short': reused_here_entry['short'] }) + for row in sorted(rows, key=lambda x: x['flat_nesting']): - text += render_nesting_row(row) + text += nestings_row( + nesting_name=row['name'], + flat_nesting=row['flat_nesting'], + nesting_short=row['short'] + ) + text += table_footer() return text @@ -165,268 +196,74 @@ def render_fieldset_reuses_text(fieldset): return text -def render_nesting_row(nesting): - text = nestings_row().format( - nesting_name=nesting['name'], - flat_nesting=nesting['flat_nesting'], - nesting_short=nesting['short'], - ) - return text - - # Templates - def table_footer(): return ''' |===== ''' - -# Field Index - - -def index_header(ecs_version): - # Not using format() because then asciidoc {ecs}, {es}, etc are resolved. - return ''' -[[ecs-field-reference]] -== {ecs} Field Reference - -This is the documentation of ECS version ''' + ecs_version + '''. - -ECS defines multiple groups of related fields. They are called "field sets". -The <> field set is the only one whose fields are defined -at the root of the event. - -All other field sets are defined as objects in {es}, under which -all fields are defined. - -[float] -[[ecs-fieldsets]] -=== Field Sets -[cols="<,<",options="header",] -|===== -| Field Set | Description -''' - - -def index_row(): - return ''' -| <<{fieldset_id},{fieldset_title}>> | {fieldset_short} -''' - - -def index_footer(): - return ''' -include::field-details.asciidoc[] -''' - - # Field Details Page # Main Fields Table -def field_details_table_header(): - return ''' -[[ecs-{fieldset_name}]] -=== {fieldset_title} Fields - -{fieldset_description} - -==== {fieldset_title} Field Details - -[options="header"] -|===== -| Field | Description | Level - -// =============================================================== -''' - - -def field_details_row(): - return ''' -| {field_flat_name} -| {field_description} - -type: {field_type} - -{field_normalization} - -{field_example} +@templated('field_details/table_header.j2') +def field_details_table_header(title, name, description): + return dict(name=name, title=title, description=description) -| {field_level} -// =============================================================== -''' - - -def field_acceptable_value_names(): - return ''' -*Important*: The field value must be one of the following: +@templated('field_details/row.j2') +def field_details_row(flat_name, description, field_type, normalization, example, level): + return dict( + flat_name=flat_name, + description=description, + field_type=field_type, + normalization=normalization, + example=example, + level=level + ) -{allowed_values} -To learn more about when to use which value, visit the page -<> -''' +@templated('field_details/acceptable_value_names.j2') +def field_acceptable_value_names(allowed_values, dashed_name, flat_name): + return dict( + allowed_values=allowed_values, + dashed_name=dashed_name, + flat_name=flat_name + ) # Field reuse -def field_reuse_section(): - return ''' -==== Field Reuse - -{reuse_of_fieldset} - -''' +@templated('field_details/field_reuse_section.j2') +def field_reuse_section(reuse_of_fieldset): + return dict(reuse_of_fieldset=reuse_of_fieldset) # Nestings table -def nestings_table_header(): - return ''' -[[ecs-{fieldset_name}-nestings]] -===== Field sets that can be nested under {fieldset_title} - -[options="header"] -|===== -| Nested fields | Description - -// =============================================================== - -''' +@templated('field_details/nestings_table_header.j2') +def nestings_table_header(name, title): + return dict(name=name, title=title) -def nestings_row(): - return ''' -| <> -| {nesting_short} - -// =============================================================== - -''' +@templated('field_details/nestings_row.j2') +def nestings_row(nesting_name, flat_nesting, nesting_short): + return dict( + nesting_name=nesting_name, + flat_nesting=flat_nesting, + nesting_short=nesting_short + ) # Allowed values section - -def page_field_values(nested): - section_text = values_section_header() +@templated('field_values_template.j2') +def page_field_values(nested, template_name='field_values_template.j2'): category_fields = ['event.kind', 'event.category', 'event.type', 'event.outcome'] + nested_fields = [] for cat_field in category_fields: - section_text += render_field_values_page(nested['event']['fields'][cat_field]) - return section_text - - -def values_section_header(): - return ''' -[[ecs-category-field-values-reference]] -== {ecs} Categorization Fields - -WARNING: This section of ECS is in beta and is subject to change. These allowed values -are still under active development. Additional values will be published gradually, -and some of the values or relationships described here may change. -Users who want to provide feedback, or who want to have a look at -upcoming allowed values can visit this public feedback document -https://ela.st/ecs-categories-draft. - -At a high level, ECS provides fields to classify events in two different ways: -"Where it's from" (e.g., `event.module`, `event.dataset`, `agent.type`, `observer.type`, etc.), -and "What it is." The categorization fields hold the "What it is" information, -independent of the source of the events. - -ECS defines four categorization fields for this purpose, each of which falls under the `event.*` field set. - -[float] -[[ecs-category-fields]] -=== Categorization Fields - -* <> -* <> -* <> -* <> - -NOTE: If your events don't match any of these categorization values, you should -leave the fields empty. This will ensure you can start populating the fields -once the appropriate categorization values are published, in a later release. -''' - - -def render_field_values_page(field): - # Page heading - heading = field_values_page_template().format( - dashed_name=field['dashed_name'], - flat_name=field['flat_name'], - field_description=render_asciidoc_paragraphs(field['description']), - ) - - # Each allowed value - body = '' - toc = '' - try: - for value_details in field['allowed_values']: - toc += "* <>\n".format( - field_dashed_name=field['dashed_name'], - value_name=value_details['name'] - ) - if 'expected_event_types' in value_details: - additional_details = render_expected_event_types(value_details) - else: - additional_details = '' - body += field_value_template().format( - field_dashed_name=field['dashed_name'], - value_name=value_details['name'], - value_description=render_asciidoc_paragraphs(value_details['description']), - additional_details=additional_details - ) - except UnicodeEncodeError: - print("Problem with field {}, field value:".format(field['flat_name'])) - print(value_details) - raise - return heading + toc + body - - -def render_expected_event_types(value_details): - return expected_event_types_template().format( - category_name=value_details['name'], - expected_types=', '.join(value_details['expected_event_types']), - ) - - -def expected_event_types_template(): - return ''' -*Expected event types for category {category_name}:* + nested_fields.append(nested['event']['fields'][cat_field]) -{expected_types} -''' - - -def field_values_page_template(): - return ''' -[[ecs-allowed-values-{dashed_name}]] -=== ECS Categorization Field: {flat_name} - -{field_description} - -WARNING: After the beta period for categorization, only the allowed categorization -values listed in the ECS repository and official ECS documentation should be considered -official. Use of any other values may result in incompatible implementations -that will require subsequent breaking changes. - -*Allowed Values* - -''' - - -def field_value_template(): - return ''' -[float] -[[ecs-{field_dashed_name}-{value_name}]] -==== {value_name} - -{value_description} - -{additional_details} -''' + return dict(fields=nested_fields) diff --git a/scripts/requirements.txt b/scripts/requirements.txt index 7eaa0b4e30..a199139681 100644 --- a/scripts/requirements.txt +++ b/scripts/requirements.txt @@ -4,3 +4,4 @@ autopep8==1.4.4 yamllint==1.19.0 mock==4.0.2 gitpython==3.1.2 +Jinja2==2.11.2 diff --git a/scripts/templates/field_details/acceptable_value_names.j2 b/scripts/templates/field_details/acceptable_value_names.j2 new file mode 100644 index 0000000000..6080445742 --- /dev/null +++ b/scripts/templates/field_details/acceptable_value_names.j2 @@ -0,0 +1,8 @@ + +*Important*: The field value must be one of the following: + +{{ allowed_values }} + +To learn more about when to use which value, visit the page +<> + diff --git a/scripts/templates/field_details/field_reuse_section.j2 b/scripts/templates/field_details/field_reuse_section.j2 new file mode 100644 index 0000000000..37aa7ded45 --- /dev/null +++ b/scripts/templates/field_details/field_reuse_section.j2 @@ -0,0 +1,6 @@ + +==== Field Reuse + +{{ reuse_of_fieldset }} + + diff --git a/scripts/templates/field_details/nestings_row.j2 b/scripts/templates/field_details/nestings_row.j2 new file mode 100644 index 0000000000..826af848bb --- /dev/null +++ b/scripts/templates/field_details/nestings_row.j2 @@ -0,0 +1,7 @@ + +| <> +| {{ nesting_short }} + +// =============================================================== + + diff --git a/scripts/templates/field_details/nestings_table_header.j2 b/scripts/templates/field_details/nestings_table_header.j2 new file mode 100644 index 0000000000..2ef25791d9 --- /dev/null +++ b/scripts/templates/field_details/nestings_table_header.j2 @@ -0,0 +1,11 @@ + +[[ecs-{{ name }}-nestings]] +===== Field sets that can be nested under {{ title }} + +[options="header"] +|===== +| Nested fields | Description + +// =============================================================== + + diff --git a/scripts/templates/field_details/row.j2 b/scripts/templates/field_details/row.j2 new file mode 100644 index 0000000000..90e2c8877f --- /dev/null +++ b/scripts/templates/field_details/row.j2 @@ -0,0 +1,14 @@ + +| {{ flat_name }} +| {{ description|replace("\n", "\n\n") }} + +type: {{ field_type }} + +{{ normalization }} + +{{ example }} + +| {{ level }} + +// =============================================================== + diff --git a/scripts/templates/field_details/table_header.j2 b/scripts/templates/field_details/table_header.j2 new file mode 100644 index 0000000000..4496e8e768 --- /dev/null +++ b/scripts/templates/field_details/table_header.j2 @@ -0,0 +1,14 @@ + +[[ecs-{{ name }}]] +=== {{ title }} Fields + +{{ description|replace("\n", "\n\n") }} + +==== {{ title }} Field Details + +[options="header"] +|===== +| Field | Description | Level + +// =============================================================== + diff --git a/scripts/templates/field_values_template.j2 b/scripts/templates/field_values_template.j2 new file mode 100644 index 0000000000..1ee2ab9890 --- /dev/null +++ b/scripts/templates/field_values_template.j2 @@ -0,0 +1,59 @@ + +[[ecs-category-field-values-reference]] +== {ecs} Categorization Fields + +WARNING: This section of ECS is in beta and is subject to change. These allowed values +are still under active development. Additional values will be published gradually, +and some of the values or relationships described here may change. +Users who want to provide feedback, or who want to have a look at +upcoming allowed values can visit this public feedback document +https://ela.st/ecs-categories-draft. + +At a high level, ECS provides fields to classify events in two different ways: +"Where it's from" (e.g., `event.module`, `event.dataset`, `agent.type`, `observer.type`, etc.), +and "What it is." The categorization fields hold the "What it is" information, +independent of the source of the events. + +ECS defines four categorization fields for this purpose, each of which falls under the `event.*` field set. + +[float] +[[ecs-category-fields]] +=== Categorization Fields + +* <> +* <> +* <> +* <> + +NOTE: If your events don't match any of these categorization values, you should +leave the fields empty. This will ensure you can start populating the fields +once the appropriate categorization values are published, in a later release. +{% for field in fields %} +[[ecs-allowed-values-{{ field['dashed_name'] }}]] +=== ECS Categorization Field: {{ field['flat_name'] }} + +{{ field['description']|replace("\n", "\n\n") }} + +WARNING: After the beta period for categorization, only the allowed categorization +values listed in the ECS repository and official ECS documentation should be considered +official. Use of any other values may result in incompatible implementations +that will require subsequent breaking changes. + +*Allowed Values* +{% for value_details in field['allowed_values'] %} +* <> +{%- endfor %} +{% for value_details in field['allowed_values'] %} +[float] +[[ecs-{{ field['dashed_name'] }}-{{ value_details['name'] }}]] +==== {{ value_details['name'] }} + +{{ value_details['description']|replace("\n", "\n\n") }} + +{% if 'expected_event_types' in value_details %} +*Expected event types for category {{ value_details['name'] }}:* + +{{ value_details['expected_event_types']|join(', ') }} +{% endif %} +{% endfor %} +{%- endfor %} diff --git a/scripts/templates/fields_template.j2 b/scripts/templates/fields_template.j2 new file mode 100644 index 0000000000..72dad35f60 --- /dev/null +++ b/scripts/templates/fields_template.j2 @@ -0,0 +1,26 @@ + +[[ecs-field-reference]] +== {ecs} Field Reference + +This is the documentation of ECS version {{ ecs_version }}. + +ECS defines multiple groups of related fields. They are called "field sets". +The <> field set is the only one whose fields are defined +at the root of the event. + +All other field sets are defined as objects in {es}, under which +all fields are defined. + +[float] +[[ecs-fieldsets]] +=== Field Sets +[cols="<,<",options="header",] +|===== +| Field Set | Description +{% for fieldset in fieldsets %} +| <> | {{ fieldset.get('short', fieldset['description']) }} +{% endfor %} +|===== + +include::field-details.asciidoc[] +