From 0a81e9148e3af04efb982e26a38ad7fc2fd419d1 Mon Sep 17 00:00:00 2001 From: Ben Skelker Date: Wed, 25 Sep 2019 12:25:56 +0300 Subject: [PATCH 1/9] Example field reuse link for as fields --- docs/field-details.asciidoc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/field-details.asciidoc b/docs/field-details.asciidoc index 72347fb58e..3632ac40b1 100644 --- a/docs/field-details.asciidoc +++ b/docs/field-details.asciidoc @@ -159,6 +159,8 @@ example: `6.0.0-rc2` An autonomous system (AS) is a collection of connected Internet Protocol (IP) routing prefixes under the control of one or more network operators on behalf of a single administrative entity or domain that presents a common, clearly defined routing policy to the internet. +NOTE: <> + ==== Autonomous System Field Details [options="header"] @@ -191,6 +193,7 @@ example: `Google LLC` |===== +[[as-field-reuse]] ==== Field Reuse The `as` fields are expected to be nested at: `client.as`, `destination.as`, `server.as`, `source.as`. From 2c6e8e636d673227276104aa5dc988e836152fad Mon Sep 17 00:00:00 2001 From: Ben Skelker Date: Wed, 2 Oct 2019 18:27:29 +0300 Subject: [PATCH 2/9] Updated py file to create reuse links --- scripts/generators/asciidoc_fields.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/scripts/generators/asciidoc_fields.py b/scripts/generators/asciidoc_fields.py index 356028ad9c..8c85e48b36 100644 --- a/scripts/generators/asciidoc_fields.py +++ b/scripts/generators/asciidoc_fields.py @@ -53,7 +53,8 @@ def render_fieldset(fieldset, ecs_nested): text = field_details_table_header().format( fieldset_title=fieldset['title'], fieldset_name=fieldset['name'], - fieldset_description=render_asciidoc_paragraphs(fieldset['description']) + fieldset_description=render_asciidoc_paragraphs(fieldset['description']), + fieldset_reuse_links=render_fieldset_reuse_link(fieldset) ) for field in ecs_helpers.dict_sorted_by_keys(fieldset['fields'], 'flat_name'): @@ -92,7 +93,8 @@ def render_fieldset_reuse_section(fieldset, ecs_nested): return '' text = field_reuse_section().format( - reuse_of_fieldset=render_fieldset_reuses_text(fieldset) + reuse_of_fieldset=render_fieldset_reuses_text(fieldset), + fieldset_name=fieldset['name'] ) if 'nestings' in fieldset: text += nestings_table_header().format( @@ -108,6 +110,13 @@ def render_fieldset_reuse_section(fieldset, ecs_nested): }) text += table_footer() return text + +def render_fieldset_reuse_link(fieldset): + '''Render a link to field reuse section, only when appropriate''' + if ('nestings' in fieldset or 'reusable' in fieldset): + return 'NOTE: <>'.format(fieldset['name']) + else: + return 'NOTE: This field set is not nested.' def render_fieldset_reuses_text(fieldset): @@ -199,6 +208,8 @@ def field_details_table_header(): {fieldset_description} +{fieldset_reuse_links} + ==== {fieldset_title} Field Details [options="header"] @@ -228,6 +239,7 @@ def field_details_row(): def field_reuse_section(): return ''' +[[ecs-{fieldset_name}-reuse]] ==== Field Reuse {reuse_of_fieldset} From 2f6bd13e89253593941ac1b67efc8c26ee65a5db Mon Sep 17 00:00:00 2001 From: Ben Skelker Date: Thu, 3 Oct 2019 09:03:35 +0300 Subject: [PATCH 3/9] Updated note text for field reuse --- docs/field-details.asciidoc | 86 ++++++++++++++++++++++++++- scripts/generators/asciidoc_fields.py | 7 ++- 2 files changed, 88 insertions(+), 5 deletions(-) diff --git a/docs/field-details.asciidoc b/docs/field-details.asciidoc index 3632ac40b1..a06db40bf1 100644 --- a/docs/field-details.asciidoc +++ b/docs/field-details.asciidoc @@ -4,6 +4,8 @@ The `base` field set contains all fields which are on the top level. These fields are common across all types of events. +NOTE: This field set is not reused. + ==== Base Field Details [options="header"] @@ -79,6 +81,8 @@ The agent fields contain the data about the software entity, if any, that collec Examples include Beats. Agents may also run on observers. ECS agent.* fields shall be populated with details of the agent running on the host or observer where the event happened or the measurement was taken. +NOTE: This field set is not reused. + ==== Agent Field Details [options="header"] @@ -159,7 +163,7 @@ example: `6.0.0-rc2` An autonomous system (AS) is a collection of connected Internet Protocol (IP) routing prefixes under the control of one or more network operators on behalf of a single administrative entity or domain that presents a common, clearly defined routing policy to the internet. -NOTE: <> +NOTE: See <> information. ==== Autonomous System Field Details @@ -193,7 +197,7 @@ example: `Google LLC` |===== -[[as-field-reuse]] +[[ecs-as-reuse]] ==== Field Reuse The `as` fields are expected to be nested at: `client.as`, `destination.as`, `server.as`, `source.as`. @@ -212,6 +216,8 @@ For TCP events, the client is the initiator of the TCP connection that sends the Client / server representations can add semantic context to an exchange, which is helpful to visualize the data in certain situations. If your context falls in that category, you should still ensure that source and destination are filled appropriately. +NOTE: See <> information. + ==== Client Field Details [options="header"] @@ -357,6 +363,7 @@ example: `co.uk` |===== +[[ecs-client-reuse]] ==== Field Reuse @@ -397,6 +404,8 @@ example: `co.uk` Fields related to the cloud or infrastructure the events are coming from. +NOTE: This field set is not reused. + ==== Cloud Field Details [options="header"] @@ -493,6 +502,8 @@ Container fields are used for meta information about the specific container that These fields help correlate data based containers from any runtime. +NOTE: This field set is not reused. + ==== Container Field Details [options="header"] @@ -576,6 +587,8 @@ Destination fields describe details about the destination of a packet/event. Destination fields are usually populated in conjunction with source fields. +NOTE: See <> information. + ==== Destination Field Details [options="header"] @@ -721,6 +734,7 @@ example: `co.uk` |===== +[[ecs-destination-reuse]] ==== Field Reuse @@ -763,6 +777,8 @@ Fields describing DNS queries and answers. DNS events should either represent a single DNS query prior to getting answers (`dns.type:query`) or they should represent a full exchange and contain the query details as well as all of the answers that were provided for this query (`dns.type:answer`). +NOTE: This field set is not reused. + ==== DNS Field Details [options="header"] @@ -1002,6 +1018,8 @@ example: `answer` Meta-information specific to ECS. +NOTE: This field set is not reused. + ==== ECS Field Details [options="header"] @@ -1032,6 +1050,8 @@ These fields can represent errors of any kind. Use them for errors that happen while fetching events or in cases where the event itself contains an error. +NOTE: This field set is not reused. + ==== Error Field Details [options="header"] @@ -1104,6 +1124,8 @@ The event fields are used for context information about the log or metric event A log is defined as an event containing details of something that happened. Log events must include the time at which the thing happened. Examples of log events include a process starting on a host, a network packet being sent from a source to a destination, or a network connection between a client and a server being initiated or closed. A metric is defined as an event containing one or more numerical or categorical measurements and the time at which the measurement was taken. Examples of metric events include memory pressure measured on a host, or vulnerabilities measured on a scanned host. +NOTE: This field set is not reused. + ==== Event Field Details [options="header"] @@ -1407,6 +1429,8 @@ A file is defined as a set of information that has been created on, or has exist File objects can be associated with host events, network events, and/or file events (e.g., those produced by File Integrity Monitoring [FIM] products or services). File fields provide details about the affected file associated with the event or metric. +NOTE: See <> information. + ==== File Field Details [options="header"] @@ -1623,6 +1647,7 @@ example: `1001` |===== +[[ecs-file-reuse]] ==== Field Reuse @@ -1653,6 +1678,8 @@ Geo fields can carry data about a specific location related to an event. This geolocation information can be derived from techniques such as Geo IP, or be user-supplied. +NOTE: See <> information. + ==== Geo Field Details [options="header"] @@ -1755,6 +1782,7 @@ example: `Quebec` |===== +[[ecs-geo-reuse]] ==== Field Reuse The `geo` fields are expected to be nested at: `client.geo`, `destination.geo`, `host.geo`, `observer.geo`, `server.geo`, `source.geo`. @@ -1769,6 +1797,8 @@ Note also that the `geo` fields are not expected to be used directly at the top The group fields are meant to represent groups that are relevant to the event. +NOTE: See <> information. + ==== Group Field Details [options="header"] @@ -1814,6 +1844,7 @@ type: keyword |===== +[[ecs-group-reuse]] ==== Field Reuse The `group` fields are expected to be nested at: `user.group`. @@ -1830,6 +1861,8 @@ The hash fields represent different hash algorithms and their values. Field names for common hashes (e.g. MD5, SHA1) are predefined. Add fields for other hashes by lowercasing the hash algorithm name and using underscore separators as appropriate (snake case, e.g. sha3_512). +NOTE: See <> information. + ==== Hash Field Details [options="header"] @@ -1884,6 +1917,7 @@ type: keyword |===== +[[ecs-hash-reuse]] ==== Field Reuse The `hash` fields are expected to be nested at: `file.hash`, `process.hash`. @@ -1900,6 +1934,8 @@ A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes. +NOTE: See <> information. + ==== Host Field Details [options="header"] @@ -2021,6 +2057,7 @@ example: `1325` |===== +[[ecs-host-reuse]] ==== Field Reuse @@ -2061,6 +2098,8 @@ example: `1325` Fields related to HTTP activity. Use the `url` field set to store the url of the request. +NOTE: This field set is not reused. + ==== HTTP Field Details [options="header"] @@ -2192,6 +2231,8 @@ The log.* fields are typically populated with details about the logging mechanis The details specific to your event source are typically not logged under `log.*`, but rather in `event.*` or in other ECS fields. +NOTE: This field set is not reused. + ==== Log Field Details [options="header"] @@ -2357,6 +2398,8 @@ The network is defined as the communication path over which a host or network ev The network.* fields should be populated with details about the network activity associated with an event. +NOTE: This field set is not reused. + ==== Network Field Details [options="header"] @@ -2527,6 +2570,8 @@ An observer is defined as a special network, security, or application device use This could be a custom hardware appliance or a server that has been configured to run special network, security, or application software. Examples include firewalls, web proxies, intrusion detection/prevention systems, network monitoring sensors, web application firewalls, data loss prevention systems, and APM servers. The observer.* fields shall be populated with details of the system, if any, that detects, observes and/or creates a network, security, or application event or metric. Message queues and ETL components used in processing events or metrics are not considered observers in ECS. +NOTE: See <> information. + ==== Observer Field Details [options="header"] @@ -2642,6 +2687,7 @@ type: keyword |===== +[[ecs-observer-reuse]] ==== Field Reuse @@ -2678,6 +2724,8 @@ The organization fields enrich data with information about the company or entity These fields help you arrange or filter data stored in an index by one or multiple organizations. +NOTE: This field set is not reused. + ==== Organization Field Details [options="header"] @@ -2715,6 +2763,8 @@ type: keyword The OS fields contain information about the operating system. +NOTE: See <> information. + ==== Operating System Field Details [options="header"] @@ -2791,6 +2841,7 @@ example: `10.14.1` |===== +[[ecs-os-reuse]] ==== Field Reuse The `os` fields are expected to be nested at: `host.os`, `observer.os`, `user_agent.os`. @@ -2805,6 +2856,8 @@ Note also that the `os` fields are not expected to be used directly at the top l These fields contain information about an installed software package. It contains general information about a package, such as name, version or size. It also contains installation details, such as time or location. +NOTE: This field set is not reused. + ==== Package Field Details [options="header"] @@ -2971,6 +3024,8 @@ These fields contain information about a process. These fields can help you correlate metrics information with a process id/name from a log message. The `process.pid` often stays in the metric itself and is copied to the global field for correlation. +NOTE: See <> information. + ==== Process Field Details [options="header"] @@ -3335,6 +3390,7 @@ example: `/home/alice` |===== +[[ecs-process-reuse]] ==== Field Reuse @@ -3367,6 +3423,8 @@ Some pieces of information can be seen in many places in an ECS event. To facili A concrete example is IP addresses, which can be under host, observer, source, destination, client, server, and network.forwarded_ip. If you append all IPs to `related.ip`, you can then search for a given IP trivially, no matter where it appeared, by querying `related.ip:a.b.c.d`. +NOTE: This field set is not reused. + ==== Related Field Details [options="header"] @@ -3397,6 +3455,8 @@ For TCP events, the server is the receiver of the initial SYN packet(s) of the T Client / server representations can add semantic context to an exchange, which is helpful to visualize the data in certain situations. If your context falls in that category, you should still ensure that source and destination are filled appropriately. +NOTE: See <> information. + ==== Server Field Details [options="header"] @@ -3542,6 +3602,7 @@ example: `co.uk` |===== +[[ecs-server-reuse]] ==== Field Reuse @@ -3584,6 +3645,8 @@ The service fields describe the service for or from which the data was collected These fields help you find and correlate logs for a specific service and version. +NOTE: This field set is not reused. + ==== Service Field Details [options="header"] @@ -3698,6 +3761,8 @@ Source fields describe details about the source of a packet/event. Source fields are usually populated in conjunction with destination fields. +NOTE: See <> information. + ==== Source Field Details [options="header"] @@ -3843,6 +3908,7 @@ example: `co.uk` |===== +[[ecs-source-reuse]] ==== Field Reuse @@ -3885,6 +3951,8 @@ Fields to classify events and alerts according to a threat taxonomy such as the These fields are for users to classify alerts from all of their sources (e.g. IDS, NGFW, etc.) within a common taxonomy. The threat.tactic.* are meant to capture the high level category of the threat (e.g. "impact"). The threat.technique.* fields are meant to capture which kind of approach is used by this detected threat, to accomplish the goal (e.g. "endpoint denial of service"). +NOTE: This field set is not reused. + ==== Threat Field Details [options="header"] @@ -3977,6 +4045,8 @@ example: `https://attack.mitre.org/techniques/T1499/` Fields related to a TLS connection. These fields focus on the TLS protocol itself and intentionally avoids in-depth analysis of the related x.509 certificate files. +NOTE: This field set is not reused. + ==== TLS Field Details [options="header"] @@ -4322,6 +4392,8 @@ example: `tls` Distributed tracing makes it possible to analyze performance throughout a microservice architecture all in one view. This is accomplished by tracing all of the requests - from the initial web request in the front-end service - to queries made through multiple back-end services. +NOTE: This field set is not reused. + ==== Tracing Field Details [options="header"] @@ -4363,6 +4435,8 @@ example: `00f067aa0ba902b7` URL fields provide support for complete or partial URLs, and supports the breaking down into scheme, domain, path, and so on. +NOTE: This field set is not reused. + ==== URL Field Details [options="header"] @@ -4545,6 +4619,8 @@ The user fields describe information about the user that is relevant to the even Fields can have one entry or multiple entries. If a user has more than one id, provide an array that includes all of them. +NOTE: See <> information. + ==== User Field Details [options="header"] @@ -4625,6 +4701,7 @@ example: `albert` |===== +[[ecs-user-reuse]] ==== Field Reuse The `user` fields are expected to be nested at: `client.user`, `destination.user`, `host.user`, `server.user`, `source.user`. @@ -4659,6 +4736,8 @@ The user_agent fields normally come from a browser request. They often show up in web service logs coming from the parsed user agent string. +NOTE: See <> information. + ==== User agent Field Details [options="header"] @@ -4713,6 +4792,7 @@ example: `12.0` |===== +[[ecs-user_agent-reuse]] ==== Field Reuse @@ -4741,6 +4821,8 @@ example: `12.0` The vulnerability fields describe information about a vulnerability that is relevant to an event. +NOTE: This field set is not reused. + ==== Vulnerability Field Details [options="header"] diff --git a/scripts/generators/asciidoc_fields.py b/scripts/generators/asciidoc_fields.py index 8c85e48b36..bc9c0ea0ef 100644 --- a/scripts/generators/asciidoc_fields.py +++ b/scripts/generators/asciidoc_fields.py @@ -110,13 +110,14 @@ def render_fieldset_reuse_section(fieldset, ecs_nested): }) text += table_footer() return text - + + def render_fieldset_reuse_link(fieldset): '''Render a link to field reuse section, only when appropriate''' if ('nestings' in fieldset or 'reusable' in fieldset): - return 'NOTE: <>'.format(fieldset['name']) + return 'NOTE: See <> information.'.format(fieldset['name']) else: - return 'NOTE: This field set is not nested.' + return 'NOTE: This field set is not reused.' def render_fieldset_reuses_text(fieldset): From 03beb424304c46a881587b9473307e812ea8c881 Mon Sep 17 00:00:00 2001 From: Ben Skelker Date: Sun, 6 Oct 2019 13:16:25 +0300 Subject: [PATCH 4/9] Update field reuse note and section structure --- docs/field-details.asciidoc | 400 ++++++++++++++++++++++---- scripts/generators/asciidoc_fields.py | 79 +++-- 2 files changed, 396 insertions(+), 83 deletions(-) diff --git a/docs/field-details.asciidoc b/docs/field-details.asciidoc index a06db40bf1..b67fcc2376 100644 --- a/docs/field-details.asciidoc +++ b/docs/field-details.asciidoc @@ -4,7 +4,7 @@ The `base` field set contains all fields which are on the top level. These fields are common across all types of events. -NOTE: This field set is not reused. + ==== Base Field Details @@ -74,6 +74,12 @@ example: `["production", "env2"]` |===== +[[ecs-base-reuse]] +==== Field Reuse + +These fields are not reused. + + [[ecs-agent]] === Agent Fields @@ -81,7 +87,7 @@ The agent fields contain the data about the software entity, if any, that collec Examples include Beats. Agents may also run on observers. ECS agent.* fields shall be populated with details of the agent running on the host or observer where the event happened or the measurement was taken. -NOTE: This field set is not reused. + ==== Agent Field Details @@ -158,12 +164,18 @@ example: `6.0.0-rc2` |===== +[[ecs-agent-reuse]] +==== Field Reuse + +These fields are not reused. + + [[ecs-as]] === Autonomous System Fields An autonomous system (AS) is a collection of connected Internet Protocol (IP) routing prefixes under the control of one or more network operators on behalf of a single administrative entity or domain that presents a common, clearly defined routing policy to the internet. -NOTE: See <> information. +NOTE: See information on `as` <>. ==== Autonomous System Field Details @@ -200,9 +212,43 @@ example: `Google LLC` [[ecs-as-reuse]] ==== Field Reuse -The `as` fields are expected to be nested at: `client.as`, `destination.as`, `server.as`, `source.as`. -Note also that the `as` fields are not expected to be used directly at the top level. +[[ecs-as-parents]] +The `as` fields must be nested under: + +[options="header"] +|===== +| Parent fields | Description + +// =============================================================== + + +| <> +| Fields about the client side of a network connection, used with server. + +// =============================================================== + + +| <> +| Fields about the destination side of a network connection, used with source. + +// =============================================================== + + +| <> +| Fields about the server side of a network connection, used with client. + +// =============================================================== + + +| <> +| Fields about the source side of a network connection, used with destination. + +// =============================================================== + + +|===== +NOTE: The `as` fields should *not* be used directly as root fields. @@ -216,7 +262,7 @@ For TCP events, the client is the initiator of the TCP connection that sends the Client / server representations can add semantic context to an exchange, which is helpful to visualize the data in certain situations. If your context falls in that category, you should still ensure that source and destination are filled appropriately. -NOTE: See <> information. +NOTE: See information on `client` <>. ==== Client Field Details @@ -370,11 +416,11 @@ example: `co.uk` [[ecs-client-nestings]] -===== Field sets that can be nested under Client +The `client` field can be a parent of: [options="header"] |===== -| Nested fields | Description +| Child fields | Description // =============================================================== @@ -404,7 +450,7 @@ example: `co.uk` Fields related to the cloud or infrastructure the events are coming from. -NOTE: This field set is not reused. + ==== Cloud Field Details @@ -495,6 +541,12 @@ example: `us-east-1` |===== +[[ecs-cloud-reuse]] +==== Field Reuse + +These fields are not reused. + + [[ecs-container]] === Container Fields @@ -502,7 +554,7 @@ Container fields are used for meta information about the specific container that These fields help correlate data based containers from any runtime. -NOTE: This field set is not reused. + ==== Container Field Details @@ -580,6 +632,12 @@ example: `docker` |===== +[[ecs-container-reuse]] +==== Field Reuse + +These fields are not reused. + + [[ecs-destination]] === Destination Fields @@ -587,7 +645,7 @@ Destination fields describe details about the destination of a packet/event. Destination fields are usually populated in conjunction with source fields. -NOTE: See <> information. +NOTE: See information on `destination` <>. ==== Destination Field Details @@ -741,11 +799,11 @@ example: `co.uk` [[ecs-destination-nestings]] -===== Field sets that can be nested under Destination +The `destination` field can be a parent of: [options="header"] |===== -| Nested fields | Description +| Child fields | Description // =============================================================== @@ -777,7 +835,7 @@ Fields describing DNS queries and answers. DNS events should either represent a single DNS query prior to getting answers (`dns.type:query`) or they should represent a full exchange and contain the query details as well as all of the answers that were provided for this query (`dns.type:answer`). -NOTE: This field set is not reused. + ==== DNS Field Details @@ -1013,12 +1071,18 @@ example: `answer` |===== +[[ecs-dns-reuse]] +==== Field Reuse + +These fields are not reused. + + [[ecs-ecs]] === ECS Fields Meta-information specific to ECS. -NOTE: This field set is not reused. + ==== ECS Field Details @@ -1043,6 +1107,12 @@ example: `1.0.0` |===== +[[ecs-ecs-reuse]] +==== Field Reuse + +These fields are not reused. + + [[ecs-error]] === Error Fields @@ -1050,7 +1120,7 @@ These fields can represent errors of any kind. Use them for errors that happen while fetching events or in cases where the event itself contains an error. -NOTE: This field set is not reused. + ==== Error Field Details @@ -1117,6 +1187,12 @@ example: `java.lang.NullPointerException` |===== +[[ecs-error-reuse]] +==== Field Reuse + +These fields are not reused. + + [[ecs-event]] === Event Fields @@ -1124,7 +1200,7 @@ The event fields are used for context information about the log or metric event A log is defined as an event containing details of something that happened. Log events must include the time at which the thing happened. Examples of log events include a process starting on a host, a network packet being sent from a source to a destination, or a network connection between a client and a server being initiated or closed. A metric is defined as an event containing one or more numerical or categorical measurements and the time at which the measurement was taken. Examples of metric events include memory pressure measured on a host, or vulnerabilities measured on a scanned host. -NOTE: This field set is not reused. + ==== Event Field Details @@ -1422,6 +1498,12 @@ type: keyword |===== +[[ecs-event-reuse]] +==== Field Reuse + +These fields are not reused. + + [[ecs-file]] === File Fields @@ -1429,7 +1511,7 @@ A file is defined as a set of information that has been created on, or has exist File objects can be associated with host events, network events, and/or file events (e.g., those produced by File Integrity Monitoring [FIM] products or services). File fields provide details about the affected file associated with the event or metric. -NOTE: See <> information. +NOTE: See information on `file` <>. ==== File Field Details @@ -1654,11 +1736,11 @@ example: `1001` [[ecs-file-nestings]] -===== Field sets that can be nested under File +The `file` field can be a parent of: [options="header"] |===== -| Nested fields | Description +| Child fields | Description // =============================================================== @@ -1678,7 +1760,7 @@ Geo fields can carry data about a specific location related to an event. This geolocation information can be derived from techniques such as Geo IP, or be user-supplied. -NOTE: See <> information. +NOTE: See information on `geo` <>. ==== Geo Field Details @@ -1785,9 +1867,55 @@ example: `Quebec` [[ecs-geo-reuse]] ==== Field Reuse -The `geo` fields are expected to be nested at: `client.geo`, `destination.geo`, `host.geo`, `observer.geo`, `server.geo`, `source.geo`. -Note also that the `geo` fields are not expected to be used directly at the top level. +[[ecs-geo-parents]] +The `geo` fields must be nested under: + +[options="header"] +|===== +| Parent fields | Description + +// =============================================================== + + +| <> +| Fields about the client side of a network connection, used with server. + +// =============================================================== + + +| <> +| Fields about the destination side of a network connection, used with source. + +// =============================================================== + + +| <> +| Fields describing the relevant computing instance. + +// =============================================================== + + +| <> +| Fields describing an entity observing the event from outside the host. + +// =============================================================== + + +| <> +| Fields about the server side of a network connection, used with client. + +// =============================================================== + + +| <> +| Fields about the source side of a network connection, used with destination. + +// =============================================================== + + +|===== +NOTE: The `geo` fields should *not* be used directly as root fields. @@ -1797,7 +1925,7 @@ Note also that the `geo` fields are not expected to be used directly at the top The group fields are meant to represent groups that are relevant to the event. -NOTE: See <> information. +NOTE: See information on `group` <>. ==== Group Field Details @@ -1847,9 +1975,25 @@ type: keyword [[ecs-group-reuse]] ==== Field Reuse -The `group` fields are expected to be nested at: `user.group`. -Note also that the `group` fields may be used directly at the top level. +[[ecs-group-parents]] +The `group` fields can be nested under: + +[options="header"] +|===== +| Parent fields | Description + +// =============================================================== + + +| <> +| Fields to describe the user relevant to the event. + +// =============================================================== + + +|===== +NOTE: The `group` fields can also be used directly as root fields. @@ -1861,7 +2005,7 @@ The hash fields represent different hash algorithms and their values. Field names for common hashes (e.g. MD5, SHA1) are predefined. Add fields for other hashes by lowercasing the hash algorithm name and using underscore separators as appropriate (snake case, e.g. sha3_512). -NOTE: See <> information. +NOTE: See information on `hash` <>. ==== Hash Field Details @@ -1920,9 +2064,31 @@ type: keyword [[ecs-hash-reuse]] ==== Field Reuse -The `hash` fields are expected to be nested at: `file.hash`, `process.hash`. -Note also that the `hash` fields are not expected to be used directly at the top level. +[[ecs-hash-parents]] +The `hash` fields must be nested under: + +[options="header"] +|===== +| Parent fields | Description + +// =============================================================== + + +| <> +| Fields describing files. + +// =============================================================== + + +| <> +| These fields contain information about a process. + +// =============================================================== + + +|===== +NOTE: The `hash` fields should *not* be used directly as root fields. @@ -1934,7 +2100,7 @@ A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes. -NOTE: See <> information. +NOTE: See information on `host` <>. ==== Host Field Details @@ -2064,11 +2230,11 @@ example: `1325` [[ecs-host-nestings]] -===== Field sets that can be nested under Host +The `host` field can be a parent of: [options="header"] |===== -| Nested fields | Description +| Child fields | Description // =============================================================== @@ -2098,7 +2264,7 @@ example: `1325` Fields related to HTTP activity. Use the `url` field set to store the url of the request. -NOTE: This field set is not reused. + ==== HTTP Field Details @@ -2222,6 +2388,12 @@ example: `1.1` |===== +[[ecs-http-reuse]] +==== Field Reuse + +These fields are not reused. + + [[ecs-log]] === Log Fields @@ -2231,7 +2403,7 @@ The log.* fields are typically populated with details about the logging mechanis The details specific to your event source are typically not logged under `log.*`, but rather in `event.*` or in other ECS fields. -NOTE: This field set is not reused. + ==== Log Field Details @@ -2391,6 +2563,12 @@ example: `Error` |===== +[[ecs-log-reuse]] +==== Field Reuse + +These fields are not reused. + + [[ecs-network]] === Network Fields @@ -2398,7 +2576,7 @@ The network is defined as the communication path over which a host or network ev The network.* fields should be populated with details about the network activity associated with an event. -NOTE: This field set is not reused. + ==== Network Field Details @@ -2563,6 +2741,12 @@ example: `ipv4` |===== +[[ecs-network-reuse]] +==== Field Reuse + +These fields are not reused. + + [[ecs-observer]] === Observer Fields @@ -2570,7 +2754,7 @@ An observer is defined as a special network, security, or application device use This could be a custom hardware appliance or a server that has been configured to run special network, security, or application software. Examples include firewalls, web proxies, intrusion detection/prevention systems, network monitoring sensors, web application firewalls, data loss prevention systems, and APM servers. The observer.* fields shall be populated with details of the system, if any, that detects, observes and/or creates a network, security, or application event or metric. Message queues and ETL components used in processing events or metrics are not considered observers in ECS. -NOTE: See <> information. +NOTE: See information on `observer` <>. ==== Observer Field Details @@ -2694,11 +2878,11 @@ type: keyword [[ecs-observer-nestings]] -===== Field sets that can be nested under Observer +The `observer` field can be a parent of: [options="header"] |===== -| Nested fields | Description +| Child fields | Description // =============================================================== @@ -2724,7 +2908,7 @@ The organization fields enrich data with information about the company or entity These fields help you arrange or filter data stored in an index by one or multiple organizations. -NOTE: This field set is not reused. + ==== Organization Field Details @@ -2758,12 +2942,18 @@ type: keyword |===== +[[ecs-organization-reuse]] +==== Field Reuse + +These fields are not reused. + + [[ecs-os]] === Operating System Fields The OS fields contain information about the operating system. -NOTE: See <> information. +NOTE: See information on `os` <>. ==== Operating System Field Details @@ -2844,9 +3034,37 @@ example: `10.14.1` [[ecs-os-reuse]] ==== Field Reuse -The `os` fields are expected to be nested at: `host.os`, `observer.os`, `user_agent.os`. -Note also that the `os` fields are not expected to be used directly at the top level. +[[ecs-os-parents]] +The `os` fields must be nested under: + +[options="header"] +|===== +| Parent fields | Description + +// =============================================================== + + +| <> +| Fields describing the relevant computing instance. + +// =============================================================== + + +| <> +| Fields describing an entity observing the event from outside the host. + +// =============================================================== + + +| <> +| Fields to describe a browser user_agent string. + +// =============================================================== + + +|===== +NOTE: The `os` fields should *not* be used directly as root fields. @@ -3024,7 +3242,7 @@ These fields contain information about a process. These fields can help you correlate metrics information with a process id/name from a log message. The `process.pid` often stays in the metric itself and is copied to the global field for correlation. -NOTE: See <> information. +NOTE: See information on `process` <>. ==== Process Field Details @@ -3397,11 +3615,11 @@ example: `/home/alice` [[ecs-process-nestings]] -===== Field sets that can be nested under Process +The `process` field can be a parent of: [options="header"] |===== -| Nested fields | Description +| Child fields | Description // =============================================================== @@ -3423,7 +3641,7 @@ Some pieces of information can be seen in many places in an ECS event. To facili A concrete example is IP addresses, which can be under host, observer, source, destination, client, server, and network.forwarded_ip. If you append all IPs to `related.ip`, you can then search for a given IP trivially, no matter where it appeared, by querying `related.ip:a.b.c.d`. -NOTE: This field set is not reused. + ==== Related Field Details @@ -3446,6 +3664,12 @@ type: ip |===== +[[ecs-related-reuse]] +==== Field Reuse + +These fields are not reused. + + [[ecs-server]] === Server Fields @@ -3455,7 +3679,7 @@ For TCP events, the server is the receiver of the initial SYN packet(s) of the T Client / server representations can add semantic context to an exchange, which is helpful to visualize the data in certain situations. If your context falls in that category, you should still ensure that source and destination are filled appropriately. -NOTE: See <> information. +NOTE: See information on `server` <>. ==== Server Field Details @@ -3609,11 +3833,11 @@ example: `co.uk` [[ecs-server-nestings]] -===== Field sets that can be nested under Server +The `server` field can be a parent of: [options="header"] |===== -| Nested fields | Description +| Child fields | Description // =============================================================== @@ -3645,7 +3869,7 @@ The service fields describe the service for or from which the data was collected These fields help you find and correlate logs for a specific service and version. -NOTE: This field set is not reused. + ==== Service Field Details @@ -3754,6 +3978,12 @@ example: `3.2.4` |===== +[[ecs-service-reuse]] +==== Field Reuse + +These fields are not reused. + + [[ecs-source]] === Source Fields @@ -3761,7 +3991,7 @@ Source fields describe details about the source of a packet/event. Source fields are usually populated in conjunction with destination fields. -NOTE: See <> information. +NOTE: See information on `source` <>. ==== Source Field Details @@ -3915,11 +4145,11 @@ example: `co.uk` [[ecs-source-nestings]] -===== Field sets that can be nested under Source +The `source` field can be a parent of: [options="header"] |===== -| Nested fields | Description +| Child fields | Description // =============================================================== @@ -4045,7 +4275,7 @@ example: `https://attack.mitre.org/techniques/T1499/` Fields related to a TLS connection. These fields focus on the TLS protocol itself and intentionally avoids in-depth analysis of the related x.509 certificate files. -NOTE: This field set is not reused. + ==== TLS Field Details @@ -4612,6 +4842,12 @@ type: keyword |===== +[[ecs-url-reuse]] +==== Field Reuse + +These fields are not reused. + + [[ecs-user]] === User Fields @@ -4619,7 +4855,7 @@ The user fields describe information about the user that is relevant to the even Fields can have one entry or multiple entries. If a user has more than one id, provide an array that includes all of them. -NOTE: See <> information. +NOTE: See information on `user` <>. ==== User Field Details @@ -4704,19 +4940,59 @@ example: `albert` [[ecs-user-reuse]] ==== Field Reuse -The `user` fields are expected to be nested at: `client.user`, `destination.user`, `host.user`, `server.user`, `source.user`. -Note also that the `user` fields may be used directly at the top level. +[[ecs-user-parents]] +The `user` fields can be nested under: + +[options="header"] +|===== +| Parent fields | Description + +// =============================================================== + + +| <> +| Fields about the client side of a network connection, used with server. + +// =============================================================== + + +| <> +| Fields about the destination side of a network connection, used with source. + +// =============================================================== + + +| <> +| Fields describing the relevant computing instance. + +// =============================================================== + + +| <> +| Fields about the server side of a network connection, used with client. + +// =============================================================== + + +| <> +| Fields about the source side of a network connection, used with destination. + +// =============================================================== + + +|===== +NOTE: The `user` fields can also be used directly as root fields. [[ecs-user-nestings]] -===== Field sets that can be nested under User +The `user` field can be a parent of: [options="header"] |===== -| Nested fields | Description +| Child fields | Description // =============================================================== @@ -4736,7 +5012,7 @@ The user_agent fields normally come from a browser request. They often show up in web service logs coming from the parsed user agent string. -NOTE: See <> information. +NOTE: See information on `user_agent` <>. ==== User agent Field Details @@ -4799,11 +5075,11 @@ example: `12.0` [[ecs-user_agent-nestings]] -===== Field sets that can be nested under User agent +The `user_agent` field can be a parent of: [options="header"] |===== -| Nested fields | Description +| Child fields | Description // =============================================================== diff --git a/scripts/generators/asciidoc_fields.py b/scripts/generators/asciidoc_fields.py index bc9c0ea0ef..68df6126b2 100644 --- a/scripts/generators/asciidoc_fields.py +++ b/scripts/generators/asciidoc_fields.py @@ -50,13 +50,17 @@ def page_field_details(ecs_nested): def render_fieldset(fieldset, ecs_nested): - text = field_details_table_header().format( + text = field_set_title_description_para().format( fieldset_title=fieldset['title'], fieldset_name=fieldset['name'], fieldset_description=render_asciidoc_paragraphs(fieldset['description']), fieldset_reuse_links=render_fieldset_reuse_link(fieldset) ) + text += field_details_table_header().format( + fieldset_title=fieldset['title'] + ) + for field in ecs_helpers.dict_sorted_by_keys(fieldset['fields'], 'flat_name'): if 'original_fieldset' not in field: text += render_field_details_row(field) @@ -90,16 +94,19 @@ def render_field_details_row(field): def render_fieldset_reuse_section(fieldset, ecs_nested): '''Render the section on where field set can be nested, and which field sets can be nested here''' if not ('nestings' in fieldset or 'reusable' in fieldset): - return '' + text = field_reuse_section().format( + reuse_of_fieldset='These fields are not reused.', + fieldset_name=fieldset['name'] + ) + return text text = field_reuse_section().format( - reuse_of_fieldset=render_fieldset_reuses_text(fieldset), + reuse_of_fieldset=render_fieldset_reuses_text(fieldset, ecs_nested), fieldset_name=fieldset['name'] ) if 'nestings' in fieldset: text += nestings_table_header().format( - fieldset_name=fieldset['name'], - fieldset_title=fieldset['title'] + fieldset_name=fieldset['name'] ) nestings = [] for nested_fs_name in sorted(fieldset['nestings']): @@ -115,28 +122,39 @@ def render_fieldset_reuse_section(fieldset, ecs_nested): def render_fieldset_reuse_link(fieldset): '''Render a link to field reuse section, only when appropriate''' if ('nestings' in fieldset or 'reusable' in fieldset): - return 'NOTE: See <> information.'.format(fieldset['name']) + return 'NOTE: See information on `{field_name}` <>.'.format(field_name=fieldset['name']) else: - return 'NOTE: This field set is not reused.' + return '' -def render_fieldset_reuses_text(fieldset): +def render_fieldset_reuses_text(fieldset, ecs_nested): '''Render where a given field set is expected to be reused''' if 'reusable' not in fieldset: return '' + if 'top_level' in fieldset['reusable'] and fieldset['reusable']['top_level']: + text = parent_table_header().format( + fieldset_name=fieldset['name'], + nested_condition='can' + ) + else: + text = parent_table_header().format( + fieldset_name=fieldset['name'], + nested_condition='must' + ) - section_name = fieldset['name'] - sorted_fields = sorted(fieldset['reusable']['expected']) - rendered_fields = map(lambda f: "`{}.{}`".format(f, section_name), sorted_fields) - text = "The `{}` fields are expected to be nested at: {}.\n\n".format( - section_name, ', '.join(rendered_fields)) + for parent_fs_name in sorted(fieldset['reusable']['expected']): + text += render_nesting_row({ + 'flat_nesting': "{}.{}.*".format(parent_fs_name, fieldset['name']), + 'name': parent_fs_name, + 'short': ecs_nested[parent_fs_name]['short'] + }) + text += table_footer() if 'top_level' in fieldset['reusable'] and fieldset['reusable']['top_level']: - template = "Note also that the `{}` fields may be used directly at the top level.\n\n" + text += "NOTE: The `{}` fields can also be used directly as root fields.\n\n".format(fieldset['name']) else: - template = "Note also that the `{}` fields are not expected to " + \ - "be used directly at the top level.\n\n" - text += template.format(section_name) + text += "NOTE: The `{}` fields should *not* be used directly as root fields.\n\n".format(fieldset['name']) + return text @@ -201,8 +219,7 @@ def index_footer(): # Main Fields Table - -def field_details_table_header(): +def field_set_title_description_para(): return ''' [[ecs-{fieldset_name}]] === {fieldset_title} Fields @@ -210,7 +227,11 @@ def field_details_table_header(): {fieldset_description} {fieldset_reuse_links} +''' + +def field_details_table_header(): + return ''' ==== {fieldset_title} Field Details [options="header"] @@ -253,11 +274,27 @@ def field_reuse_section(): def nestings_table_header(): return ''' [[ecs-{fieldset_name}-nestings]] -===== Field sets that can be nested under {fieldset_title} +The `{fieldset_name}` field can be a parent of: + +[options="header"] +|===== +| Child fields | Description + +// =============================================================== + +''' + +# Parent field table + + +def parent_table_header(): + return ''' +[[ecs-{fieldset_name}-parents]] +The `{fieldset_name}` fields {nested_condition} be nested under: [options="header"] |===== -| Nested fields | Description +| Parent fields | Description // =============================================================== From 7eec4941f56e765759ed2aa0345348125f66171b Mon Sep 17 00:00:00 2001 From: Ben Skelker Date: Tue, 8 Oct 2019 13:26:25 +0300 Subject: [PATCH 5/9] Add note when fields cannot be nested --- docs/field-details.asciidoc | 56 +++++++++++++-------------- scripts/generators/asciidoc_fields.py | 10 +++-- 2 files changed, 35 insertions(+), 31 deletions(-) diff --git a/docs/field-details.asciidoc b/docs/field-details.asciidoc index b67fcc2376..b43f9a4b90 100644 --- a/docs/field-details.asciidoc +++ b/docs/field-details.asciidoc @@ -77,7 +77,7 @@ example: `["production", "env2"]` [[ecs-base-reuse]] ==== Field Reuse -These fields are not reused. +These fields are never nested under or a parent of other field sets. [[ecs-agent]] @@ -167,7 +167,7 @@ example: `6.0.0-rc2` [[ecs-agent-reuse]] ==== Field Reuse -These fields are not reused. +These fields are never nested under or a parent of other field sets. [[ecs-as]] @@ -248,7 +248,7 @@ The `as` fields must be nested under: |===== -NOTE: The `as` fields should *not* be used directly as root fields. +NOTE: The `as` fields should *not* be used directly as top-level fields. @@ -444,7 +444,7 @@ The `client` field can be a parent of: |===== - +NOTE: The `client` fields *cannot* be nested under other field sets. [[ecs-cloud]] === Cloud Fields @@ -544,7 +544,7 @@ example: `us-east-1` [[ecs-cloud-reuse]] ==== Field Reuse -These fields are not reused. +These fields are never nested under or a parent of other field sets. [[ecs-container]] @@ -635,7 +635,7 @@ example: `docker` [[ecs-container-reuse]] ==== Field Reuse -These fields are not reused. +These fields are never nested under or a parent of other field sets. [[ecs-destination]] @@ -827,7 +827,7 @@ The `destination` field can be a parent of: |===== - +NOTE: The `destination` fields *cannot* be nested under other field sets. [[ecs-dns]] === DNS Fields @@ -1074,7 +1074,7 @@ example: `answer` [[ecs-dns-reuse]] ==== Field Reuse -These fields are not reused. +These fields are never nested under or a parent of other field sets. [[ecs-ecs]] @@ -1110,7 +1110,7 @@ example: `1.0.0` [[ecs-ecs-reuse]] ==== Field Reuse -These fields are not reused. +These fields are never nested under or a parent of other field sets. [[ecs-error]] @@ -1190,7 +1190,7 @@ example: `java.lang.NullPointerException` [[ecs-error-reuse]] ==== Field Reuse -These fields are not reused. +These fields are never nested under or a parent of other field sets. [[ecs-event]] @@ -1501,7 +1501,7 @@ type: keyword [[ecs-event-reuse]] ==== Field Reuse -These fields are not reused. +These fields are never nested under or a parent of other field sets. [[ecs-file]] @@ -1752,7 +1752,7 @@ The `file` field can be a parent of: |===== - +NOTE: The `file` fields *cannot* be nested under other field sets. [[ecs-geo]] === Geo Fields @@ -1915,7 +1915,7 @@ The `geo` fields must be nested under: |===== -NOTE: The `geo` fields should *not* be used directly as root fields. +NOTE: The `geo` fields should *not* be used directly as top-level fields. @@ -1993,7 +1993,7 @@ The `group` fields can be nested under: |===== -NOTE: The `group` fields can also be used directly as root fields. +NOTE: The `group` fields can also be used directly as top-level fields. @@ -2088,7 +2088,7 @@ The `hash` fields must be nested under: |===== -NOTE: The `hash` fields should *not* be used directly as root fields. +NOTE: The `hash` fields should *not* be used directly as top-level fields. @@ -2258,7 +2258,7 @@ The `host` field can be a parent of: |===== - +NOTE: The `host` fields *cannot* be nested under other field sets. [[ecs-http]] === HTTP Fields @@ -2391,7 +2391,7 @@ example: `1.1` [[ecs-http-reuse]] ==== Field Reuse -These fields are not reused. +These fields are never nested under or a parent of other field sets. [[ecs-log]] @@ -2566,7 +2566,7 @@ example: `Error` [[ecs-log-reuse]] ==== Field Reuse -These fields are not reused. +These fields are never nested under or a parent of other field sets. [[ecs-network]] @@ -2744,7 +2744,7 @@ example: `ipv4` [[ecs-network-reuse]] ==== Field Reuse -These fields are not reused. +These fields are never nested under or a parent of other field sets. [[ecs-observer]] @@ -2900,7 +2900,7 @@ The `observer` field can be a parent of: |===== - +NOTE: The `observer` fields *cannot* be nested under other field sets. [[ecs-organization]] === Organization Fields @@ -2945,7 +2945,7 @@ type: keyword [[ecs-organization-reuse]] ==== Field Reuse -These fields are not reused. +These fields are never nested under or a parent of other field sets. [[ecs-os]] @@ -3064,7 +3064,7 @@ The `os` fields must be nested under: |===== -NOTE: The `os` fields should *not* be used directly as root fields. +NOTE: The `os` fields should *not* be used directly as top-level fields. @@ -3631,7 +3631,7 @@ The `process` field can be a parent of: |===== - +NOTE: The `process` fields *cannot* be nested under other field sets. [[ecs-related]] === Related Fields @@ -3667,7 +3667,7 @@ type: ip [[ecs-related-reuse]] ==== Field Reuse -These fields are not reused. +These fields are never nested under or a parent of other field sets. [[ecs-server]] @@ -3861,7 +3861,7 @@ The `server` field can be a parent of: |===== - +NOTE: The `server` fields *cannot* be nested under other field sets. [[ecs-service]] === Service Fields @@ -3981,7 +3981,7 @@ example: `3.2.4` [[ecs-service-reuse]] ==== Field Reuse -These fields are not reused. +These fields are never nested under or a parent of other field sets. [[ecs-source]] @@ -4845,7 +4845,7 @@ type: keyword [[ecs-url-reuse]] ==== Field Reuse -These fields are not reused. +These fields are never nested under or a parent of other field sets. [[ecs-user]] @@ -4982,7 +4982,7 @@ The `user` fields can be nested under: |===== -NOTE: The `user` fields can also be used directly as root fields. +NOTE: The `user` fields can also be used directly as top-level fields. diff --git a/scripts/generators/asciidoc_fields.py b/scripts/generators/asciidoc_fields.py index 68df6126b2..bb72d7aec9 100644 --- a/scripts/generators/asciidoc_fields.py +++ b/scripts/generators/asciidoc_fields.py @@ -95,7 +95,7 @@ def render_fieldset_reuse_section(fieldset, ecs_nested): '''Render the section on where field set can be nested, and which field sets can be nested here''' if not ('nestings' in fieldset or 'reusable' in fieldset): text = field_reuse_section().format( - reuse_of_fieldset='These fields are not reused.', + reuse_of_fieldset='These fields are never nested under or a parent of other field sets.', fieldset_name=fieldset['name'] ) return text @@ -116,6 +116,10 @@ def render_fieldset_reuse_section(fieldset, ecs_nested): 'short': ecs_nested[nested_fs_name]['short'] }) text += table_footer() + if 'reusable' not in fieldset: + text += "NOTE: The `{}` fields *cannot* be nested under other field sets.".format( + fieldset['name'] + ) return text @@ -151,9 +155,9 @@ def render_fieldset_reuses_text(fieldset, ecs_nested): text += table_footer() if 'top_level' in fieldset['reusable'] and fieldset['reusable']['top_level']: - text += "NOTE: The `{}` fields can also be used directly as root fields.\n\n".format(fieldset['name']) + text += "NOTE: The `{}` fields can also be used directly as top-level fields.\n\n".format(fieldset['name']) else: - text += "NOTE: The `{}` fields should *not* be used directly as root fields.\n\n".format(fieldset['name']) + text += "NOTE: The `{}` fields should *not* be used directly as top-level fields.\n\n".format(fieldset['name']) return text From af119d15b4608ad61b591e39edfa7f72cad574fe Mon Sep 17 00:00:00 2001 From: Ben Skelker Date: Tue, 8 Oct 2019 14:12:25 +0300 Subject: [PATCH 6/9] Rebased with master to add Threat fields --- docs/field-details.asciidoc | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/field-details.asciidoc b/docs/field-details.asciidoc index b43f9a4b90..64f942175c 100644 --- a/docs/field-details.asciidoc +++ b/docs/field-details.asciidoc @@ -3235,6 +3235,12 @@ example: `1.12.9` |===== +[[ecs-package-reuse]] +==== Field Reuse + +These fields are never nested under or a parent of other field sets. + + [[ecs-process]] === Process Fields @@ -4173,7 +4179,7 @@ The `source` field can be a parent of: |===== - +NOTE: The `source` fields *cannot* be nested under other field sets. [[ecs-threat]] === Threat Fields From df6a6256ed99664257b7cd9bcf47b55fee4820dd Mon Sep 17 00:00:00 2001 From: Ben Skelker Date: Tue, 29 Oct 2019 21:25:16 +0200 Subject: [PATCH 7/9] Expands note whenfields can be a child but not a parent --- docs/field-details.asciidoc | 114 ++++++++++++++------------ scripts/generators/asciidoc_fields.py | 28 +++++-- 2 files changed, 83 insertions(+), 59 deletions(-) diff --git a/docs/field-details.asciidoc b/docs/field-details.asciidoc index 64f942175c..d9499f207d 100644 --- a/docs/field-details.asciidoc +++ b/docs/field-details.asciidoc @@ -77,7 +77,7 @@ example: `["production", "env2"]` [[ecs-base-reuse]] ==== Field Reuse -These fields are never nested under or a parent of other field sets. +The `base` field set must *not* be reused as a parent or child of other fields. [[ecs-agent]] @@ -167,7 +167,7 @@ example: `6.0.0-rc2` [[ecs-agent-reuse]] ==== Field Reuse -These fields are never nested under or a parent of other field sets. +The `agent` field set must *not* be reused as a parent or child of other fields. [[ecs-as]] @@ -214,11 +214,11 @@ example: `Google LLC` [[ecs-as-parents]] -The `as` fields must be nested under: +The `as` fields must be a child of: [options="header"] |===== -| Parent fields | Description +| Parent field | Description // =============================================================== @@ -248,7 +248,7 @@ The `as` fields must be nested under: |===== -NOTE: The `as` fields should *not* be used directly as top-level fields. +NOTE: The `as` field set must *not* be used directly as top-level fields. @@ -420,7 +420,7 @@ The `client` field can be a parent of: [options="header"] |===== -| Child fields | Description +| Child field | Description // =============================================================== @@ -444,7 +444,7 @@ The `client` field can be a parent of: |===== -NOTE: The `client` fields *cannot* be nested under other field sets. +NOTE: The `client` field set must *not* be reused as a child of other fields. [[ecs-cloud]] === Cloud Fields @@ -544,7 +544,7 @@ example: `us-east-1` [[ecs-cloud-reuse]] ==== Field Reuse -These fields are never nested under or a parent of other field sets. +The `cloud` field set must *not* be reused as a parent or child of other fields. [[ecs-container]] @@ -635,7 +635,7 @@ example: `docker` [[ecs-container-reuse]] ==== Field Reuse -These fields are never nested under or a parent of other field sets. +The `container` field set must *not* be reused as a parent or child of other fields. [[ecs-destination]] @@ -803,7 +803,7 @@ The `destination` field can be a parent of: [options="header"] |===== -| Child fields | Description +| Child field | Description // =============================================================== @@ -827,7 +827,7 @@ The `destination` field can be a parent of: |===== -NOTE: The `destination` fields *cannot* be nested under other field sets. +NOTE: The `destination` field set must *not* be reused as a child of other fields. [[ecs-dns]] === DNS Fields @@ -1074,7 +1074,7 @@ example: `answer` [[ecs-dns-reuse]] ==== Field Reuse -These fields are never nested under or a parent of other field sets. +The `dns` field set must *not* be reused as a parent or child of other fields. [[ecs-ecs]] @@ -1110,7 +1110,7 @@ example: `1.0.0` [[ecs-ecs-reuse]] ==== Field Reuse -These fields are never nested under or a parent of other field sets. +The `ecs` field set must *not* be reused as a parent or child of other fields. [[ecs-error]] @@ -1190,7 +1190,7 @@ example: `java.lang.NullPointerException` [[ecs-error-reuse]] ==== Field Reuse -These fields are never nested under or a parent of other field sets. +The `error` field set must *not* be reused as a parent or child of other fields. [[ecs-event]] @@ -1501,7 +1501,7 @@ type: keyword [[ecs-event-reuse]] ==== Field Reuse -These fields are never nested under or a parent of other field sets. +The `event` field set must *not* be reused as a parent or child of other fields. [[ecs-file]] @@ -1740,7 +1740,7 @@ The `file` field can be a parent of: [options="header"] |===== -| Child fields | Description +| Child field | Description // =============================================================== @@ -1752,7 +1752,7 @@ The `file` field can be a parent of: |===== -NOTE: The `file` fields *cannot* be nested under other field sets. +NOTE: The `file` field set must *not* be reused as a child of other fields. [[ecs-geo]] === Geo Fields @@ -1869,11 +1869,11 @@ example: `Quebec` [[ecs-geo-parents]] -The `geo` fields must be nested under: +The `geo` fields must be a child of: [options="header"] |===== -| Parent fields | Description +| Parent field | Description // =============================================================== @@ -1915,7 +1915,7 @@ The `geo` fields must be nested under: |===== -NOTE: The `geo` fields should *not* be used directly as top-level fields. +NOTE: The `geo` field set must *not* be used directly as top-level fields. @@ -1977,11 +1977,11 @@ type: keyword [[ecs-group-parents]] -The `group` fields can be nested under: +The `group` fields can be a child of: [options="header"] |===== -| Parent fields | Description +| Parent field | Description // =============================================================== @@ -1993,9 +1993,15 @@ The `group` fields can be nested under: |===== -NOTE: The `group` fields can also be used directly as top-level fields. +[NOTE] +========================= +The `group` field set: +* Can also be used directly as top-level fields. +* Must *not* be reused as a parent of other fields. +========================= + [[ecs-hash]] @@ -2066,11 +2072,11 @@ type: keyword [[ecs-hash-parents]] -The `hash` fields must be nested under: +The `hash` fields must be a child of: [options="header"] |===== -| Parent fields | Description +| Parent field | Description // =============================================================== @@ -2088,7 +2094,7 @@ The `hash` fields must be nested under: |===== -NOTE: The `hash` fields should *not* be used directly as top-level fields. +NOTE: The `hash` field set must *not* be used directly as top-level fields. @@ -2234,7 +2240,7 @@ The `host` field can be a parent of: [options="header"] |===== -| Child fields | Description +| Child field | Description // =============================================================== @@ -2258,7 +2264,7 @@ The `host` field can be a parent of: |===== -NOTE: The `host` fields *cannot* be nested under other field sets. +NOTE: The `host` field set must *not* be reused as a child of other fields. [[ecs-http]] === HTTP Fields @@ -2391,7 +2397,7 @@ example: `1.1` [[ecs-http-reuse]] ==== Field Reuse -These fields are never nested under or a parent of other field sets. +The `http` field set must *not* be reused as a parent or child of other fields. [[ecs-log]] @@ -2566,7 +2572,7 @@ example: `Error` [[ecs-log-reuse]] ==== Field Reuse -These fields are never nested under or a parent of other field sets. +The `log` field set must *not* be reused as a parent or child of other fields. [[ecs-network]] @@ -2744,7 +2750,7 @@ example: `ipv4` [[ecs-network-reuse]] ==== Field Reuse -These fields are never nested under or a parent of other field sets. +The `network` field set must *not* be reused as a parent or child of other fields. [[ecs-observer]] @@ -2882,7 +2888,7 @@ The `observer` field can be a parent of: [options="header"] |===== -| Child fields | Description +| Child field | Description // =============================================================== @@ -2900,7 +2906,7 @@ The `observer` field can be a parent of: |===== -NOTE: The `observer` fields *cannot* be nested under other field sets. +NOTE: The `observer` field set must *not* be reused as a child of other fields. [[ecs-organization]] === Organization Fields @@ -2945,7 +2951,7 @@ type: keyword [[ecs-organization-reuse]] ==== Field Reuse -These fields are never nested under or a parent of other field sets. +The `organization` field set must *not* be reused as a parent or child of other fields. [[ecs-os]] @@ -3036,11 +3042,11 @@ example: `10.14.1` [[ecs-os-parents]] -The `os` fields must be nested under: +The `os` fields must be a child of: [options="header"] |===== -| Parent fields | Description +| Parent field | Description // =============================================================== @@ -3064,7 +3070,7 @@ The `os` fields must be nested under: |===== -NOTE: The `os` fields should *not* be used directly as top-level fields. +NOTE: The `os` field set must *not* be used directly as top-level fields. @@ -3238,7 +3244,7 @@ example: `1.12.9` [[ecs-package-reuse]] ==== Field Reuse -These fields are never nested under or a parent of other field sets. +The `package` field set must *not* be reused as a parent or child of other fields. [[ecs-process]] @@ -3625,7 +3631,7 @@ The `process` field can be a parent of: [options="header"] |===== -| Child fields | Description +| Child field | Description // =============================================================== @@ -3637,7 +3643,7 @@ The `process` field can be a parent of: |===== -NOTE: The `process` fields *cannot* be nested under other field sets. +NOTE: The `process` field set must *not* be reused as a child of other fields. [[ecs-related]] === Related Fields @@ -3673,7 +3679,7 @@ type: ip [[ecs-related-reuse]] ==== Field Reuse -These fields are never nested under or a parent of other field sets. +The `related` field set must *not* be reused as a parent or child of other fields. [[ecs-server]] @@ -3843,7 +3849,7 @@ The `server` field can be a parent of: [options="header"] |===== -| Child fields | Description +| Child field | Description // =============================================================== @@ -3867,7 +3873,7 @@ The `server` field can be a parent of: |===== -NOTE: The `server` fields *cannot* be nested under other field sets. +NOTE: The `server` field set must *not* be reused as a child of other fields. [[ecs-service]] === Service Fields @@ -3987,7 +3993,7 @@ example: `3.2.4` [[ecs-service-reuse]] ==== Field Reuse -These fields are never nested under or a parent of other field sets. +The `service` field set must *not* be reused as a parent or child of other fields. [[ecs-source]] @@ -4155,7 +4161,7 @@ The `source` field can be a parent of: [options="header"] |===== -| Child fields | Description +| Child field | Description // =============================================================== @@ -4179,7 +4185,7 @@ The `source` field can be a parent of: |===== -NOTE: The `source` fields *cannot* be nested under other field sets. +NOTE: The `source` field set must *not* be reused as a child of other fields. [[ecs-threat]] === Threat Fields @@ -4851,7 +4857,7 @@ type: keyword [[ecs-url-reuse]] ==== Field Reuse -These fields are never nested under or a parent of other field sets. +The `url` field set must *not* be reused as a parent or child of other fields. [[ecs-user]] @@ -4948,11 +4954,11 @@ example: `albert` [[ecs-user-parents]] -The `user` fields can be nested under: +The `user` fields can be a child of: [options="header"] |===== -| Parent fields | Description +| Parent field | Description // =============================================================== @@ -4988,7 +4994,7 @@ The `user` fields can be nested under: |===== -NOTE: The `user` fields can also be used directly as top-level fields. +NOTE: The `user` field set can also be used directly as top-level fields. @@ -4998,7 +5004,7 @@ The `user` field can be a parent of: [options="header"] |===== -| Child fields | Description +| Child field | Description // =============================================================== @@ -5085,7 +5091,7 @@ The `user_agent` field can be a parent of: [options="header"] |===== -| Child fields | Description +| Child field | Description // =============================================================== @@ -5097,6 +5103,7 @@ The `user_agent` field can be a parent of: |===== +<<<<<<< HEAD [[ecs-vulnerability]] === Vulnerability Fields @@ -5267,3 +5274,6 @@ example: `Critical` // =============================================================== |===== +======= +NOTE: The `user_agent` field set must *not* be reused as a child of other fields. +>>>>>>> Expands note whenfields can be a child but not a parent diff --git a/scripts/generators/asciidoc_fields.py b/scripts/generators/asciidoc_fields.py index bb72d7aec9..461f414e6f 100644 --- a/scripts/generators/asciidoc_fields.py +++ b/scripts/generators/asciidoc_fields.py @@ -95,7 +95,8 @@ def render_fieldset_reuse_section(fieldset, ecs_nested): '''Render the section on where field set can be nested, and which field sets can be nested here''' if not ('nestings' in fieldset or 'reusable' in fieldset): text = field_reuse_section().format( - reuse_of_fieldset='These fields are never nested under or a parent of other field sets.', + reuse_of_fieldset='The `{}` field set must *not* be reused as a parent or child of other fields.'.format( + fieldset['name']), fieldset_name=fieldset['name'] ) return text @@ -117,7 +118,7 @@ def render_fieldset_reuse_section(fieldset, ecs_nested): }) text += table_footer() if 'reusable' not in fieldset: - text += "NOTE: The `{}` fields *cannot* be nested under other field sets.".format( + text += "NOTE: The `{}` field set must *not* be reused as a child of other fields.".format( fieldset['name'] ) return text @@ -155,9 +156,22 @@ def render_fieldset_reuses_text(fieldset, ecs_nested): text += table_footer() if 'top_level' in fieldset['reusable'] and fieldset['reusable']['top_level']: - text += "NOTE: The `{}` fields can also be used directly as top-level fields.\n\n".format(fieldset['name']) + if 'nestings' not in fieldset: + text += ''' +[NOTE] +========================= +The `{}` field set: + +* Can also be used directly as top-level fields. +* Must *not* be reused as a parent of other fields. +========================= + '''.format( + fieldset['name']) + else: + text += "NOTE: The `{}` field set can also be used directly as top-level fields.\n\n".format( + fieldset['name']) else: - text += "NOTE: The `{}` fields should *not* be used directly as top-level fields.\n\n".format(fieldset['name']) + text += "NOTE: The `{}` field set must *not* be used directly as top-level fields.\n\n".format(fieldset['name']) return text @@ -282,7 +296,7 @@ def nestings_table_header(): [options="header"] |===== -| Child fields | Description +| Child field | Description // =============================================================== @@ -294,11 +308,11 @@ def nestings_table_header(): def parent_table_header(): return ''' [[ecs-{fieldset_name}-parents]] -The `{fieldset_name}` fields {nested_condition} be nested under: +The `{fieldset_name}` fields {nested_condition} be a child of: [options="header"] |===== -| Parent fields | Description +| Parent field | Description // =============================================================== From 9b86472c1dbde89da4f9832634b39eb2c2313228 Mon Sep 17 00:00:00 2001 From: Ben Skelker Date: Tue, 29 Oct 2019 21:30:44 +0200 Subject: [PATCH 8/9] Tries to imrpove wording --- docs/field-details.asciidoc | 12 ++++++------ scripts/generators/asciidoc_fields.py | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/field-details.asciidoc b/docs/field-details.asciidoc index d9499f207d..b105d1aa96 100644 --- a/docs/field-details.asciidoc +++ b/docs/field-details.asciidoc @@ -214,7 +214,7 @@ example: `Google LLC` [[ecs-as-parents]] -The `as` fields must be a child of: +The `as` field set must be a child of: [options="header"] |===== @@ -1869,7 +1869,7 @@ example: `Quebec` [[ecs-geo-parents]] -The `geo` fields must be a child of: +The `geo` field set must be a child of: [options="header"] |===== @@ -1977,7 +1977,7 @@ type: keyword [[ecs-group-parents]] -The `group` fields can be a child of: +The `group` field set can be a child of: [options="header"] |===== @@ -2072,7 +2072,7 @@ type: keyword [[ecs-hash-parents]] -The `hash` fields must be a child of: +The `hash` field set must be a child of: [options="header"] |===== @@ -3042,7 +3042,7 @@ example: `10.14.1` [[ecs-os-parents]] -The `os` fields must be a child of: +The `os` field set must be a child of: [options="header"] |===== @@ -4954,7 +4954,7 @@ example: `albert` [[ecs-user-parents]] -The `user` fields can be a child of: +The `user` field set can be a child of: [options="header"] |===== diff --git a/scripts/generators/asciidoc_fields.py b/scripts/generators/asciidoc_fields.py index 461f414e6f..3c7bf6840d 100644 --- a/scripts/generators/asciidoc_fields.py +++ b/scripts/generators/asciidoc_fields.py @@ -308,7 +308,7 @@ def nestings_table_header(): def parent_table_header(): return ''' [[ecs-{fieldset_name}-parents]] -The `{fieldset_name}` fields {nested_condition} be a child of: +The `{fieldset_name}` field set {nested_condition} be a child of: [options="header"] |===== From 973ae309eaaf51880675ba6dd7bf8317f17b865b Mon Sep 17 00:00:00 2001 From: Ben Skelker Date: Sun, 1 Dec 2019 10:44:17 +0200 Subject: [PATCH 9/9] rebase from master --- docs/field-details.asciidoc | 40 +++++++++++++++++++++++++++---------- 1 file changed, 30 insertions(+), 10 deletions(-) diff --git a/docs/field-details.asciidoc b/docs/field-details.asciidoc index b105d1aa96..3f45af94a2 100644 --- a/docs/field-details.asciidoc +++ b/docs/field-details.asciidoc @@ -3080,7 +3080,7 @@ NOTE: The `os` field set must *not* be used directly as top-level fields. These fields contain information about an installed software package. It contains general information about a package, such as name, version or size. It also contains installation details, such as time or location. -NOTE: This field set is not reused. + ==== Package Field Details @@ -4193,7 +4193,7 @@ Fields to classify events and alerts according to a threat taxonomy such as the These fields are for users to classify alerts from all of their sources (e.g. IDS, NGFW, etc.) within a common taxonomy. The threat.tactic.* are meant to capture the high level category of the threat (e.g. "impact"). The threat.technique.* fields are meant to capture which kind of approach is used by this detected threat, to accomplish the goal (e.g. "endpoint denial of service"). -NOTE: This field set is not reused. + ==== Threat Field Details @@ -4282,6 +4282,12 @@ example: `https://attack.mitre.org/techniques/T1499/` |===== +[[ecs-threat-reuse]] +==== Field Reuse + +The `threat` field set must *not* be reused as a parent or child of other fields. + + [[ecs-tls]] === TLS Fields @@ -4629,12 +4635,18 @@ example: `tls` |===== +[[ecs-tls-reuse]] +==== Field Reuse + +The `tls` field set must *not* be reused as a parent or child of other fields. + + [[ecs-tracing]] === Tracing Fields Distributed tracing makes it possible to analyze performance throughout a microservice architecture all in one view. This is accomplished by tracing all of the requests - from the initial web request in the front-end service - to queries made through multiple back-end services. -NOTE: This field set is not reused. + ==== Tracing Field Details @@ -4672,12 +4684,18 @@ example: `00f067aa0ba902b7` |===== +[[ecs-tracing-reuse]] +==== Field Reuse + +The `tracing` field set must *not* be reused as a parent or child of other fields. + + [[ecs-url]] === URL Fields URL fields provide support for complete or partial URLs, and supports the breaking down into scheme, domain, path, and so on. -NOTE: This field set is not reused. + ==== URL Field Details @@ -5103,14 +5121,13 @@ The `user_agent` field can be a parent of: |===== -<<<<<<< HEAD - +NOTE: The `user_agent` field set must *not* be reused as a child of other fields. [[ecs-vulnerability]] === Vulnerability Fields The vulnerability fields describe information about a vulnerability that is relevant to an event. -NOTE: This field set is not reused. + ==== Vulnerability Field Details @@ -5274,6 +5291,9 @@ example: `Critical` // =============================================================== |===== -======= -NOTE: The `user_agent` field set must *not* be reused as a child of other fields. ->>>>>>> Expands note whenfields can be a child but not a parent + +[[ecs-vulnerability-reuse]] +==== Field Reuse + +The `vulnerability` field set must *not* be reused as a parent or child of other fields. +