From 84f656f92fe9df6fd05e1222895fe074f8e7b8c0 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Mon, 4 Apr 2022 18:36:46 -0400 Subject: [PATCH 01/16] Add fields honored by Fleet --- versions/1/data_stream/fields/fields.spec.yml | 236 +++++++++++++++--- 1 file changed, 204 insertions(+), 32 deletions(-) diff --git a/versions/1/data_stream/fields/fields.spec.yml b/versions/1/data_stream/fields/fields.spec.yml index e534f436a..7e9358839 100644 --- a/versions/1/data_stream/fields/fields.spec.yml +++ b/versions/1/data_stream/fields/fields.spec.yml @@ -6,66 +6,82 @@ spec: type: array items: type: object + # TODO: We should disable additionalProperties. There are properties used in + # elastic/integrations that have no purpose or are typos. additionalProperties: true properties: name: - description: Name of field - type: string - title: - description: Title of field + description: > + Name of field. Names containing dots are automatically split into + sub-fields. type: string + # TODO: Should we allow wildcards if they are not implemented in Fleet? + pattern: '^[_@A-Za-z0-9]+(\.[_@A-Za-z0-9]+)*$' type: description: Datatype of field type: string enum: - alias - - histogram - - constant_keyword - - text - - match_only_text - - keyword - - long - - integer - - short + - array + - binary + - boolean - byte - - double - - float - - half_float - - scaled_float + - constant_keyword - date - date_nanos - - boolean - - binary - - integer_range - - float_range - - long_range - - double_range - date_range - - ip_range - - group + - double + - double_range + - flattened + - float + - float_range - geo_point - - object + - group + - half_float + - histogram + - integer # Fleet always maps this as 'long'. + - integer_range - ip + - ip_range + - keyword + - long + - long_range + - match_only_text - nested - - array - - flattened - - wildcard - - version + - object + - scaled_float + - short + - text - unsigned_long + - version + - wildcard + description: description: Short description of field type: string + value: description: The value to associate with a constant_keyword field. type: string + metric_type: - description: Metric type + description: > + The metric type of a numeric field. This is attached to the + field as metadata (via `meta`). A gauge is a single-value measurement + that can go up or down over time, such as a temperature. A counter is + a single-value cumulative counter that only goes up, such as the + number of requests processed by a web server. By default, no metric + type is associated with a field. type: string enum: - counter - gauge + unit: - description: Unit + description: > + Unit type to associate with a numeric field. This is attached to the + field as metadata (via `meta`). By default, a field does not have a + unit. The convention for percents is to use value 1 to mean 100%. type: string enum: - byte @@ -77,22 +93,178 @@ spec: - ms - micros - nanos + dimension: description: Declare a field as dimension of time series type: boolean default: false + pattern: - description: Regular expression pattern of the field value + description: > + Regular expression pattern matching the allowed values for the field. + This is used for development-time data validation. type: string examples: - '^[a-zA-Z]$' + external: description: External source reference type: string enum: - ecs + fields: description: Sub-fields, when type is group $ref: "#" # JSON-schema syntax for pointing to the root of the schema + + doc_values: + description: > + Controls whether doc values are enabled for a field. All fields which + support doc values have them enabled by default. If you are sure that + you don’t need to sort or aggregate on a field, or access the field + value from a script, you can disable doc values in order to save disk + space. You cannot disable doc values for wildcard fields. + type: boolean + + index: + description: > + The index option controls whether field values are indexed. Fields + that are not indexed are typically not queryable. + type: boolean + default: true + + copy_to: + description: > + The copy_to parameter allows you to copy the values of multiple fields + into a group field, which can then be queried as a single field. + type: string + + enabled: + description: > + The enabled setting, which can be applied only to the top-level + mapping definition and to object fields, causes Elasticsearch to skip + parsing of the contents of the field entirely. The JSON can still be + retrieved from the _source field, but it is not searchable or stored + in any other way. + type: boolean + + dynamic: + description: > + The dynamic parameter controls whether new fields are added + dynamically. It accepts the following values: + + true - New fields are added to the mapping (default). + + runtime - New fields are added to the mapping as runtime fields. These + fields are not indexed, and are loaded from _source at query time. + + false - New fields are ignored. These fields will not be indexed or + searchable, but will still appear in the _source field of returned + hits. These fields will not be added to the mapping, and new fields + must be added explicitly. + + strict - If new fields are detected, an exception is thrown and the + document is rejected. New fields must be explicitly added to the + mapping. + default: true + enum: + - true + - false + - strict + - runtime # Not supported at this time by Fleet. + + scaling_factor: + description: > + The scaling factor to use when encoding values. Values will be + multiplied by this factor at index time and rounded to the closest + long value. For instance, a scaled_float with a scaling_factor of 10 + would internally store 2.34 as 23 and all search-time operations + (queries, aggregations, sorting) will behave as if the document had a + value of 2.3. High values of scaling_factor improve accuracy but also + increase space requirements. Only valid for 'type: scaled_float'. + type: integer + default: 1000 + + analyzer: + description: > + Name of the analyzer to use for indexing. Unless search_analyzer is + specified this analyzer is used for both indexing and searching. + Only valid for 'type: text'. + type: string + + search_analyzer: + description: > + Name of the analyzer to use for searching. Only valid for 'type: text'. + type: string + + multi_fields: + description: > + It is often useful to index the same field in different ways for + different purposes. This is the purpose of multi-fields. For instance, + a string field could be mapped as a text field for full-text search, + and as a keyword field for sorting or aggregations. + + Only honored for 'type: text/keyword/wildcard'. + $ref: "#" # JSON-schema syntax for pointing to the root of the schema + + # TODO: Fleet only honors this for 'type: wildcard'. That seems wrong. + null_value: + description: > + The null_value parameter allows you to replace explicit null values + with the specified value so that it can be indexed and searched. + + A null value cannot be indexed or searched. When a field is set to + null, (or an empty array or an array of null values) it is treated as + though that field has no values. + + The null_value needs to be the same data type as the field. For + instance, a long field cannot have a string null_value. + + The null_value only influences how data is indexed, it doesn’t modify + the _source document. + examples: + - "NULL" + + ignore_above: + description: > + Strings longer than the ignore_above setting will not be indexed or + stored. For arrays of strings, ignore_above will be applied for each + array element separately and string elements longer than ignore_above + will not be indexed or stored. Only honored for + 'type: keyword/wildcard'. Defaults to 1024. + type: integer + default: 1024 + + object_type: + description: > + Required for `type: array` to specify the data type in the array. + type: string + + path: + description: > + For alias type fields this is the path to the target field. Note that + this must be the full path, including any parent objects + (e.g. object1.object2.field). + type: string + + normalizer: + description: > + Specifies the name of a normalizer to apply to keyword fields. A + simple normalizer called lowercase ships with elasticsearch and can be + used. Custom normalizers can be defined as part of analysis index + settings. + type: string + + # TODO: This two are read by Kibana, but AFAICT they have no use in + # Elasticsearch. And nothing in elastic/integrations uses it. + include_in_parent: + description: > + Only valid for 'type: group-nested/nested'. + type: boolean + include_in_root: + description: > + Only valid for 'type: group-nested/nested'. + type: boolean + required: - name From e56cc7f0b88875f846e50f1a6681b2b6c46ff517 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Wed, 6 Apr 2022 17:00:04 -0400 Subject: [PATCH 02/16] Revert sorting of types --- versions/1/data_stream/fields/fields.spec.yml | 50 +++++++++---------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/versions/1/data_stream/fields/fields.spec.yml b/versions/1/data_stream/fields/fields.spec.yml index 7e9358839..c90585ebd 100644 --- a/versions/1/data_stream/fields/fields.spec.yml +++ b/versions/1/data_stream/fields/fields.spec.yml @@ -22,39 +22,39 @@ spec: type: string enum: - alias - - array - - binary - - boolean - - byte + - histogram - constant_keyword - - date - - date_nanos - - date_range + - text + - match_only_text + - keyword + - long + - integer # Fleet always maps this as 'long'. + - short + - byte - double - - double_range - - flattened - float - - float_range - - geo_point - - group - half_float - - histogram - - integer # Fleet always maps this as 'long'. + - scaled_float + - date + - date_nanos + - boolean + - binary - integer_range - - ip - - ip_range - - keyword - - long + - float_range - long_range - - match_only_text - - nested + - double_range + - date_range + - ip_range + - group + - geo_point - object - - scaled_float - - short - - text - - unsigned_long - - version + - ip + - nested + - array + - flattened - wildcard + - version + - unsigned_long description: description: Short description of field From 00c5fa112256011ec6e777e331489c9d7bc3b367 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Wed, 6 Apr 2022 17:06:30 -0400 Subject: [PATCH 03/16] Add test to check that asterisk is accepted in field name --- test/packages/good/data_stream/foo/fields/some_fields.yml | 2 ++ versions/1/data_stream/fields/fields.spec.yml | 5 +---- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/test/packages/good/data_stream/foo/fields/some_fields.yml b/test/packages/good/data_stream/foo/fields/some_fields.yml index 099bb968f..53a630910 100644 --- a/test/packages/good/data_stream/foo/fields/some_fields.yml +++ b/test/packages/good/data_stream/foo/fields/some_fields.yml @@ -35,3 +35,5 @@ - name: error.message description: Error message. type: match_only_text +- name: metric.*_bytes + type: long diff --git a/versions/1/data_stream/fields/fields.spec.yml b/versions/1/data_stream/fields/fields.spec.yml index c90585ebd..aa1515584 100644 --- a/versions/1/data_stream/fields/fields.spec.yml +++ b/versions/1/data_stream/fields/fields.spec.yml @@ -6,8 +6,6 @@ spec: type: array items: type: object - # TODO: We should disable additionalProperties. There are properties used in - # elastic/integrations that have no purpose or are typos. additionalProperties: true properties: name: @@ -15,8 +13,7 @@ spec: Name of field. Names containing dots are automatically split into sub-fields. type: string - # TODO: Should we allow wildcards if they are not implemented in Fleet? - pattern: '^[_@A-Za-z0-9]+(\.[_@A-Za-z0-9]+)*$' + pattern: '^[*_@A-Za-z0-9]+(\.[*_@A-Za-z0-9]+)*$' type: description: Datatype of field type: string From afccc994644f15bdf8cbbf7c0f9cec92ed7cb4fc Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Wed, 6 Apr 2022 17:20:21 -0400 Subject: [PATCH 04/16] Disallow additional properties in fields files --- versions/1/data_stream/fields/fields.spec.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versions/1/data_stream/fields/fields.spec.yml b/versions/1/data_stream/fields/fields.spec.yml index aa1515584..af272a179 100644 --- a/versions/1/data_stream/fields/fields.spec.yml +++ b/versions/1/data_stream/fields/fields.spec.yml @@ -6,7 +6,7 @@ spec: type: array items: type: object - additionalProperties: true + additionalProperties: false properties: name: description: > From b9b5fc1241e5ca24d72d35221691b7b1028578c3 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Wed, 6 Apr 2022 17:20:53 -0400 Subject: [PATCH 05/16] Remove level, group, title properties from test files --- .../bad_deploy_variants/data_stream/foo/fields/fields.yml | 3 --- .../deploy_docker/data_stream/foo/fields/fields.yml | 3 --- .../deploy_terraform/data_stream/foo/fields/fields.yml | 3 --- .../docs_extra_files/data_stream/pe/fields/some_fields.yml | 6 ------ test/packages/good/data_stream/foo/fields/some_fields.yml | 6 ------ .../data_stream/hidden_data_stream/fields/some_fields.yml | 3 --- .../good/data_stream/ilm_policy/fields/some_fields.yml | 3 --- test/packages/good/data_stream/pe/fields/some_fields.yml | 6 ------ .../good/data_stream/skipped_tests/fields/some_fields.yml | 6 ------ .../data_stream/pe/fields/some_fields.yml | 6 ------ .../data_stream/foo/fields/some_fields.yml | 6 ------ .../data_stream/foo/fields/some_fields.yml | 6 ------ 12 files changed, 57 deletions(-) diff --git a/test/packages/bad_deploy_variants/data_stream/foo/fields/fields.yml b/test/packages/bad_deploy_variants/data_stream/foo/fields/fields.yml index 6e427ddf6..2f58defee 100644 --- a/test/packages/bad_deploy_variants/data_stream/foo/fields/fields.yml +++ b/test/packages/bad_deploy_variants/data_stream/foo/fields/fields.yml @@ -1,10 +1,7 @@ - name: source - title: Source - group: 2 type: group fields: - name: geo.city_name - level: core type: keyword description: City name. ignore_above: 1024 \ No newline at end of file diff --git a/test/packages/deploy_docker/data_stream/foo/fields/fields.yml b/test/packages/deploy_docker/data_stream/foo/fields/fields.yml index 6e427ddf6..2f58defee 100644 --- a/test/packages/deploy_docker/data_stream/foo/fields/fields.yml +++ b/test/packages/deploy_docker/data_stream/foo/fields/fields.yml @@ -1,10 +1,7 @@ - name: source - title: Source - group: 2 type: group fields: - name: geo.city_name - level: core type: keyword description: City name. ignore_above: 1024 \ No newline at end of file diff --git a/test/packages/deploy_terraform/data_stream/foo/fields/fields.yml b/test/packages/deploy_terraform/data_stream/foo/fields/fields.yml index 6e427ddf6..2f58defee 100644 --- a/test/packages/deploy_terraform/data_stream/foo/fields/fields.yml +++ b/test/packages/deploy_terraform/data_stream/foo/fields/fields.yml @@ -1,10 +1,7 @@ - name: source - title: Source - group: 2 type: group fields: - name: geo.city_name - level: core type: keyword description: City name. ignore_above: 1024 \ No newline at end of file diff --git a/test/packages/docs_extra_files/data_stream/pe/fields/some_fields.yml b/test/packages/docs_extra_files/data_stream/pe/fields/some_fields.yml index e6e1d439f..4adfa4af4 100644 --- a/test/packages/docs_extra_files/data_stream/pe/fields/some_fields.yml +++ b/test/packages/docs_extra_files/data_stream/pe/fields/some_fields.yml @@ -1,24 +1,18 @@ - name: source - title: Source - group: 2 type: group fields: - name: geo.city_name - level: core type: keyword description: City name. ignore_above: 1024 - name: geo.location - level: core type: geo_point description: Longitude and latitude. - name: geo.region_iso_code - level: core type: keyword description: Region ISO code. ignore_above: 1024 - name: geo.region_name - level: core type: keyword description: Region name. ignore_above: 1024 \ No newline at end of file diff --git a/test/packages/good/data_stream/foo/fields/some_fields.yml b/test/packages/good/data_stream/foo/fields/some_fields.yml index 53a630910..2c32949a0 100644 --- a/test/packages/good/data_stream/foo/fields/some_fields.yml +++ b/test/packages/good/data_stream/foo/fields/some_fields.yml @@ -1,24 +1,18 @@ - name: source - title: Source - group: 2 type: group fields: - name: geo.city_name - level: core type: keyword description: City name. ignore_above: 1024 - name: geo.location - level: core type: geo_point description: Longitude and latitude. - name: geo.region_iso_code - level: core type: keyword description: Region ISO code. ignore_above: 1024 - name: geo.region_name - level: core type: keyword description: Region name. ignore_above: 1024 diff --git a/test/packages/good/data_stream/hidden_data_stream/fields/some_fields.yml b/test/packages/good/data_stream/hidden_data_stream/fields/some_fields.yml index 4f46a95bf..137f10b8f 100644 --- a/test/packages/good/data_stream/hidden_data_stream/fields/some_fields.yml +++ b/test/packages/good/data_stream/hidden_data_stream/fields/some_fields.yml @@ -1,10 +1,7 @@ - name: source - title: Source - group: 2 type: group fields: - name: geo.city_name - level: core type: keyword description: City name. ignore_above: 1024 diff --git a/test/packages/good/data_stream/ilm_policy/fields/some_fields.yml b/test/packages/good/data_stream/ilm_policy/fields/some_fields.yml index f9b0e048d..db268a685 100644 --- a/test/packages/good/data_stream/ilm_policy/fields/some_fields.yml +++ b/test/packages/good/data_stream/ilm_policy/fields/some_fields.yml @@ -1,10 +1,7 @@ - name: source - title: Source - group: 2 type: group fields: - name: geo.city_name - level: core type: keyword description: City name. ignore_above: 1024 diff --git a/test/packages/good/data_stream/pe/fields/some_fields.yml b/test/packages/good/data_stream/pe/fields/some_fields.yml index e6e1d439f..4adfa4af4 100644 --- a/test/packages/good/data_stream/pe/fields/some_fields.yml +++ b/test/packages/good/data_stream/pe/fields/some_fields.yml @@ -1,24 +1,18 @@ - name: source - title: Source - group: 2 type: group fields: - name: geo.city_name - level: core type: keyword description: City name. ignore_above: 1024 - name: geo.location - level: core type: geo_point description: Longitude and latitude. - name: geo.region_iso_code - level: core type: keyword description: Region ISO code. ignore_above: 1024 - name: geo.region_name - level: core type: keyword description: Region name. ignore_above: 1024 \ No newline at end of file diff --git a/test/packages/good/data_stream/skipped_tests/fields/some_fields.yml b/test/packages/good/data_stream/skipped_tests/fields/some_fields.yml index bde93f380..61834719e 100644 --- a/test/packages/good/data_stream/skipped_tests/fields/some_fields.yml +++ b/test/packages/good/data_stream/skipped_tests/fields/some_fields.yml @@ -1,24 +1,18 @@ - name: source - title: Source - group: 2 type: group fields: - name: geo.city_name - level: core type: keyword description: City name. ignore_above: 1024 - name: geo.location - level: core type: geo_point description: Longitude and latitude. - name: geo.region_iso_code - level: core type: keyword description: Region ISO code. ignore_above: 1024 - name: geo.region_name - level: core type: keyword description: Region name. ignore_above: 1024 diff --git a/test/packages/missing_image_files/data_stream/pe/fields/some_fields.yml b/test/packages/missing_image_files/data_stream/pe/fields/some_fields.yml index e6e1d439f..4adfa4af4 100644 --- a/test/packages/missing_image_files/data_stream/pe/fields/some_fields.yml +++ b/test/packages/missing_image_files/data_stream/pe/fields/some_fields.yml @@ -1,24 +1,18 @@ - name: source - title: Source - group: 2 type: group fields: - name: geo.city_name - level: core type: keyword description: City name. ignore_above: 1024 - name: geo.location - level: core type: geo_point description: Longitude and latitude. - name: geo.region_iso_code - level: core type: keyword description: Region ISO code. ignore_above: 1024 - name: geo.region_name - level: core type: keyword description: Region name. ignore_above: 1024 \ No newline at end of file diff --git a/test/packages/missing_pipeline_dashes/data_stream/foo/fields/some_fields.yml b/test/packages/missing_pipeline_dashes/data_stream/foo/fields/some_fields.yml index e6e1d439f..4adfa4af4 100644 --- a/test/packages/missing_pipeline_dashes/data_stream/foo/fields/some_fields.yml +++ b/test/packages/missing_pipeline_dashes/data_stream/foo/fields/some_fields.yml @@ -1,24 +1,18 @@ - name: source - title: Source - group: 2 type: group fields: - name: geo.city_name - level: core type: keyword description: City name. ignore_above: 1024 - name: geo.location - level: core type: geo_point description: Longitude and latitude. - name: geo.region_iso_code - level: core type: keyword description: Region ISO code. ignore_above: 1024 - name: geo.region_name - level: core type: keyword description: Region name. ignore_above: 1024 \ No newline at end of file diff --git a/test/packages/missing_required_fields/data_stream/foo/fields/some_fields.yml b/test/packages/missing_required_fields/data_stream/foo/fields/some_fields.yml index 099bb968f..643aa7c15 100644 --- a/test/packages/missing_required_fields/data_stream/foo/fields/some_fields.yml +++ b/test/packages/missing_required_fields/data_stream/foo/fields/some_fields.yml @@ -1,24 +1,18 @@ - name: source - title: Source - group: 2 type: group fields: - name: geo.city_name - level: core type: keyword description: City name. ignore_above: 1024 - name: geo.location - level: core type: geo_point description: Longitude and latitude. - name: geo.region_iso_code - level: core type: keyword description: Region ISO code. ignore_above: 1024 - name: geo.region_name - level: core type: keyword description: Region name. ignore_above: 1024 From 92959995c33f8d06a92d4a8ae999db2613e486ad Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Wed, 6 Apr 2022 17:22:35 -0400 Subject: [PATCH 06/16] Remove include_in_parent, include_in_root We'll need to investigate why these are in Fleet. Then either add them here or remove them from Fleet. --- versions/1/data_stream/fields/fields.spec.yml | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/versions/1/data_stream/fields/fields.spec.yml b/versions/1/data_stream/fields/fields.spec.yml index af272a179..30603291f 100644 --- a/versions/1/data_stream/fields/fields.spec.yml +++ b/versions/1/data_stream/fields/fields.spec.yml @@ -251,17 +251,6 @@ spec: used. Custom normalizers can be defined as part of analysis index settings. type: string - - # TODO: This two are read by Kibana, but AFAICT they have no use in - # Elasticsearch. And nothing in elastic/integrations uses it. - include_in_parent: - description: > - Only valid for 'type: group-nested/nested'. - type: boolean - include_in_root: - description: > - Only valid for 'type: group-nested/nested'. - type: boolean - + required: - name From f8fa2ba88d3149b054fd696a51d21eb74a6af57d Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Wed, 6 Apr 2022 17:28:42 -0400 Subject: [PATCH 07/16] Update multi_fields to mention match_only_text --- versions/1/data_stream/fields/fields.spec.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/versions/1/data_stream/fields/fields.spec.yml b/versions/1/data_stream/fields/fields.spec.yml index 30603291f..6eede08a3 100644 --- a/versions/1/data_stream/fields/fields.spec.yml +++ b/versions/1/data_stream/fields/fields.spec.yml @@ -201,10 +201,10 @@ spec: a string field could be mapped as a text field for full-text search, and as a keyword field for sorting or aggregations. - Only honored for 'type: text/keyword/wildcard'. + Fleet honors this for `keyword`, `match_only_text`, `text`, and + `wildcard` types. $ref: "#" # JSON-schema syntax for pointing to the root of the schema - # TODO: Fleet only honors this for 'type: wildcard'. That seems wrong. null_value: description: > The null_value parameter allows you to replace explicit null values @@ -251,6 +251,6 @@ spec: used. Custom normalizers can be defined as part of analysis index settings. type: string - + required: - name From 23bf6786c490b996f0cbb4862aab18d2207085e7 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Wed, 6 Apr 2022 17:42:40 -0400 Subject: [PATCH 08/16] Improve ignore_above description --- versions/1/data_stream/fields/fields.spec.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/versions/1/data_stream/fields/fields.spec.yml b/versions/1/data_stream/fields/fields.spec.yml index 6eede08a3..c1d67e9be 100644 --- a/versions/1/data_stream/fields/fields.spec.yml +++ b/versions/1/data_stream/fields/fields.spec.yml @@ -227,8 +227,8 @@ spec: Strings longer than the ignore_above setting will not be indexed or stored. For arrays of strings, ignore_above will be applied for each array element separately and string elements longer than ignore_above - will not be indexed or stored. Only honored for - 'type: keyword/wildcard'. Defaults to 1024. + will not be indexed or stored. Fleet honors this for `keyword` and + `wildcard` types. Defaults to 1024. type: integer default: 1024 From ec1318b948479b2c086fa203581ff75ef0047766 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Wed, 6 Apr 2022 17:56:40 -0400 Subject: [PATCH 09/16] Add changelog entry --- versions/1/changelog.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/versions/1/changelog.yml b/versions/1/changelog.yml index a88688d50..c53632c3a 100644 --- a/versions/1/changelog.yml +++ b/versions/1/changelog.yml @@ -7,6 +7,9 @@ - description: Validate that fields are only defined once per data stream. type: enhancement link: https://github.com/elastic/package-spec/pull/309 + - description: Define all fields properties currently supported and disallow unknown properties. + type: enhancement + link: https://github.com/elastic/package-spec/pull/314 - version: 1.7.0 changes: - description: Add kibana/osquery-pack-asset From 116168d7a641d113e787b6da32e5f8c6db17174e Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Thu, 28 Apr 2022 17:13:57 -0400 Subject: [PATCH 10/16] Clarify metric_type and dimension docs --- .../integration/data_stream/fields/fields.spec.yml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/versions/1/integration/data_stream/fields/fields.spec.yml b/versions/1/integration/data_stream/fields/fields.spec.yml index c1d67e9be..9b1b88e19 100644 --- a/versions/1/integration/data_stream/fields/fields.spec.yml +++ b/versions/1/integration/data_stream/fields/fields.spec.yml @@ -64,11 +64,11 @@ spec: metric_type: description: > The metric type of a numeric field. This is attached to the - field as metadata (via `meta`). A gauge is a single-value measurement - that can go up or down over time, such as a temperature. A counter is - a single-value cumulative counter that only goes up, such as the - number of requests processed by a web server. By default, no metric - type is associated with a field. + field as a `time_series_metric` mapping parameter. A gauge is a + single-value measurement that can go up or down over time, such as a + temperature. A counter is a single-value cumulative counter that only + goes up, such as the number of requests processed by a web server. By + default, no metric type is associated with a field. type: string enum: - counter @@ -92,7 +92,9 @@ spec: - nanos dimension: - description: Declare a field as dimension of time series + description: > + Declare a field as dimension of time series. This is + attached to the field as a `time_series_dimension` mapping parameter. type: boolean default: false From f77a923929af8ed58eca5e89f486f2d23023a198 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Thu, 28 Apr 2022 17:21:35 -0400 Subject: [PATCH 11/16] Remove match_only_text reference from multi_fields --- versions/1/integration/data_stream/fields/fields.spec.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/versions/1/integration/data_stream/fields/fields.spec.yml b/versions/1/integration/data_stream/fields/fields.spec.yml index 9b1b88e19..1ae3e0313 100644 --- a/versions/1/integration/data_stream/fields/fields.spec.yml +++ b/versions/1/integration/data_stream/fields/fields.spec.yml @@ -203,8 +203,7 @@ spec: a string field could be mapped as a text field for full-text search, and as a keyword field for sorting or aggregations. - Fleet honors this for `keyword`, `match_only_text`, `text`, and - `wildcard` types. + Fleet honors this for `keyword`, `text`, and `wildcard` types. $ref: "#" # JSON-schema syntax for pointing to the root of the schema null_value: From 2223ec9f7aca350a862fd58f245d35234f24871d Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Thu, 28 Apr 2022 17:45:17 -0400 Subject: [PATCH 12/16] Add include_in_parent and include_in_root for nested fields --- .../good/data_stream/foo/fields/some_fields.yml | 10 ++++++++++ .../data_stream/fields/fields.spec.yml | 16 ++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/test/packages/good/data_stream/foo/fields/some_fields.yml b/test/packages/good/data_stream/foo/fields/some_fields.yml index 2c32949a0..0748dc579 100644 --- a/test/packages/good/data_stream/foo/fields/some_fields.yml +++ b/test/packages/good/data_stream/foo/fields/some_fields.yml @@ -31,3 +31,13 @@ type: match_only_text - name: metric.*_bytes type: long +- name: a + type: nested + include_in_parent: true +- name: a.b + type: keyword +- name: c + type: nested + include_in_root: true +- name: c.d + type: keyword diff --git a/versions/1/integration/data_stream/fields/fields.spec.yml b/versions/1/integration/data_stream/fields/fields.spec.yml index 1ae3e0313..770a0d5b0 100644 --- a/versions/1/integration/data_stream/fields/fields.spec.yml +++ b/versions/1/integration/data_stream/fields/fields.spec.yml @@ -253,5 +253,21 @@ spec: settings. type: string + include_in_parent: + description: > + For nested field types, this specifies if all fields in the nested + object are also added to the parent document as standard (flat) + fields. + type: boolean + default: false + + include_in_root: + description: > + For nested field types, this specifies if all fields in the nested + object are also added to the root document as standard (flat) + fields. + type: boolean + default: false + required: - name From 81b7637779de4960d897d47d4e146000e3e3e54c Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Fri, 29 Apr 2022 09:13:14 -0400 Subject: [PATCH 13/16] Set additionalProperties: true --- versions/1/integration/data_stream/fields/fields.spec.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/versions/1/integration/data_stream/fields/fields.spec.yml b/versions/1/integration/data_stream/fields/fields.spec.yml index 770a0d5b0..dcf2a736d 100644 --- a/versions/1/integration/data_stream/fields/fields.spec.yml +++ b/versions/1/integration/data_stream/fields/fields.spec.yml @@ -6,7 +6,7 @@ spec: type: array items: type: object - additionalProperties: false + additionalProperties: true properties: name: description: > @@ -268,6 +268,6 @@ spec: fields. type: boolean default: false - + required: - name From f86db770737ef46b8a1531723f93adda5c61a0ce Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Fri, 29 Apr 2022 09:18:27 -0400 Subject: [PATCH 14/16] Revert "Remove level, group, title properties from test files" This reverts commit b9b5fc1241e5ca24d72d35221691b7b1028578c3. --- .../bad_deploy_variants/data_stream/foo/fields/fields.yml | 3 +++ .../deploy_docker/data_stream/foo/fields/fields.yml | 3 +++ .../deploy_terraform/data_stream/foo/fields/fields.yml | 3 +++ .../docs_extra_files/data_stream/pe/fields/some_fields.yml | 6 ++++++ test/packages/good/data_stream/foo/fields/some_fields.yml | 6 ++++++ .../data_stream/hidden_data_stream/fields/some_fields.yml | 3 +++ .../good/data_stream/ilm_policy/fields/some_fields.yml | 3 +++ test/packages/good/data_stream/pe/fields/some_fields.yml | 6 ++++++ .../good/data_stream/skipped_tests/fields/some_fields.yml | 6 ++++++ .../data_stream/pe/fields/some_fields.yml | 6 ++++++ .../data_stream/foo/fields/some_fields.yml | 6 ++++++ .../data_stream/foo/fields/some_fields.yml | 6 ++++++ 12 files changed, 57 insertions(+) diff --git a/test/packages/bad_deploy_variants/data_stream/foo/fields/fields.yml b/test/packages/bad_deploy_variants/data_stream/foo/fields/fields.yml index 2f58defee..6e427ddf6 100644 --- a/test/packages/bad_deploy_variants/data_stream/foo/fields/fields.yml +++ b/test/packages/bad_deploy_variants/data_stream/foo/fields/fields.yml @@ -1,7 +1,10 @@ - name: source + title: Source + group: 2 type: group fields: - name: geo.city_name + level: core type: keyword description: City name. ignore_above: 1024 \ No newline at end of file diff --git a/test/packages/deploy_docker/data_stream/foo/fields/fields.yml b/test/packages/deploy_docker/data_stream/foo/fields/fields.yml index 2f58defee..6e427ddf6 100644 --- a/test/packages/deploy_docker/data_stream/foo/fields/fields.yml +++ b/test/packages/deploy_docker/data_stream/foo/fields/fields.yml @@ -1,7 +1,10 @@ - name: source + title: Source + group: 2 type: group fields: - name: geo.city_name + level: core type: keyword description: City name. ignore_above: 1024 \ No newline at end of file diff --git a/test/packages/deploy_terraform/data_stream/foo/fields/fields.yml b/test/packages/deploy_terraform/data_stream/foo/fields/fields.yml index 2f58defee..6e427ddf6 100644 --- a/test/packages/deploy_terraform/data_stream/foo/fields/fields.yml +++ b/test/packages/deploy_terraform/data_stream/foo/fields/fields.yml @@ -1,7 +1,10 @@ - name: source + title: Source + group: 2 type: group fields: - name: geo.city_name + level: core type: keyword description: City name. ignore_above: 1024 \ No newline at end of file diff --git a/test/packages/docs_extra_files/data_stream/pe/fields/some_fields.yml b/test/packages/docs_extra_files/data_stream/pe/fields/some_fields.yml index 4adfa4af4..e6e1d439f 100644 --- a/test/packages/docs_extra_files/data_stream/pe/fields/some_fields.yml +++ b/test/packages/docs_extra_files/data_stream/pe/fields/some_fields.yml @@ -1,18 +1,24 @@ - name: source + title: Source + group: 2 type: group fields: - name: geo.city_name + level: core type: keyword description: City name. ignore_above: 1024 - name: geo.location + level: core type: geo_point description: Longitude and latitude. - name: geo.region_iso_code + level: core type: keyword description: Region ISO code. ignore_above: 1024 - name: geo.region_name + level: core type: keyword description: Region name. ignore_above: 1024 \ No newline at end of file diff --git a/test/packages/good/data_stream/foo/fields/some_fields.yml b/test/packages/good/data_stream/foo/fields/some_fields.yml index 0748dc579..14b180701 100644 --- a/test/packages/good/data_stream/foo/fields/some_fields.yml +++ b/test/packages/good/data_stream/foo/fields/some_fields.yml @@ -1,18 +1,24 @@ - name: source + title: Source + group: 2 type: group fields: - name: geo.city_name + level: core type: keyword description: City name. ignore_above: 1024 - name: geo.location + level: core type: geo_point description: Longitude and latitude. - name: geo.region_iso_code + level: core type: keyword description: Region ISO code. ignore_above: 1024 - name: geo.region_name + level: core type: keyword description: Region name. ignore_above: 1024 diff --git a/test/packages/good/data_stream/hidden_data_stream/fields/some_fields.yml b/test/packages/good/data_stream/hidden_data_stream/fields/some_fields.yml index 137f10b8f..4f46a95bf 100644 --- a/test/packages/good/data_stream/hidden_data_stream/fields/some_fields.yml +++ b/test/packages/good/data_stream/hidden_data_stream/fields/some_fields.yml @@ -1,7 +1,10 @@ - name: source + title: Source + group: 2 type: group fields: - name: geo.city_name + level: core type: keyword description: City name. ignore_above: 1024 diff --git a/test/packages/good/data_stream/ilm_policy/fields/some_fields.yml b/test/packages/good/data_stream/ilm_policy/fields/some_fields.yml index db268a685..f9b0e048d 100644 --- a/test/packages/good/data_stream/ilm_policy/fields/some_fields.yml +++ b/test/packages/good/data_stream/ilm_policy/fields/some_fields.yml @@ -1,7 +1,10 @@ - name: source + title: Source + group: 2 type: group fields: - name: geo.city_name + level: core type: keyword description: City name. ignore_above: 1024 diff --git a/test/packages/good/data_stream/pe/fields/some_fields.yml b/test/packages/good/data_stream/pe/fields/some_fields.yml index 4adfa4af4..e6e1d439f 100644 --- a/test/packages/good/data_stream/pe/fields/some_fields.yml +++ b/test/packages/good/data_stream/pe/fields/some_fields.yml @@ -1,18 +1,24 @@ - name: source + title: Source + group: 2 type: group fields: - name: geo.city_name + level: core type: keyword description: City name. ignore_above: 1024 - name: geo.location + level: core type: geo_point description: Longitude and latitude. - name: geo.region_iso_code + level: core type: keyword description: Region ISO code. ignore_above: 1024 - name: geo.region_name + level: core type: keyword description: Region name. ignore_above: 1024 \ No newline at end of file diff --git a/test/packages/good/data_stream/skipped_tests/fields/some_fields.yml b/test/packages/good/data_stream/skipped_tests/fields/some_fields.yml index 61834719e..bde93f380 100644 --- a/test/packages/good/data_stream/skipped_tests/fields/some_fields.yml +++ b/test/packages/good/data_stream/skipped_tests/fields/some_fields.yml @@ -1,18 +1,24 @@ - name: source + title: Source + group: 2 type: group fields: - name: geo.city_name + level: core type: keyword description: City name. ignore_above: 1024 - name: geo.location + level: core type: geo_point description: Longitude and latitude. - name: geo.region_iso_code + level: core type: keyword description: Region ISO code. ignore_above: 1024 - name: geo.region_name + level: core type: keyword description: Region name. ignore_above: 1024 diff --git a/test/packages/missing_image_files/data_stream/pe/fields/some_fields.yml b/test/packages/missing_image_files/data_stream/pe/fields/some_fields.yml index 4adfa4af4..e6e1d439f 100644 --- a/test/packages/missing_image_files/data_stream/pe/fields/some_fields.yml +++ b/test/packages/missing_image_files/data_stream/pe/fields/some_fields.yml @@ -1,18 +1,24 @@ - name: source + title: Source + group: 2 type: group fields: - name: geo.city_name + level: core type: keyword description: City name. ignore_above: 1024 - name: geo.location + level: core type: geo_point description: Longitude and latitude. - name: geo.region_iso_code + level: core type: keyword description: Region ISO code. ignore_above: 1024 - name: geo.region_name + level: core type: keyword description: Region name. ignore_above: 1024 \ No newline at end of file diff --git a/test/packages/missing_pipeline_dashes/data_stream/foo/fields/some_fields.yml b/test/packages/missing_pipeline_dashes/data_stream/foo/fields/some_fields.yml index 4adfa4af4..e6e1d439f 100644 --- a/test/packages/missing_pipeline_dashes/data_stream/foo/fields/some_fields.yml +++ b/test/packages/missing_pipeline_dashes/data_stream/foo/fields/some_fields.yml @@ -1,18 +1,24 @@ - name: source + title: Source + group: 2 type: group fields: - name: geo.city_name + level: core type: keyword description: City name. ignore_above: 1024 - name: geo.location + level: core type: geo_point description: Longitude and latitude. - name: geo.region_iso_code + level: core type: keyword description: Region ISO code. ignore_above: 1024 - name: geo.region_name + level: core type: keyword description: Region name. ignore_above: 1024 \ No newline at end of file diff --git a/test/packages/missing_required_fields/data_stream/foo/fields/some_fields.yml b/test/packages/missing_required_fields/data_stream/foo/fields/some_fields.yml index 643aa7c15..099bb968f 100644 --- a/test/packages/missing_required_fields/data_stream/foo/fields/some_fields.yml +++ b/test/packages/missing_required_fields/data_stream/foo/fields/some_fields.yml @@ -1,18 +1,24 @@ - name: source + title: Source + group: 2 type: group fields: - name: geo.city_name + level: core type: keyword description: City name. ignore_above: 1024 - name: geo.location + level: core type: geo_point description: Longitude and latitude. - name: geo.region_iso_code + level: core type: keyword description: Region ISO code. ignore_above: 1024 - name: geo.region_name + level: core type: keyword description: Region name. ignore_above: 1024 From 2baa12cfb3a4c86ff8702cdb3f6962c01aad7a03 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Fri, 29 Apr 2022 09:20:35 -0400 Subject: [PATCH 15/16] Amend changelog --- versions/1/changelog.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versions/1/changelog.yml b/versions/1/changelog.yml index 69de5e6aa..688b40209 100644 --- a/versions/1/changelog.yml +++ b/versions/1/changelog.yml @@ -13,7 +13,7 @@ - description: Prepare for support of multiple, independent package types. Require "type" to be present in manifest. type: enhancement link: https://github.com/elastic/package-spec/pull/323 - - description: Define all fields properties currently supported and disallow unknown properties. + - description: Define all fields properties currently supported. type: enhancement link: https://github.com/elastic/package-spec/pull/314 - version: 1.7.0 From 86254fe051a8057a886cf2fbf7ae1d8b3c9e3212 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Fri, 29 Apr 2022 16:20:11 -0400 Subject: [PATCH 16/16] Allow dash in field names --- test/packages/good/data_stream/foo/fields/some_fields.yml | 2 ++ versions/1/integration/data_stream/fields/fields.spec.yml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/test/packages/good/data_stream/foo/fields/some_fields.yml b/test/packages/good/data_stream/foo/fields/some_fields.yml index 14b180701..bfdb029c3 100644 --- a/test/packages/good/data_stream/foo/fields/some_fields.yml +++ b/test/packages/good/data_stream/foo/fields/some_fields.yml @@ -47,3 +47,5 @@ include_in_root: true - name: c.d type: keyword +- name: name-with-dash + type: keyword diff --git a/versions/1/integration/data_stream/fields/fields.spec.yml b/versions/1/integration/data_stream/fields/fields.spec.yml index dcf2a736d..267f37bca 100644 --- a/versions/1/integration/data_stream/fields/fields.spec.yml +++ b/versions/1/integration/data_stream/fields/fields.spec.yml @@ -13,7 +13,7 @@ spec: Name of field. Names containing dots are automatically split into sub-fields. type: string - pattern: '^[*_@A-Za-z0-9]+(\.[*_@A-Za-z0-9]+)*$' + pattern: '^[\-*_@A-Za-z0-9]+(\.[\-*_@A-Za-z0-9]+)*$' type: description: Datatype of field type: string