From 93803deb2d1088e38a01fa1544a35ec78f08eb6c Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Thu, 15 Sep 2022 09:50:51 +0200 Subject: [PATCH 1/3] Add setting to configure the source mode of a data stream --- spec/changelog.yml | 3 +++ spec/integration/data_stream/manifest.spec.yml | 14 ++++++++++++++ .../good/data_stream/k8s_data_stream/manifest.yml | 2 ++ .../time_series/data_stream/example/manifest.yml | 1 + 4 files changed, 20 insertions(+) diff --git a/spec/changelog.yml b/spec/changelog.yml index 41e152a6..4b1ff5b8 100644 --- a/spec/changelog.yml +++ b/spec/changelog.yml @@ -24,6 +24,9 @@ - description: Document and validate `object_type` and `object_type_mapping_type`. type: enhancement link: https://github.com/elastic/package-spec/pull/417 + - description: Add setting to configure the source mode of a data stream + type: enhancement + link: https://github.com/elastic/package-spec/issues/340 - version: 1.18.0 changes: - description: Update Go runtime to 1.19.1. diff --git a/spec/integration/data_stream/manifest.spec.yml b/spec/integration/data_stream/manifest.spec.yml index b54411cc..f08b874f 100644 --- a/spec/integration/data_stream/manifest.spec.yml +++ b/spec/integration/data_stream/manifest.spec.yml @@ -222,5 +222,19 @@ spec: type: array items: type: string + source_mode: + description: |- + Source mode to use. This configures how the document source (`_source`) is stored + for this data stream. + If configured as `default`, this mode is not configured and it uses Elasticsearch defaults. + If configured as `synthetic`, it doesn't store the source, but tries to rebuild it + from the indexed fields when queried. + If no configured, users may override the setting from Fleet UI. + type: string + enum: + - "default" + - "synthetic" + examples: + - "synthetic" required: - title diff --git a/test/packages/good/data_stream/k8s_data_stream/manifest.yml b/test/packages/good/data_stream/k8s_data_stream/manifest.yml index 9348ebc2..bedfa836 100644 --- a/test/packages/good/data_stream/k8s_data_stream/manifest.yml +++ b/test/packages/good/data_stream/k8s_data_stream/manifest.yml @@ -12,3 +12,5 @@ streams: default: /server-status title: Nginx access logs description: Collect Nginx access logs +elasticsearch: + source_mode: "default" diff --git a/test/packages/time_series/data_stream/example/manifest.yml b/test/packages/time_series/data_stream/example/manifest.yml index 3a3147c6..a4d2bbeb 100644 --- a/test/packages/time_series/data_stream/example/manifest.yml +++ b/test/packages/time_series/data_stream/example/manifest.yml @@ -11,6 +11,7 @@ streams: default: 10s elasticsearch: + source_mode: "synthetic" index_mode: "time_series" index_template: settings: From 07509077a5f34c4ecd0041ce3a025bb39354cd81 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Thu, 15 Sep 2022 10:13:53 +0200 Subject: [PATCH 2/3] Add link to synthetic source docs --- spec/integration/data_stream/manifest.spec.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spec/integration/data_stream/manifest.spec.yml b/spec/integration/data_stream/manifest.spec.yml index f08b874f..1247f779 100644 --- a/spec/integration/data_stream/manifest.spec.yml +++ b/spec/integration/data_stream/manifest.spec.yml @@ -227,8 +227,8 @@ spec: Source mode to use. This configures how the document source (`_source`) is stored for this data stream. If configured as `default`, this mode is not configured and it uses Elasticsearch defaults. - If configured as `synthetic`, it doesn't store the source, but tries to rebuild it - from the indexed fields when queried. + If configured as `synthetic`, it enables [synthetic source](https://www.elastic.co/guide/en/elasticsearch/reference/8.4/mapping-source-field.html#synthetic-source), + that doesn't store the source, but tries to rebuild it from the indexed fields when queried. If no configured, users may override the setting from Fleet UI. type: string enum: From fcdc7b9201522ecd3f4645698a808cc8b24f033d Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Thu, 15 Sep 2022 10:31:06 +0200 Subject: [PATCH 3/3] Allow to override synthetic mode from UI. --- spec/integration/data_stream/manifest.spec.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/integration/data_stream/manifest.spec.yml b/spec/integration/data_stream/manifest.spec.yml index 1247f779..5cb27c5f 100644 --- a/spec/integration/data_stream/manifest.spec.yml +++ b/spec/integration/data_stream/manifest.spec.yml @@ -229,7 +229,7 @@ spec: If configured as `default`, this mode is not configured and it uses Elasticsearch defaults. If configured as `synthetic`, it enables [synthetic source](https://www.elastic.co/guide/en/elasticsearch/reference/8.4/mapping-source-field.html#synthetic-source), that doesn't store the source, but tries to rebuild it from the indexed fields when queried. - If no configured, users may override the setting from Fleet UI. + If no configured or set to `synthetic`, users may override the setting from Fleet UI. type: string enum: - "default"