Follow up changes in documentation and configuration reference about …

…data streams (#29819)

auditbeat/auditbeat.reference.yml

@@ -414,10 +414,9 @@ output.elasticsearch:
   # Number of workers per Elasticsearch host.
   #worker: 1
 
-  # Optional index name. The default is "auditbeat" plus date
-  # and generates [auditbeat-]YYYY.MM.DD keys.
+  # Optional data stream or index name. The default is "auditbeat-%{[agent.version]}".
   # In case you modify this pattern you must update setup.template.name and setup.template.pattern accordingly.
-  #index: "auditbeat-%{[agent.version]}-%{+yyyy.MM.dd}"
+  #index: "auditbeat-%{[agent.version]}"
 
   # Optional ingest pipeline. By default no pipeline will be used.
   #pipeline: ""
@@ -1226,10 +1225,8 @@ setup.template.settings:
 
 # ====================== Index Lifecycle Management (ILM) ======================
 
-# Configure index lifecycle management (ILM). These settings create a write
-# alias and add additional settings to the index template. When ILM is enabled,
-# output.elasticsearch.index is ignored, and the write alias is used to set the
-# index name.
+# Configure index lifecycle management (ILM) to manage the backing indices
+# of your data streams.
 
 # Enable ILM support. Valid values are true, false.
 #setup.ilm.enabled: true

filebeat/filebeat.reference.yml

@@ -1358,10 +1358,9 @@ output.elasticsearch:
   # Number of workers per Elasticsearch host.
   #worker: 1
 
-  # Optional index name. The default is "filebeat" plus date
-  # and generates [filebeat-]YYYY.MM.DD keys.
+  # Optional data stream or index name. The default is "filebeat-%{[agent.version]}".
   # In case you modify this pattern you must update setup.template.name and setup.template.pattern accordingly.
-  #index: "filebeat-%{[agent.version]}-%{+yyyy.MM.dd}"
+  #index: "filebeat-%{[agent.version]}"
 
   # Optional ingest pipeline. By default no pipeline will be used.
   #pipeline: ""
@@ -2170,10 +2169,8 @@ setup.template.settings:
 
 # ====================== Index Lifecycle Management (ILM) ======================
 
-# Configure index lifecycle management (ILM). These settings create a write
-# alias and add additional settings to the index template. When ILM is enabled,
-# output.elasticsearch.index is ignored, and the write alias is used to set the
-# index name.
+# Configure index lifecycle management (ILM) to manage the backing indices
+# of your data streams.
 
 # Enable ILM support. Valid values are true, false.
 #setup.ilm.enabled: true

heartbeat/heartbeat.reference.yml

@@ -560,10 +560,9 @@ output.elasticsearch:
   # Number of workers per Elasticsearch host.
   #worker: 1
 
-  # Optional index name. The default is "heartbeat" plus date
-  # and generates [heartbeat-]YYYY.MM.DD keys.
+  # Optional data stream or index name. The default is "heartbeat-%{[agent.version]}".
   # In case you modify this pattern you must update setup.template.name and setup.template.pattern accordingly.
-  #index: "heartbeat-%{[agent.version]}-%{+yyyy.MM.dd}"
+  #index: "heartbeat-%{[agent.version]}"
 
   # Optional ingest pipeline. By default no pipeline will be used.
   #pipeline: ""
@@ -1372,10 +1371,8 @@ setup.template.settings:
 
 # ====================== Index Lifecycle Management (ILM) ======================
 
-# Configure index lifecycle management (ILM). These settings create a write
-# alias and add additional settings to the index template. When ILM is enabled,
-# output.elasticsearch.index is ignored, and the write alias is used to set the
-# index name.
+# Configure index lifecycle management (ILM) to manage the backing indices
+# of your data streams.
 
 # Enable ILM support. Valid values are true, false.
 #setup.ilm.enabled: true

libbeat/_meta/config/output-elasticsearch.reference.yml.tmpl

@@ -31,10 +31,9 @@ output.elasticsearch:
   # Number of workers per Elasticsearch host.
   #worker: 1
 
-  # Optional index name. The default is "{{.BeatIndexPrefix}}" plus date
-  # and generates [{{.BeatIndexPrefix}}-]YYYY.MM.DD keys.
+  # Optional data stream or index name. The default is "{{.BeatIndexPrefix}}-%{[agent.version]}".
   # In case you modify this pattern you must update setup.template.name and setup.template.pattern accordingly.
-  #index: "{{.BeatIndexPrefix}}-%{[agent.version]}-%{+yyyy.MM.dd}"
+  #index: "{{.BeatIndexPrefix}}-%{[agent.version]}"
 
   # Optional ingest pipeline. By default no pipeline will be used.
   #pipeline: ""

libbeat/_meta/config/setup.ilm.reference.yml.tmpl

@@ -1,9 +1,7 @@
 {{header "Index Lifecycle Management (ILM)"}}
 
-# Configure index lifecycle management (ILM). These settings create a write
-# alias and add additional settings to the index template. When ILM is enabled,
-# output.elasticsearch.index is ignored, and the write alias is used to set the
-# index name.
+# Configure index lifecycle management (ILM) to manage the backing indices
+# of your data streams.
 
 # Enable ILM support. Valid values are true, false.
 #setup.ilm.enabled: true

libbeat/docs/howto/change-index-name.asciidoc

@@ -1,32 +1,18 @@
 [id="change-index-name"]
 == Change the index name
 
-ifndef::no_ilm[]
-TIP: If you're sending events to a cluster that supports index lifecycle
-management, you need to change the index name in the ILM policy.
-See <<ilm>> to learn how to change it.
-endif::no_ilm[]
-
-{beatname_uc} uses time series indices, by default, when index lifecycle
-management is disabled or unsupported. The indices are named
-+{beatname_lc}-{version}-yyyy.MM.dd+, where `yyyy.MM.dd` is the date when the
-events were indexed. To use a different name, set the
-<<index-option-es,`index`>> option in the {es} output. The value that
-you specify should include the root name of the index plus version and date
-information. You also need to configure the `setup.template.name` and
+{beatname_uc} uses data streams named +{beatname_lc}-{version}+.
+To use a different name, set the <<index-option-es,`index`>> option
+in the {es} output. You also need to configure the `setup.template.name` and
 `setup.template.pattern` options to match the new name. For example:
 
 ["source","sh",subs="attributes,callouts"]
 -----
-output.elasticsearch.index: "customname-%{[{beat_version_key}]}-%{+yyyy.MM.dd}"
+output.elasticsearch.index: "customname-%{[{beat_version_key}]}"
 setup.template.name: "customname"
-setup.template.pattern: "customname-*"
+setup.template.pattern: "customname-%{[{beat_version_key}]}"
 -----
 
-ifndef::no_ilm[]
-WARNING: If <<ilm,index lifecycle management>> is enabled (which is typically the default), `setup.template.name` and `setup.template.pattern` are ignored.
-endif::no_ilm[]
-
 ifndef::no_dashboards[]
 If you're using pre-built Kibana dashboards, also set the
 `setup.dashboards.index` option. For example: 

libbeat/docs/howto/load-index-templates.asciidoc

@@ -3,8 +3,8 @@
 
 {es} uses {ref}/index-templates.html[index templates] to define:
 
-* Settings that control the behavior of your indices. The settings include the
-lifecycle policy used to manage indices as they grow and age.
+* Settings that control the behavior of your data stream and backing indices.
+The settings include the lifecycle policy used to manage backing indices as they grow and age.
 * Mappings that determine how fields are analyzed. Each mapping sets the
 {ref}/mapping-types.html[{es} datatype] to use for a specific data field.
 
@@ -44,6 +44,8 @@ setup.template.fields: "path/to/fields.yml"
 If the template already exists, it’s not overwritten unless you configure
 {beatname_uc} to do so.
 
+You can load templates for both data streams and indices.
+
 [float]
 [[overwrite-template]]
 === Overwrite an existing index template

libbeat/docs/shared-ilm.asciidoc

@@ -7,16 +7,10 @@
 ++++
 
 Use the {ref}/getting-started-index-lifecycle-management.html[index lifecycle
-management] (ILM) feature in {es} to manage your {beatname_uc} indices as they age.
-For example, instead of creating daily indices where index size can vary based
-on the number of Beats and number of events sent, use an index lifecycle policy
-to automate a rollover to a new index when the existing index reaches a
-specified size or age.
-
-Starting with version 7.0, {beatname_uc} uses index lifecycle management by
-default when it connects to a cluster that supports lifecycle management.
-{beatname_uc} loads the default policy automatically and applies it to any
-indices created by {beatname_uc}.
+management] (ILM) feature in {es} to manage your {beatname_uc}
+their backing indices of your data streams as they age. {beatname_uc} loads
+the default policy automatically and applies it to any
+data streams created by {beatname_uc}.
 
 You can view and edit the policy in the *Index lifecycle policies* UI in {kib}.
 For more information about working with the UI, see
@@ -27,12 +21,7 @@ Example configuration:
 ["source","yaml",subs="attributes"]
 ----
 setup.ilm.enabled: true
-setup.ilm.rollover_alias: "{beatname_lc}"
-setup.ilm.pattern: "{now/d}-000001" <1>
 ----
-<1> Date math is supported here. For more information, see
-{ref}/indices-rollover-index.html#_using_date_math_with_the_rollover_api[Using
-date math with the rollover API].
 
 WARNING: If <<ilm,index lifecycle management>> is enabled (which is typically the default), `setup.template.name` and `setup.template.pattern` are ignored.
 
@@ -49,38 +38,6 @@ You can specify the following settings in the `setup.ilm` section of the
 Enables or disables index lifecycle management on any new indices created by
 {beatname_uc}. Valid values are `true` and `false`.
 
-[float]
-[[setup-ilm-rollover_alias-option]]
-==== `setup.ilm.rollover_alias`
-
-The index lifecycle write alias name. The default is
-+{beatname_lc}-%{[{beat_version_key}]}+. Setting this option changes the alias name.
-
-NOTE: If you modify this setting after loading the index template, you must
-overwrite the template to apply the changes.
-
-[float]
-[[setup-ilm-pattern-option]]
-==== `setup.ilm.pattern`
-
-The rollover index pattern. The default is `%{now/d}-000001`.
-
-Date math is supported in this setting. For example:
-
-[source,yaml]
-----
-setup.ilm.pattern: "{now/M{yyyy.MM}}-000001"
-----
-
-For more information, see
-{ref}/indices-rollover-index.html#_using_date_math_with_the_rollover_api[Using
-date math with the rollover API].
-
-NOTE: Before modifying this setting for an existing ILM setup, you must manually
-remove any aliases related to the previous pattern, then overwrite the policy.
-Existing indices that don't match the new pattern might no longer be subject to
-index lifecycle management.
-
 [float]
 [[setup-ilm-policy_name-option]]
 ==== `setup.ilm.policy_name`

metricbeat/metricbeat.reference.yml

@@ -1277,10 +1277,9 @@ output.elasticsearch:
   # Number of workers per Elasticsearch host.
   #worker: 1
 
-  # Optional index name. The default is "metricbeat" plus date
-  # and generates [metricbeat-]YYYY.MM.DD keys.
+  # Optional data stream or index name. The default is "metricbeat-%{[agent.version]}".
   # In case you modify this pattern you must update setup.template.name and setup.template.pattern accordingly.
-  #index: "metricbeat-%{[agent.version]}-%{+yyyy.MM.dd}"
+  #index: "metricbeat-%{[agent.version]}"
 
   # Optional ingest pipeline. By default no pipeline will be used.
   #pipeline: ""
@@ -2089,10 +2088,8 @@ setup.template.settings:
 
 # ====================== Index Lifecycle Management (ILM) ======================
 
-# Configure index lifecycle management (ILM). These settings create a write
-# alias and add additional settings to the index template. When ILM is enabled,
-# output.elasticsearch.index is ignored, and the write alias is used to set the
-# index name.
+# Configure index lifecycle management (ILM) to manage the backing indices
+# of your data streams.
 
 # Enable ILM support. Valid values are true, false.
 #setup.ilm.enabled: true

packetbeat/packetbeat.reference.yml

@@ -909,10 +909,9 @@ output.elasticsearch:
   # Number of workers per Elasticsearch host.
   #worker: 1
 
-  # Optional index name. The default is "packetbeat" plus date
-  # and generates [packetbeat-]YYYY.MM.DD keys.
+  # Optional data stream or index name. The default is "packetbeat-%{[agent.version]}".
   # In case you modify this pattern you must update setup.template.name and setup.template.pattern accordingly.
-  #index: "packetbeat-%{[agent.version]}-%{+yyyy.MM.dd}"
+  #index: "packetbeat-%{[agent.version]}"
 
   # Optional ingest pipeline. By default no pipeline will be used.
   #pipeline: ""
@@ -1721,10 +1720,8 @@ setup.template.settings:
 
 # ====================== Index Lifecycle Management (ILM) ======================
 
-# Configure index lifecycle management (ILM). These settings create a write
-# alias and add additional settings to the index template. When ILM is enabled,
-# output.elasticsearch.index is ignored, and the write alias is used to set the
-# index name.
+# Configure index lifecycle management (ILM) to manage the backing indices
+# of your data streams.
 
 # Enable ILM support. Valid values are true, false.
 #setup.ilm.enabled: true

winlogbeat/winlogbeat.reference.yml

@@ -350,10 +350,9 @@ output.elasticsearch:
   # Number of workers per Elasticsearch host.
   #worker: 1
 
-  # Optional index name. The default is "winlogbeat" plus date
-  # and generates [winlogbeat-]YYYY.MM.DD keys.
+  # Optional data stream or index name. The default is "winlogbeat-%{[agent.version]}".
   # In case you modify this pattern you must update setup.template.name and setup.template.pattern accordingly.
-  #index: "winlogbeat-%{[agent.version]}-%{+yyyy.MM.dd}"
+  #index: "winlogbeat-%{[agent.version]}"
 
   # Optional ingest pipeline. By default no pipeline will be used.
   #pipeline: ""
@@ -1162,10 +1161,8 @@ setup.template.settings:
 
 # ====================== Index Lifecycle Management (ILM) ======================
 
-# Configure index lifecycle management (ILM). These settings create a write
-# alias and add additional settings to the index template. When ILM is enabled,
-# output.elasticsearch.index is ignored, and the write alias is used to set the
-# index name.
+# Configure index lifecycle management (ILM) to manage the backing indices
+# of your data streams.
 
 # Enable ILM support. Valid values are true, false.
 #setup.ilm.enabled: true

x-pack/auditbeat/auditbeat.reference.yml

@@ -470,10 +470,9 @@ output.elasticsearch:
   # Number of workers per Elasticsearch host.
   #worker: 1
 
-  # Optional index name. The default is "auditbeat" plus date
-  # and generates [auditbeat-]YYYY.MM.DD keys.
+  # Optional data stream or index name. The default is "auditbeat-%{[agent.version]}".
   # In case you modify this pattern you must update setup.template.name and setup.template.pattern accordingly.
-  #index: "auditbeat-%{[agent.version]}-%{+yyyy.MM.dd}"
+  #index: "auditbeat-%{[agent.version]}"
 
   # Optional ingest pipeline. By default no pipeline will be used.
   #pipeline: ""
@@ -1282,10 +1281,8 @@ setup.template.settings:
 
 # ====================== Index Lifecycle Management (ILM) ======================
 
-# Configure index lifecycle management (ILM). These settings create a write
-# alias and add additional settings to the index template. When ILM is enabled,
-# output.elasticsearch.index is ignored, and the write alias is used to set the
-# index name.
+# Configure index lifecycle management (ILM) to manage the backing indices
+# of your data streams.
 
 # Enable ILM support. Valid values are true, false.
 #setup.ilm.enabled: true

x-pack/filebeat/filebeat.reference.yml

@@ -3588,10 +3588,9 @@ output.elasticsearch:
   # Number of workers per Elasticsearch host.
   #worker: 1
 
-  # Optional index name. The default is "filebeat" plus date
-  # and generates [filebeat-]YYYY.MM.DD keys.
+  # Optional data stream or index name. The default is "filebeat-%{[agent.version]}".
   # In case you modify this pattern you must update setup.template.name and setup.template.pattern accordingly.
-  #index: "filebeat-%{[agent.version]}-%{+yyyy.MM.dd}"
+  #index: "filebeat-%{[agent.version]}"
 
   # Optional ingest pipeline. By default no pipeline will be used.
   #pipeline: ""
@@ -4400,10 +4399,8 @@ setup.template.settings:
 
 # ====================== Index Lifecycle Management (ILM) ======================
 
-# Configure index lifecycle management (ILM). These settings create a write
-# alias and add additional settings to the index template. When ILM is enabled,
-# output.elasticsearch.index is ignored, and the write alias is used to set the
-# index name.
+# Configure index lifecycle management (ILM) to manage the backing indices
+# of your data streams.
 
 # Enable ILM support. Valid values are true, false.
 #setup.ilm.enabled: true

x-pack/functionbeat/functionbeat.reference.yml

@@ -592,10 +592,9 @@ output.elasticsearch:
   # Number of workers per Elasticsearch host.
   #worker: 1
 
-  # Optional index name. The default is "functionbeat" plus date
-  # and generates [functionbeat-]YYYY.MM.DD keys.
+  # Optional data stream or index name. The default is "functionbeat-%{[agent.version]}".
   # In case you modify this pattern you must update setup.template.name and setup.template.pattern accordingly.
-  #index: "functionbeat-%{[agent.version]}-%{+yyyy.MM.dd}"
+  #index: "functionbeat-%{[agent.version]}"
 
   # Optional ingest pipeline. By default no pipeline will be used.
   #pipeline: ""
@@ -1005,10 +1004,8 @@ setup.template.settings:
 
 # ====================== Index Lifecycle Management (ILM) ======================
 
-# Configure index lifecycle management (ILM). These settings create a write
-# alias and add additional settings to the index template. When ILM is enabled,
-# output.elasticsearch.index is ignored, and the write alias is used to set the
-# index name.
+# Configure index lifecycle management (ILM) to manage the backing indices
+# of your data streams.
 
 # Enable ILM support. Valid values are true, false.
 #setup.ilm.enabled: true