From 093e365e8b4b94356cb3902e9546922188588cd3 Mon Sep 17 00:00:00 2001 From: DeDe Morton Date: Mon, 1 Apr 2019 13:40:49 -0700 Subject: [PATCH] Indicate that ILM is on by default and add new settings (#11485) * Indicate that ILM is on by default and add new settings * Add changes from the review --- libbeat/docs/shared-ilm.asciidoc | 177 ++++++++------------- libbeat/docs/shared-kibana-config.asciidoc | 2 +- libbeat/docs/shared-path-config.asciidoc | 2 +- 3 files changed, 72 insertions(+), 109 deletions(-) diff --git a/libbeat/docs/shared-ilm.asciidoc b/libbeat/docs/shared-ilm.asciidoc index acfc479266c0..e73b73b270be 100644 --- a/libbeat/docs/shared-ilm.asciidoc +++ b/libbeat/docs/shared-ilm.asciidoc @@ -1,140 +1,103 @@ [[ilm]] [role="xpack"] -== Set up index lifecycle management +== Configure index lifecycle management -beta[] +Use the {ref}/getting-started-index-lifecycle-management.html[index lifecycle +management] feature in {es} to manage your {beatname_uc} indices as they age. +For example, instead of creating daily indices where index size can vary based +on the number of Beats and number of events sent, use an index lifecycle policy +to automate a rollover to a new index when the existing index reaches a +specified size or age. -You can use the {ref}/getting-started-index-lifecycle-management.html[index -lifecycle management] feature in {es} to manage your {beatname_uc} indices as -they age. For example, instead of having {beatname_uc} create daily indices -where index size can vary based on the number of Beats and number of events -sent, you can use an index lifecycle policy that automates a rollover to a new -index when the existing index reaches a specified size or age. +Starting with version 7.0, {beatname_uc} uses index lifecycle management by +default when it connects to a cluster that supports lifecycle management. +{beatname_uc} loads the default policy automatically and applies it to any +indices created by {beatname_uc}. -{beatname_uc} provides a default policy that you can load when you set up -{beatname_uc}. The default policy is applied to any new indices created by -{beatname_uc}. You can edit the policy to modify the lifecycle of both new and -existing indices. +You can view and edit the policy in the *Index lifecycle policies* UI in {kib}. +For more information about working with the UI, see +{kibana-ref}/index-lifecycle-policies.html[Index lifecyle policies]. -To use index lifecycle management on {beatname_uc} indices: - -. Enable index lifecycle management by setting `setup.ilm.enabled: true`. For example: -+ --- -[source,yaml] ------------------------------------------------------------------------------- -setup.ilm.enabled: true ------------------------------------------------------------------------------- - -This configuration overwrites your index settings and adjusts the {beatname_uc} -template to use the correct settings for index lifecycle management. - -NOTE: If you've previously loaded the index template for this version into {es}, -you must overwrite the template by setting `setup.template.overwrite: true`. - -The rollover alias is set to +{beatname_lc}-\{{beat_version_key}\}+ by default. You -can change the prefix used in the alias by setting `ilm.rollover_alias`, but you -can't remove +{beat_version_key}+ from the rollover alias name. The default pattern -used for the rollover index is `%{now/d}-000001`. You can change the -pattern by setting `ilm.pattern`. For example: +Example configuration: ["source","yaml",subs="attributes"] ---- -output.elasticsearch: - hosts: ["localhost"] - ilm.enabled: true - ilm.rollover_alias: "{beatname_lc}" - ilm.pattern: "{now/d}-000001" <1> +setup.ilm.enabled: auto +setup.ilm.rollover_alias: "{beatname_lc}" +setup.ilm.pattern: "{now/d}-000001" <1> ---- <1> Date math is supported here. For more information, see {ref}/indices-rollover-index.html#_using_date_math_with_the_rollover_api[Using date math with the rollover API]. -NOTE: If you modify the `rollover_alias` or `pattern` settings after loading the -index template, you must overwrite the template to apply the changes. --- +[float] +=== Configuration options -. Load the default policy into {es}. You can either use the `setup` command to -load the policy without modifying it, or modify the policy and load it manually. -+ --- -To use the setup command, run: +You can specify the following settings in the `setup.ilm` section of the ++{beatname_lc}.yml+ config file: -["source","shell",subs="attributes"] ----- -{beatname_lc} setup --ilm-policy ----- +[float] +[[setup-ilm-option]] +==== `setup.ilm.enabled` -After loading the default policy, you can edit it in the *Index lifecycle policies* -UI in {kib}. For more information about working with the UI, see -{kibana-ref}/index-lifecycle-policies.html[Index lifecyle policies]. +Enables or disables index lifecycle management on any new indices created by +{beatname_uc}. Valid values are `true`, `false`, and `auto`. When `auto` (the +default) is specified on version 7.0 and later, {beatname_uc} automatically uses +index lifecycle management if the feature is enabled in {es} and has the +required license; otherwise, {beatname_uc} creates daily indices. -To modify the default policy before loading it, run +{beatname_lc} export -ilm-policy+ to print the policy to stdout. Modify the policy then use the -{ref}/ilm-put-lifecycle.html[Create lifecycle policy API] to load it into {es}. +[float] +[[setup-ilm-rollover_alias-option]] +==== `setup.ilm.rollover_alias` --- +The index lifecycle write alias name. The default is ++{beatname_lc}-\{{beat_version_key}\}+. Setting this option changes the prefix +in the alias name. It doesn't remove +{beat_version_key}+ from the alias name. + +NOTE: If you modify this setting after loading the index template, you must +overwrite the template to apply the changes. [float] -=== Advanced ILM settings +[[setup-ilm-pattern-option]] +==== `setup.ilm.pattern` -WARNING: We recommend that you avoid modifying these settings unless you know -what you're doing. +The rollover index pattern. The default is `%{now/d}-000001`. -The default index lifecycle management settings work best for common use cases -that work with the automated alias setup described earlier. It is possible to -use a multiple write alias with dynamic index patterns, but this requires manual -set up. This section describes the configuration options you need to change. +NOTE: If you modify this setting after loading the index template, you must +overwrite the template to apply the changes. -Let's assume you have the index pattern `customname-%{event.module}` where -`event.module` can have the values `system` and `apache`. First you must set up -a rollover index for `customname-system` and `customname-apache`. For details on -how to do this, see -{ref}/indices-rollover-index.html#_using_date_math_with_the_rollover_api[Rollover -Index]. +[float] +[[setup-ilm-policy_name-option]] +==== `setup.ilm.policy_name` -Next, set the index pattern in the {es} output. For example: +The name to use for the lifecycle policy. The default is ++{beatname_lc}-\{{beat_version_key}\}+. -["source","yaml",subs="attributes"] ----- -output.elasticesarch.index: customname-%{event.module} <1> ----- -<1> For this example to work, every event must contain `event.module`. +[float] +[[setup-ilm-policy_file-option]] +==== `setup.ilm.policy_file` -If you change the index name, you must also set the template name, template -pattern, rollover alias, and lifecycle name. The best way to set these is -through an {es} template. It's possible to disable the template loading in -{beatname_uc} and specify these settings in your own template. Or you can use -the following config options in {beatname_uc}: +The path to a JSON file that contains a lifecycle policy configuration. Use this +setting to load your own lifecycle policy. -[source,yaml] ----- -setup.template.name: "customname" -setup.template.pattern: "customname-*" -setup.template.settings.index.lifecycle.rollover_alias: "customname" -setup.template.settings.index.lifecycle.name: "beats-default-policy" ----- +For more information about lifecycle policies, see +{ref}/set-up-lifecycle-policy.html[Set up index lifecycle management policy] in +the _{es} Reference_. -IMPORTANT: If you set the options manually as shown in this example, do *not* -set `ilm.enabled`, or the settings specified in the configuration file will be -overwritten. +[float] +[[setup-ilm-check_exists-option]] +==== `setup.ilm.check_exists` -This configuration results in a managed index named something like -+customname-{localdate}-000001+ and the following index settings: +When set to `false`, disables the check for an existing lifecycle policy. The +default is `true`. You need to disable this check if the {beatname_uc} +user connecting to a secured cluster doesn't have the `read_ilm` privilege. -["source","shell"] ----- -"aliases" : { - "customname" : { - "is_write_index" : true - } -}, -... - "index" : { - "lifecycle" : { - "name" : "beats-default-policy", - "rollover_alias" : "customname" - }, ----- +If you set this option to `false`, set `setup.ilm.overwrite: true` so the +lifecycle policy can be installed. +[float] +[[setup-ilm-overwrite-option]] +==== `setup.ilm.overwrite` +When set to `true`, the lifecycle policy is overwritten at startup. The default +is `false`. diff --git a/libbeat/docs/shared-kibana-config.asciidoc b/libbeat/docs/shared-kibana-config.asciidoc index 90dcdb012f46..0e1e8e743711 100644 --- a/libbeat/docs/shared-kibana-config.asciidoc +++ b/libbeat/docs/shared-kibana-config.asciidoc @@ -10,7 +10,7 @@ ////////////////////////////////////////////////////////////////////////// [[setup-kibana-endpoint]] -== Set up the Kibana endpoint +== Configure the Kibana endpoint Starting with {beatname_uc} 6.0.0, the Kibana dashboards are loaded into Kibana via the Kibana API. This requires a Kibana endpoint configuration. diff --git a/libbeat/docs/shared-path-config.asciidoc b/libbeat/docs/shared-path-config.asciidoc index ed82214accbf..117c5aad34be 100644 --- a/libbeat/docs/shared-path-config.asciidoc +++ b/libbeat/docs/shared-path-config.asciidoc @@ -11,7 +11,7 @@ ////////////////////////////////////////////////////////////////////////// [[configuration-path]] -== Set up project paths +== Configure project paths The `path` section of the +{beatname_lc}.yml+ config file contains configuration options that define where {beatname_uc} looks for its files. For example, {beatname_uc}