diff --git a/docs/configuring.asciidoc b/docs/configuring.asciidoc index fbdfb184f3f..0def3f96690 100644 --- a/docs/configuring.asciidoc +++ b/docs/configuring.asciidoc @@ -27,6 +27,8 @@ include::./copied-from-beats/shared-ssl-config.asciidoc[] include::./template-config.asciidoc[] +include::./ilm-setup.asciidoc[] + include::./copied-from-beats/loggingconfig.asciidoc[] include::./configuration-rum.asciidoc[] diff --git a/docs/ilm-setup.asciidoc b/docs/ilm-setup.asciidoc new file mode 100644 index 00000000000..231dc3ab7ee --- /dev/null +++ b/docs/ilm-setup.asciidoc @@ -0,0 +1,233 @@ +[role="xpack"] +[[manual-ilm-setup]] +== Manual index lifecycle management + +APM Server can take advantage of the index lifecycle management (ILM) capabilities of Elasticsearch. +ILM enables you to automate how you want to manage your indices over time. +You can base actions on factors such as shard size and performance requirements. + +The guide below will help you set up a custom ILM policy for span indices. +You can repeat the actions for any other indices. + +NOTE: If you're migrating from an existing setup, +any indices present before ILM was configured will need to be managed manually. + +. *Set up the default index template.* ++ +If you haven't already, you'll need to set up the default index template. +This is accomplished by running the <> command. ++ +-- +[source,js] +----------------------- +./apm-server setup --template +----------------------- +// CONSOLE +-- + +. *Create a policy for spans.* ++ +Index lifecycle management will manage an index based on its defined policy. +Policies only need to be created once, and will persist through version upgrades. +Let's create a policy named `apm_span_policy`. ++ +This policy defines two rollover criteria: `"max_age": "1d"`, and `"max_size": "50gb"`. +When one or more of these criteria are met, the policy rolls data over into a new index. +`apm_span_policy` also tells the old indexes to delete after _10 days_. +All of these values can be customized to your specific needs. ++ +-- +[source,js] +----------------------- +PUT _ilm/policy/apm_span_policy +{ + "policy": { + "phases": { + "hot": { + "actions": { + "rollover": { + "max_age": "1d", <1> + "max_size": "50gb" <2> + } + } + }, + "delete": { + "min_age": "10d", <3> + "actions": { + "delete": {} + } + } + } + } +} +----------------------- +// CONSOLE +<1> Rollover after _1 day_ +<2> Rollover after _50gb_ +<3> Delete old indexes after _10 days_ +-- + +. *Set up the ILM index template.* ++ +To use the index lifecycle management policy created in the previous step, +you need to specify it in the index template used to create the indices. +The following template associates `apm_span_policy` with indices created from the +apm-{stack-version}-span-ilm+ template. ++ +NOTE: Because we're utilizing the current stack-version ({stack-version}) in this step, +this action will need to be performed as a part of each version upgrade. ++ +-- +["source","js",subs="attributes"] +----------------------- +PUT _template/apm-{stack-version}-span-ilm +{ + "order": 2, + "index_patterns": ["apm-{stack-version}-span-*"], <1> + "settings": { + "index.lifecycle.name": "apm_span_policy", <2> + "index.lifecycle.rollover_alias": "apm-{stack-version}-span" + } +} +----------------------- +// CONSOLE +<1> This template applies to all indices with the prefix +apm-{stack-version}-span-+ +<2> Associates `apm_span_policy` with all indices created with this template +-- + +. *Create the index and alias.* ++ +Now we can create the first index: +apm-{stack-version}-span-000001+. ++ +NOTE: This action will need to be performed as a part of each version upgrade. ++ +-- +["source","js",subs="attributes"] +----------------------- +PUT apm-{stack-version}-span-000001 <1> +{ + "aliases": { + "apm-{stack-version}-span":{ + "is_write_index": true <2> + } + } +} +----------------------- +// CONSOLE +<1> The rollover action increments the suffix number for each subsequent index. +<2> Designates this index as the write index for this alias. +-- + +. *Verify the ILM index template was applied.* ++ +-- +[source,js] +----------------------- +GET apm-*-span/_settings +----------------------- +// CONSOLE +-- ++ +The response should be similar to this: ++ +-- +["source","js",subs="attributes"] +----------------------- +{ + "apm-{stack-version}-span-000001" : { + "settings" : { + "index" : { + "lifecycle" : { + "name" : "apm_span_policy", + "rollover_alias" : "apm-{stack-version}-span" + }, + "number_of_shards" : "1", + "provided_name" : "apm-{stack-version}-span-000001", + "creation_date" : "1553024227938", + "number_of_replicas" : "1", + "uuid" : "6b5l-H7QTRK95FAodAN-wg", + "version" : { + "created" : "7000099" + } + } + } + } +} +----------------------- +-- ++ +You can also verify and adjust your configuration via Kibana's {kibana-ref}/index-lifecycle-policies.html[management]. + +. *Repeat for other indices.* ++ +Repeat the previous steps for each index that will be using ILM. ++ +* Create a policy +* Set up the ILM index template +* Create the index and alias +* Verify the ILM index template was applied + +. *Modify APM Server's configuration.* ++ +Finally, modify the default index configuration in <>. +Trim off the date template from each index you are setting up ILM for, +so that APM Server is always writing events to the same place. +The name of your modified index configuration must match the `is_write_index` alias created previously ++ +It's important to note that `apm-server.yml` overwrites defaults rather than being merged. +This means you'll need to configure all of your indices in the file, even if some are not using ILM. ++ +-- +["source","yml",subs="attributes"] +----------------------- +output.elasticsearch: + indices: + - index: "apm-{stack-version}-sourcemap" + when.contains: + processor.event: "sourcemap" + + - index: "apm-{stack-version}-error" + when.contains: + processor.event: "error" + + - index: "apm-{stack-version}-transaction" + when.contains: + processor.event: "transaction" + + - index: "apm-{stack-version}-span" + when.contains: + processor.event: "span" + + - index: "apm-{stack-version}-metric" + when.contains: + processor.event: "metric" + + - index: "apm-{stack-version}-onboarding" + when.contains: + processor.event: "onboarding" +----------------------- +// CONSOLE +-- + +. *Start apm-server.* ++ +Your ILM configuration should now be up and running! + +.. Monitor ILM status as events flow: ++ +-- +[source,js] +----------------------- +GET apm-*/_ilm/explain?human +----------------------- +// CONSOLE +-- + +.. Monitor index status: ++ +-- +[source,js] +----------------------- +GET _cat/indices/apm*?v +----------------------- +// CONSOLE +-- \ No newline at end of file