[docs] Add manual ILM instructions (#2033)

docs/configuring.asciidoc

@@ -27,6 +27,8 @@ include::./copied-from-beats/shared-ssl-config.asciidoc[]
 
 include::./template-config.asciidoc[]
 
+include::./ilm-setup.asciidoc[]
+
 include::./copied-from-beats/loggingconfig.asciidoc[]
 
 include::./configuration-rum.asciidoc[]

docs/ilm-setup.asciidoc

@@ -0,0 +1,233 @@
+[role="xpack"]
+[[manual-ilm-setup]]
+== Manual index lifecycle management
+
+APM Server can take advantage of the index lifecycle management (ILM) capabilities of Elasticsearch.
+ILM enables you to automate how you want to manage your indices over time.
+You can base actions on factors such as shard size and performance requirements. 
+
+The guide below will help you set up a custom ILM policy for span indices.
+You can repeat the actions for any other indices.
+
+NOTE: If you're migrating from an existing setup,
+any indices present before ILM was configured will need to be managed manually.
+
+. *Set up the default index template.*
++
+If you haven't already, you'll need to set up the default index template.
+This is accomplished by running the <<setup-command,`setup --template`>> command.
++
+--
+[source,js]
+-----------------------
+./apm-server setup --template
+-----------------------
+// CONSOLE
+--
+
+. *Create a policy for spans.*
++
+Index lifecycle management will manage an index based on its defined policy.
+Policies only need to be created once, and will persist through version upgrades.
+Let's create a policy named `apm_span_policy`.
++
+This policy defines two rollover criteria: `"max_age": "1d"`, and `"max_size": "50gb"`.
+When one or more of these criteria are met, the policy rolls data over into a new index.
+`apm_span_policy` also tells the old indexes to delete after _10 days_.
+All of these values can be customized to your specific needs.
++
+--
+[source,js]
+-----------------------
+PUT _ilm/policy/apm_span_policy
+{
+  "policy": {
+    "phases": {
+      "hot": {
+        "actions": {
+          "rollover": {
+            "max_age": "1d", <1>
+            "max_size": "50gb" <2>
+          }
+        }
+      },
+      "delete": {
+        "min_age": "10d", <3>
+        "actions": {
+          "delete": {}
+        }
+      }
+    }
+  }
+}
+-----------------------
+// CONSOLE
+<1> Rollover after _1 day_
+<2> Rollover after _50gb_
+<3> Delete old indexes after _10 days_
+--
+
+. *Set up the ILM index template.*
++
+To use the index lifecycle management policy created in the previous step,
+you need to specify it in the index template used to create the indices.
+The following template associates `apm_span_policy` with indices created from the +apm-{stack-version}-span-ilm+ template.
++
+NOTE: Because we're utilizing the current stack-version ({stack-version}) in this step,
+this action will need to be performed as a part of each version upgrade.
++
+--
+["source","js",subs="attributes"]
+-----------------------
+PUT _template/apm-{stack-version}-span-ilm
+{
+  "order": 2,
+  "index_patterns": ["apm-{stack-version}-span-*"], <1>
+  "settings": {
+    "index.lifecycle.name": "apm_span_policy", <2>
+    "index.lifecycle.rollover_alias": "apm-{stack-version}-span"
+  }
+}
+-----------------------
+// CONSOLE
+<1> This template applies to all indices with the prefix +apm-{stack-version}-span-+
+<2> Associates `apm_span_policy` with all indices created with this template
+--
+
+. *Create the index and alias.*
++
+Now we can create the first index: +apm-{stack-version}-span-000001+.
++
+NOTE: This action will need to be performed as a part of each version upgrade.
++
+--
+["source","js",subs="attributes"]
+-----------------------
+PUT apm-{stack-version}-span-000001 <1>
+{
+  "aliases": {
+    "apm-{stack-version}-span":{
+      "is_write_index": true <2>
+    }
+  }
+}
+-----------------------
+// CONSOLE
+<1> The rollover action increments the suffix number for each subsequent index.
+<2> Designates this index as the write index for this alias.
+--
+
+. *Verify the ILM index template was applied.*
++
+--
+[source,js]
+-----------------------
+GET apm-*-span/_settings
+-----------------------
+// CONSOLE
+--
++
+The response should be similar to this:
++
+--
+["source","js",subs="attributes"]
+-----------------------
+{
+  "apm-{stack-version}-span-000001" : {
+    "settings" : {
+      "index" : {
+        "lifecycle" : {
+          "name" : "apm_span_policy",
+          "rollover_alias" : "apm-{stack-version}-span"
+        },
+        "number_of_shards" : "1",
+        "provided_name" : "apm-{stack-version}-span-000001",
+        "creation_date" : "1553024227938",
+        "number_of_replicas" : "1",
+        "uuid" : "6b5l-H7QTRK95FAodAN-wg",
+        "version" : {
+          "created" : "7000099"
+        }
+      }
+    }
+  }
+}
+-----------------------
+--
++
+You can also verify and adjust your configuration via Kibana's {kibana-ref}/index-lifecycle-policies.html[management]. 
+
+. *Repeat for other indices.*
++
+Repeat the previous steps for each index that will be using ILM.
++
+* Create a policy
+* Set up the ILM index template
+* Create the index and alias
+* Verify the ILM index template was applied
+
+. *Modify APM Server's configuration.*
++
+Finally, modify the default index configuration in <<apm-server-configuration,`apm-server.yml`>>.
+Trim off the date template from each index you are setting up ILM for,
+so that APM Server is always writing events to the same place.
+The name of your modified index configuration must match the `is_write_index` alias created previously
++
+It's important to note that `apm-server.yml` overwrites defaults rather than being merged.
+This means you'll need to configure all of your indices in the file, even if some are not using ILM.
++
+--
+["source","yml",subs="attributes"]
+-----------------------
+output.elasticsearch:
+  indices:
+    - index: "apm-{stack-version}-sourcemap"
+      when.contains:
+        processor.event: "sourcemap"
+    
+    - index: "apm-{stack-version}-error"
+      when.contains:
+        processor.event: "error"
+    
+    - index: "apm-{stack-version}-transaction"
+      when.contains:
+        processor.event: "transaction"
+
+    - index: "apm-{stack-version}-span"
+      when.contains:
+        processor.event: "span"
+    
+    - index: "apm-{stack-version}-metric"
+      when.contains:
+        processor.event: "metric"
+    
+    - index: "apm-{stack-version}-onboarding"
+      when.contains:
+        processor.event: "onboarding"
+-----------------------
+// CONSOLE
+--
+
+. *Start apm-server.*
++
+Your ILM configuration should now be up and running!
+
+.. Monitor ILM status as events flow:
++
+--
+[source,js]
+-----------------------
+GET apm-*/_ilm/explain?human
+-----------------------
+// CONSOLE
+--
+
+.. Monitor index status:
++
+--
+[source,js]
+-----------------------
+GET _cat/indices/apm*?v
+-----------------------
+// CONSOLE
+--