[7.x] Add ILM histore store index (#50287) (#50345)

* Add ILM histore store index (#50287)

* Add ILM histore store index

This commit adds an ILM history store that tracks the lifecycle
execution state as an index progresses through its ILM policy. ILM
history documents store output similar to what the ILM explain API
returns.

An example document with ALL fields (not all documents will have all
fields) would look like:

```json
{
  "@timestamp": 1203012389,
  "policy": "my-ilm-policy",
  "index": "index-2019.1.1-000023",
  "index_age":123120,
  "success": true,
  "state": {
    "phase": "warm",
    "action": "allocate",
    "step": "ERROR",
    "failed_step": "update-settings",
    "is_auto-retryable_error": true,
    "creation_date": 12389012039,
    "phase_time": 12908389120,
    "action_time": 1283901209,
    "step_time": 123904107140,
    "phase_definition": "{\"policy\":\"ilm-history-ilm-policy\",\"phase_definition\":{\"min_age\":\"0ms\",\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}},\"version\":1,\"modified_date_in_millis\":1576517253463}",
    "step_info": "{... etc step info here as json ...}"
  },
  "error_details": "java.lang.RuntimeException: etc\n\tcaused by:etc etc etc full stacktrace"
}
```

These documents go into the `ilm-history-1-00000N` index to provide an
audit trail of the operations ILM has performed.

This history storage is enabled by default but can be disabled by setting
`index.lifecycle.history_index_enabled` to `false.`

Resolves #49180

* Make ILMHistoryStore.putAsync truly async (#50403)

This moves the `putAsync` method in `ILMHistoryStore` never to block.
Previously due to the way that the `BulkProcessor` works, it was possible
for `BulkProcessor#add` to block executing a bulk request. This was bad
as we may be adding things to the history store in cluster state update
threads.

This also moves the index creation to be done prior to the bulk request
execution, rather than being checked every time an operation was added
to the queue. This lessens the chance of the index being created, then
deleted (by some external force), and then recreated via a bulk indexing
request.

Resolves #50353

client/rest-high-level/build.gradle

@@ -135,6 +135,7 @@ testClusters.all {
   setting 'xpack.security.authc.realms.pki.pki1.delegation.enabled', 'true'
 
   setting 'indices.lifecycle.poll_interval', '1000ms'
+  setting 'index.lifecycle.history_index_enabled', 'false'
   keystore 'xpack.security.transport.ssl.truststore.secure_password', 'testnode'
   extraConfigFile 'roles.yml', file('roles.yml')
   user username: System.getProperty('tests.rest.cluster.username', 'test_user'),

docs/build.gradle

@@ -59,6 +59,7 @@ testClusters.integTest {
   extraConfigFile 'hunspell/en_US/en_US.dic', project(":server").file('src/test/resources/indices/analyze/conf_dir/hunspell/en_US/en_US.dic')
   // Whitelist reindexing from the local node so we can test it.
   setting 'reindex.remote.whitelist', '127.0.0.1:*'
+  setting 'index.lifecycle.history_index_enabled', 'false'
 
   // TODO: remove this once cname is prepended to transport.publish_address by default in 8.0
   systemProperty 'es.transport.cname_in_publish_address', 'true'

docs/plugins/analysis-icu.asciidoc

@@ -368,7 +368,7 @@ PUT my_index
   }
 }
 
-GET _search <3>
+GET /my_index/_search <3>
 {
   "query": {
     "match": {

docs/reference/indices/rollover-index.asciidoc

@@ -431,7 +431,7 @@ PUT logs/_doc/2 <2>
 //////////////////////////
 [source,console]
 --------------------------------------------------
-GET _alias
+GET my_logs_index-000001,my_logs_index-000002/_alias
 --------------------------------------------------
 // TEST[continued]
 //////////////////////////

docs/reference/settings/ilm-settings.asciidoc

@@ -14,6 +14,11 @@ ILM REST API endpoints and functionality. Defaults to `true`.
 (<<time-units, time units>>) How often {ilm} checks for indices that meet policy
 criteria. Defaults to `10m`.
 
+`index.lifecycle.history_index_enabled`::
+Whether ILM's history index is enabled. If enabled, ILM will record the
+history of actions taken as part of ILM policies to the `ilm-history-*`
+indices. Defaults to `true`.
+
 ==== Index level settings
 These index-level {ilm-init} settings are typically configured through index
 templates. For more information, see <<ilm-gs-create-policy>>.

test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java

@@ -75,6 +75,7 @@
 import java.security.cert.CertificateException;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -472,6 +473,13 @@ protected boolean preserveILMPoliciesUponCompletion() {
         return false;
     }
 
+    /**
+     * A set of ILM policies that should be preserved between runs.
+     */
+    protected Set<String> preserveILMPolicyIds() {
+        return Collections.singleton("ilm-history-ilm-policy");
+    }
+
     /**
      * Returns whether to preserve auto-follow patterns. Defaults to not
      * preserving them. Only runs at all if xpack is installed on the cluster
@@ -560,7 +568,7 @@ private void wipeCluster() throws Exception {
         }
 
         if (hasXPack && false == preserveILMPoliciesUponCompletion()) {
-            deleteAllILMPolicies();
+            deleteAllILMPolicies(preserveILMPolicyIds());
         }
 
         if (hasXPack && false == preserveAutoFollowPatternsUponCompletion()) {
@@ -686,7 +694,7 @@ private void waitForPendingRollupTasks() throws Exception {
         waitForPendingTasks(adminClient(), taskName -> taskName.startsWith("xpack/rollup/job") == false);
     }
 
-    private static void deleteAllILMPolicies() throws IOException {
+    private static void deleteAllILMPolicies(Set<String> exclusions) throws IOException {
         Map<String, Object> policies;
 
         try {
@@ -704,9 +712,15 @@ private static void deleteAllILMPolicies() throws IOException {
             return;
         }
 
-        for (String policyName : policies.keySet()) {
-            adminClient().performRequest(new Request("DELETE", "/_ilm/policy/" + policyName));
-        }
+        policies.keySet().stream()
+            .filter(p -> exclusions.contains(p) == false)
+            .forEach(policyName -> {
+                try {
+                    adminClient().performRequest(new Request("DELETE", "/_ilm/policy/" + policyName));
+                } catch (IOException e) {
+                    throw new RuntimeException("failed to delete policy: " + policyName, e);
+                }
+            });
     }
 
     private static void deleteAllSLMPolicies() throws IOException {

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecycleSettings.java

@@ -19,6 +19,7 @@ public class LifecycleSettings {
     public static final String LIFECYCLE_INDEXING_COMPLETE = "index.lifecycle.indexing_complete";
     public static final String LIFECYCLE_ORIGINATION_DATE = "index.lifecycle.origination_date";
     public static final String LIFECYCLE_PARSE_ORIGINATION_DATE = "index.lifecycle.parse_origination_date";
+    public static final String LIFECYCLE_HISTORY_INDEX_ENABLED = "index.lifecycle.history_index_enabled";
 
     public static final String SLM_HISTORY_INDEX_ENABLED = "slm.history_index_enabled";
     public static final String SLM_RETENTION_SCHEDULE = "slm.retention_schedule";
@@ -35,6 +36,8 @@ public class LifecycleSettings {
         Setting.longSetting(LIFECYCLE_ORIGINATION_DATE, -1, -1, Setting.Property.Dynamic, Setting.Property.IndexScope);
     public static final Setting<Boolean> LIFECYCLE_PARSE_ORIGINATION_DATE_SETTING = Setting.boolSetting(LIFECYCLE_PARSE_ORIGINATION_DATE,
         false, Setting.Property.Dynamic, Setting.Property.IndexScope);
+    public static final Setting<Boolean> LIFECYCLE_HISTORY_INDEX_ENABLED_SETTING = Setting.boolSetting(LIFECYCLE_HISTORY_INDEX_ENABLED,
+        true, Setting.Property.NodeScope);
 
 
     public static final Setting<Boolean> SLM_HISTORY_INDEX_ENABLED_SETTING = Setting.boolSetting(SLM_HISTORY_INDEX_ENABLED, true,

x-pack/plugin/core/src/main/resources/ilm-history-ilm-policy.json

@@ -0,0 +1,18 @@
+{
+  "phases": {
+    "hot": {
+      "actions": {
+        "rollover": {
+          "max_size": "50GB",
+          "max_age": "30d"
+        }
+      }
+    },
+    "delete": {
+      "min_age": "90d",
+      "actions": {
+        "delete": {}
+      }
+    }
+  }
+}

x-pack/plugin/core/src/main/resources/ilm-history.json

@@ -0,0 +1,83 @@
+{
+  "index_patterns": [
+    "ilm-history-${xpack.ilm_history.template.version}*"
+  ],
+  "order": 2147483647,
+  "settings": {
+    "index.number_of_shards": 1,
+    "index.number_of_replicas": 0,
+    "index.auto_expand_replicas": "0-1",
+    "index.lifecycle.name": "ilm-history-ilm-policy",
+    "index.lifecycle.rollover_alias": "ilm-history-${xpack.ilm_history.template.version}",
+    "index.format": 1
+  },
+  "mappings": {
+    "_doc": {
+      "dynamic": false,
+      "properties": {
+        "@timestamp": {
+          "type": "date",
+          "format": "epoch_millis"
+        },
+        "policy": {
+          "type": "keyword"
+        },
+        "index": {
+          "type": "keyword"
+        },
+        "index_age":{
+          "type": "long"
+        },
+        "success": {
+          "type": "boolean"
+        },
+        "state": {
+          "type": "object",
+          "dynamic": true,
+          "properties": {
+            "phase": {
+              "type": "keyword"
+            },
+            "action": {
+              "type": "keyword"
+            },
+            "step": {
+              "type": "keyword"
+            },
+            "failed_step": {
+              "type": "keyword"
+            },
+            "is_auto-retryable_error": {
+              "type": "keyword"
+            },
+            "creation_date": {
+              "type": "date",
+              "format": "epoch_millis"
+            },
+            "phase_time": {
+              "type": "date",
+              "format": "epoch_millis"
+            },
+            "action_time": {
+              "type": "date",
+              "format": "epoch_millis"
+            },
+            "step_time": {
+              "type": "date",
+              "format": "epoch_millis"
+            },
+            "phase_definition": {
+              "type": "text"
+            },
+            "step_info": {
+              "type": "text"
+            }
+          }
+        },
+        "error_details": {
+          "type": "text"
+        }
+      }
+    }
+  }
+}

x-pack/plugin/ilm/qa/multi-node/build.gradle

@@ -27,4 +27,6 @@ testClusters.integTest {
   setting 'xpack.ml.enabled', 'false'
   setting 'xpack.license.self_generated.type', 'trial'
   setting 'indices.lifecycle.poll_interval', '1000ms'
+  setting 'logger.org.elasticsearch.xpack.core.ilm', 'TRACE'
+  setting 'logger.org.elasticsearch.xpack.ilm', 'TRACE'
 }