Add ignore_above for message.raw field in notifications index mappings (

elastic#64455) (elastic#64570) Setting ignore_above to 1024 by default for the message.raw field in .ml-notifications-* and .transform-notifications* indices's mappings, in order to prevent indexing failed when the content length of the field exceeds the limit of keyword type. Relates to elastic#63888 Co-authored-by: bellengao <[email protected]>
andreidan · Nov 4, 2020 · 6104855 · 6104855
1 parent 5219309
commit 6104855
Show file tree

Hide file tree

Showing 3 changed files with 11 additions and 2 deletions.
diff --git a/...core/src/main/resources/org/elasticsearch/xpack/core/ml/notifications_index_template.json b/...core/src/main/resources/org/elasticsearch/xpack/core/ml/notifications_index_template.json
@@ -28,7 +28,8 @@
           "type": "text",
           "fields": {
             "raw": {
-              "type": "keyword"
+              "type": "keyword",
+              "ignore_above": 1024
             }
           }
         },

diff --git a/...m/src/main/java/org/elasticsearch/xpack/transform/persistence/TransformInternalIndex.java b/...m/src/main/java/org/elasticsearch/xpack/transform/persistence/TransformInternalIndex.java
@@ -71,6 +71,7 @@ public final class TransformInternalIndex {
     public static final String TEXT = "text";
     public static final String FIELDS = "fields";
     public static final String RAW = "raw";
+    public static final String IGNORE_ABOVE = "ignore_above";
 
     // data types
     public static final String FLOAT = "float";
@@ -137,6 +138,7 @@ private static XContentBuilder auditMappings() throws IOException {
                 .startObject(FIELDS)
                     .startObject(RAW)
                         .field(TYPE, KEYWORD)
+                        .field(IGNORE_ABOVE, 1024)
                     .endObject()
                 .endObject()
             .endObject()

diff --git a/x-pack/qa/src/main/java/org/elasticsearch/xpack/test/rest/IndexMappingTemplateAsserter.java b/x-pack/qa/src/main/java/org/elasticsearch/xpack/test/rest/IndexMappingTemplateAsserter.java
@@ -84,6 +84,11 @@ public static void assertMlMappingsMatchTemplates(RestClient client) throws IOEx
         statsIndexException.add("properties.hyperparameters.properties.regularization_soft_tree_depth_tolerance.type");
         statsIndexException.add("properties.hyperparameters.properties.regularization_tree_size_penalty_multiplier.type");
 
+        // Excluding this from notifications index as `ignore_above` has been added to the `message.raw` field.
+        // The exception is necessary for Full Cluster Restart tests.
+        Set<String> notificationsIndexExceptions = new HashSet<>();
+        notificationsIndexExceptions.add("properties.message.fields.raw.ignore_above");
+
         assertLegacyTemplateMatchesIndexMappings(client, ".ml-config", ".ml-config", false, configIndexExceptions, true);
         // the true parameter means the index may not have been created
         assertLegacyTemplateMatchesIndexMappings(client, ".ml-meta", ".ml-meta", true, Collections.emptySet(), true);
@@ -92,7 +97,7 @@ public static void assertMlMappingsMatchTemplates(RestClient client) throws IOEx
         assertLegacyTemplateMatchesIndexMappings(client, ".ml-state", ".ml-state-000001", true, Collections.emptySet(), false);
         // Depending on the order Full Cluster restart tests are run there may not be an notifications index yet
         assertLegacyTemplateMatchesIndexMappings(client,
-            ".ml-notifications-000001", ".ml-notifications-000001", true, Collections.emptySet(), false);
+            ".ml-notifications-000001", ".ml-notifications-000001", true, notificationsIndexExceptions, false);
         assertLegacyTemplateMatchesIndexMappings(client,
             ".ml-inference-000003", ".ml-inference-000003", true, Collections.emptySet(), true);
         // .ml-annotations-6 does not use a template
@@ -189,6 +194,7 @@ public static void assertLegacyTemplateMatchesIndexMappings(RestClient client,
 
         SortedSet<String> keysInTemplateMissingFromIndex = new TreeSet<>(flatTemplateMap.keySet());
         keysInTemplateMissingFromIndex.removeAll(flatIndexMap.keySet());
+        keysInTemplateMissingFromIndex.removeAll(exceptions);
 
         SortedSet<String> keysInIndexMissingFromTemplate = new TreeSet<>(flatIndexMap.keySet());
         keysInIndexMissingFromTemplate.removeAll(flatTemplateMap.keySet());