[alerting] change eventLog schema to use dynamic false (#61633) (#61707)

resolves #58518

Changes the object properties in the eventLog mappings to use `dynamic: false`
instead of `dynamic: strict`.  This provides a bit of a safety net for cases
where the mappings change during development, or potentially in production
cases.  Rather than completely lose entire events and see errors in logs, we'll
silently drop properties from events.

While researching the `dynamic` property, I remember there's also a `meta`
property available, which is a convenient place to drop our meta-data about
a field being an array value, for internal processing.  Bonus, it can live in
the generated mappings as well.

references:

- https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-field-meta.html
- https://www.elastic.co/guide/en/elasticsearch/reference/current/dynamic.html

Co-authored-by: Elastic Machine <[email protected]>

x-pack/plugins/event_log/generated/mappings.json

@@ -1,12 +1,15 @@
 {
-    "dynamic": "strict",
+    "dynamic": "false",
     "properties": {
         "@timestamp": {
             "type": "date"
         },
         "tags": {
             "ignore_above": 1024,
-            "type": "keyword"
+            "type": "keyword",
+            "meta": {
+                "isArray": true
+            }
         },
         "message": {
             "norms": false,
@@ -18,8 +21,7 @@
                     "ignore_above": 1024,
                     "type": "keyword"
                 }
-            },
-            "dynamic": "strict"
+            }
         },
         "event": {
             "properties": {
@@ -40,26 +42,23 @@
                 "end": {
                     "type": "date"
                 }
-            },
-            "dynamic": "strict"
+            }
         },
         "error": {
             "properties": {
                 "message": {
                     "norms": false,
                     "type": "text"
                 }
-            },
-            "dynamic": "strict"
+            }
         },
         "user": {
             "properties": {
                 "name": {
                     "ignore_above": 1024,
                     "type": "keyword"
                 }
-            },
-            "dynamic": "strict"
+            }
         },
         "kibana": {
             "properties": {
@@ -86,11 +85,9 @@
                             "ignore_above": 1024
                         }
                     },
-                    "type": "nested",
-                    "dynamic": "strict"
+                    "type": "nested"
                 }
-            },
-            "dynamic": "strict"
+            }
         }
     }
 }

x-pack/plugins/event_log/scripts/create_schemas.js

@@ -30,11 +30,13 @@ function main() {
   const exportedProperties = mappings.EcsEventLogProperties;
   const multiValuedProperties = new Set(mappings.EcsEventLogMultiValuedProperties);
 
+  augmentMappings(ecsMappings.mappings, multiValuedProperties);
+
   const elMappings = getEventLogMappings(ecsMappings, exportedProperties);
 
   console.log(`generating files in ${PLUGIN_DIR}`);
   writeEventLogMappings(elMappings);
-  writeEventLogConfigSchema(elMappings, ecsVersion, multiValuedProperties);
+  writeEventLogConfigSchema(elMappings, ecsVersion);
 }
 
 // return a stripped down version of the ecs schema, with only exportedProperties
@@ -57,7 +59,6 @@ function getEventLogMappings(ecsSchema, exportedProperties) {
     const elValue = lodash.get(result.mappings.properties, prop);
 
     elValue.type = ecsValue.type;
-    elValue.dynamic = 'strict';
   }
 
   return result;
@@ -86,19 +87,18 @@ function writeEventLogMappings(elSchema) {
   // fixObjectTypes(elSchema.mappings);
 
   const mappings = {
-    dynamic: 'strict',
+    dynamic: 'false',
     properties: elSchema.mappings.properties,
   };
 
   writeGeneratedFile(EVENT_LOG_MAPPINGS_FILE, JSON.stringify(mappings, null, 4));
   console.log('generated:', EVENT_LOG_MAPPINGS_FILE);
 }
 
-function writeEventLogConfigSchema(elSchema, ecsVersion, multiValuedProperties) {
+function writeEventLogConfigSchema(elSchema, ecsVersion) {
   const lineWriter = LineWriter.createLineWriter();
 
-  const elSchemaMappings = augmentMappings(elSchema.mappings, multiValuedProperties);
-  generateSchemaLines(lineWriter, null, elSchemaMappings);
+  generateSchemaLines(lineWriter, null, elSchema.mappings);
   // last line will have an extraneous comma
   const schemaLines = lineWriter.getContent().replace(/,$/, '');
 
@@ -113,22 +113,21 @@ const StringTypes = new Set(['string', 'keyword', 'text', 'ip']);
 const NumberTypes = new Set(['long', 'integer', 'float']);
 
 function augmentMappings(mappings, multiValuedProperties) {
-  // clone the mappings, as we're adding some additional properties
-  mappings = JSON.parse(JSON.stringify(mappings));
-
   for (const prop of multiValuedProperties) {
     const fullProp = replaceDotWithProperties(prop);
-    lodash.set(mappings.properties, `${fullProp}.multiValued`, true);
+    const metaPropName = `${fullProp}.meta`;
+    const meta = lodash.get(mappings.properties, metaPropName) || {};
+    meta.isArray = true;
+    lodash.set(mappings.properties, metaPropName, meta);
   }
-
-  return mappings;
 }
 
 function generateSchemaLines(lineWriter, prop, mappings) {
   const propKey = legalPropertyName(prop);
+  if (mappings == null) return;
 
   if (StringTypes.has(mappings.type)) {
-    if (mappings.multiValued) {
+    if (mappings.meta && mappings.meta.isArray) {
       lineWriter.addLine(`${propKey}: ecsStringMulti(),`);
     } else {
       lineWriter.addLine(`${propKey}: ecsString(),`);
@@ -169,6 +168,7 @@ function generateSchemaLines(lineWriter, prop, mappings) {
   // write the object properties
   lineWriter.indent();
   for (const prop of Object.keys(mappings.properties)) {
+    if (prop === 'meta') continue;
     generateSchemaLines(lineWriter, prop, mappings.properties[prop]);
   }
   lineWriter.dedent();