[Security Solution][Detections] Truncate lastFailureMessage for siem-…

…detection-engine-rule-status documents (#112257)

**Ticket:** #109815

## Summary

**Background:** `siem-detection-engine-rule-status` documents stores the `lastFailureMessage` a string which is indexed as `type: "text"` but some failure messages are so large that these documents are up to 26MB. These large documents cause migrations to fail because a batch of 1000 documents easily exceed Elasticsearch's `http.max_content_length` which defaults to 100mb.

This PR truncates `lastFailureMessage` and `lastSuccessMessage` in the following cases:

1. When we write new or update existing status SOs:
    - The lists of errors/warnings are deduped -> truncated to max `20` items -> joined to a string
    - The resulting strings are truncated to max `10240` characters
2. When we migrate `siem-detection-engine-rule-status` SOs to 7.15.2:
    - The two message fields are truncated to max `10240` characters

### Checklist

Delete any items that are not applicable to this PR.

- [ ] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios

x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/index.ts

@@ -0,0 +1,10 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export * from './rule_execution_log_client';
+export * from './types';
+export * from './utils/normalization';

x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/rule_execution_log_client.ts

@@ -18,6 +18,7 @@ import {
   UpdateExecutionLogArgs,
   UnderlyingLogClient,
 } from './types';
+import { truncateMessage } from './utils/normalization';
 
 export interface RuleExecutionLogClientArgs {
   savedObjectsClient: SavedObjectsClientContract;
@@ -52,7 +53,16 @@ export class RuleExecutionLogClient implements IRuleExecutionLogClient {
   }
 
   public async update(args: UpdateExecutionLogArgs) {
-    return this.client.update(args);
+    const { lastFailureMessage, lastSuccessMessage, ...restAttributes } = args.attributes;
+
+    return this.client.update({
+      ...args,
+      attributes: {
+        lastFailureMessage: truncateMessage(lastFailureMessage),
+        lastSuccessMessage: truncateMessage(lastSuccessMessage),
+        ...restAttributes,
+      },
+    });
   }
 
   public async delete(id: string) {
@@ -64,6 +74,10 @@ export class RuleExecutionLogClient implements IRuleExecutionLogClient {
   }
 
   public async logStatusChange(args: LogStatusChangeArgs) {
-    return this.client.logStatusChange(args);
+    const message = args.message ? truncateMessage(args.message) : args.message;
+    return this.client.logStatusChange({
+      ...args,
+      message,
+    });
   }
 }

x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/utils/normalization.ts

@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { take, toString, truncate, uniq } from 'lodash';
+
+// When we write rule execution status updates to `siem-detection-engine-rule-status` saved objects
+// or to event log, we write success and failure messages as well. Those messages are built from
+// N errors collected during the "big loop" in the Detection Engine, where N can be very large.
+// When N is large the resulting message strings are so large that these documents are up to 26MB.
+// These large documents may cause migrations to fail because a batch of 1000 documents easily
+// exceed Elasticsearch's `http.max_content_length` which defaults to 100mb.
+// In order to fix that, we need to truncate those messages to an adequate MAX length.
+// https://github.com/elastic/kibana/pull/112257
+
+const MAX_MESSAGE_LENGTH = 10240;
+const MAX_LIST_LENGTH = 20;
+
+export const truncateMessage = (value: unknown): string | undefined => {
+  if (value === undefined) {
+    return value;
+  }
+
+  const str = toString(value);
+  return truncate(str, { length: MAX_MESSAGE_LENGTH });
+};
+
+export const truncateMessageList = (list: string[]): string[] => {
+  const deduplicatedList = uniq(list);
+  return take(deduplicatedList, MAX_LIST_LENGTH);
+};

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_wrapper.ts

@@ -38,7 +38,7 @@ import {
 import { getNotificationResultsLink } from '../notifications/utils';
 import { createResultObject } from './utils';
 import { bulkCreateFactory, wrapHitsFactory, wrapSequencesFactory } from './factories';
-import { RuleExecutionLogClient } from '../rule_execution_log/rule_execution_log_client';
+import { RuleExecutionLogClient, truncateMessageList } from '../rule_execution_log';
 import { RuleExecutionStatus } from '../../../../common/detection_engine/schemas/common/schemas';
 import { scheduleThrottledNotificationActions } from '../notifications/schedule_throttle_notification_actions';
 import { AlertAttributes } from '../signals/types';
@@ -282,7 +282,9 @@ export const createSecurityRuleTypeWrapper: CreateSecurityRuleTypeWrapper =
           }
 
           if (result.warningMessages.length) {
-            const warningMessage = buildRuleMessage(result.warningMessages.join());
+            const warningMessage = buildRuleMessage(
+              truncateMessageList(result.warningMessages).join()
+            );
             await ruleStatusClient.logStatusChange({
               ...basicLogArguments,
               newStatus: RuleExecutionStatus['partial failure'],
@@ -372,7 +374,7 @@ export const createSecurityRuleTypeWrapper: CreateSecurityRuleTypeWrapper =
           } else {
             const errorMessage = buildRuleMessage(
               'Bulk Indexing of signals failed:',
-              result.errors.join()
+              truncateMessageList(result.errors).join()
             );
             logger.error(errorMessage);
             await ruleStatusClient.logStatusChange({

x-pack/plugins/security_solution/server/lib/detection_engine/rules/saved_object_mappings.ts

@@ -5,7 +5,8 @@
  * 2.0.
  */
 
-import { SavedObjectsType } from '../../../../../../../src/core/server';
+import { SavedObjectsType, SavedObjectMigrationFn } from 'kibana/server';
+import { truncateMessage } from '../rule_execution_log';
 
 export const ruleStatusSavedObjectType = 'siem-detection-engine-rule-status';
 
@@ -47,11 +48,28 @@ export const ruleStatusSavedObjectMappings: SavedObjectsType['mappings'] = {
   },
 };
 
+const truncateMessageFields: SavedObjectMigrationFn<Record<string, unknown>> = (doc) => {
+  const { lastFailureMessage, lastSuccessMessage, ...restAttributes } = doc.attributes;
+
+  return {
+    ...doc,
+    attributes: {
+      lastFailureMessage: truncateMessage(lastFailureMessage),
+      lastSuccessMessage: truncateMessage(lastSuccessMessage),
+      ...restAttributes,
+    },
+    references: doc.references ?? [],
+  };
+};
+
 export const type: SavedObjectsType = {
   name: ruleStatusSavedObjectType,
   hidden: false,
   namespaceType: 'single',
   mappings: ruleStatusSavedObjectMappings,
+  migrations: {
+    '7.15.2': truncateMessageFields,
+  },
 };
 
 export const ruleAssetSavedObjectType = 'security-rule';

x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts

@@ -69,7 +69,7 @@ import { wrapSequencesFactory } from './wrap_sequences_factory';
 import { ConfigType } from '../../../config';
 import { ExperimentalFeatures } from '../../../../common/experimental_features';
 import { injectReferences, extractReferences } from './saved_object_references';
-import { RuleExecutionLogClient } from '../rule_execution_log/rule_execution_log_client';
+import { RuleExecutionLogClient, truncateMessageList } from '../rule_execution_log';
 import { RuleExecutionStatus } from '../../../../common/detection_engine/schemas/common/schemas';
 import { scheduleThrottledNotificationActions } from '../notifications/schedule_throttle_notification_actions';
 import { IEventLogService } from '../../../../../event_log/server';
@@ -384,7 +384,9 @@ export const signalRulesAlertType = ({
           throw new Error(`unknown rule type ${type}`);
         }
         if (result.warningMessages.length) {
-          const warningMessage = buildRuleMessage(result.warningMessages.join());
+          const warningMessage = buildRuleMessage(
+            truncateMessageList(result.warningMessages).join()
+          );
           await ruleStatusClient.logStatusChange({
             ...basicLogArguments,
             newStatus: RuleExecutionStatus['partial failure'],
@@ -471,7 +473,7 @@ export const signalRulesAlertType = ({
         } else {
           const errorMessage = buildRuleMessage(
             'Bulk Indexing of signals failed:',
-            result.errors.join()
+            truncateMessageList(result.errors).join()
           );
           logger.error(errorMessage);
           await ruleStatusClient.logStatusChange({