From 1465ae859d0c9fe797f24131dfe3fcc556c81be8 Mon Sep 17 00:00:00 2001 From: Georgii Gorbachev Date: Fri, 15 Oct 2021 18:04:16 +0200 Subject: [PATCH] [7.15] [Security Solution][Detections] Truncate lastFailureMessage for siem-detection-engine-rule-status documents (#112257) (#115166) **Ticket:** https://github.com/elastic/kibana/issues/109815 **Background:** `siem-detection-engine-rule-status` documents stores the `lastFailureMessage` a string which is indexed as `type: "text"` but some failure messages are so large that these documents are up to 26MB. These large documents cause migrations to fail because a batch of 1000 documents easily exceed Elasticsearch's `http.max_content_length` which defaults to 100mb. This PR truncates `lastFailureMessage` and `lastSuccessMessage` in the following cases: 1. When we write new or update existing status SOs: - The lists of errors/warnings are deduped -> truncated to max `20` items -> joined to a string - The resulting strings are truncated to max `10240` characters 2. When we migrate `siem-detection-engine-rule-status` SOs to 7.15.2: - The two message fields are truncated to max `10240` characters Delete any items that are not applicable to this PR. - [ ] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios --- .../rule_execution_log/index.ts | 10 ++++++ .../rule_execution_log_client.ts | 18 ++++++++-- .../rule_execution_log/utils/normalization.ts | 34 +++++++++++++++++++ .../rules/saved_object_mappings.ts | 20 ++++++++++- .../signals/signal_rule_alert_type.ts | 8 +++-- 5 files changed, 84 insertions(+), 6 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/index.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/utils/normalization.ts diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/index.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/index.ts new file mode 100644 index 0000000000000..5c7d9a875056a --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/index.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './rule_execution_log_client'; +export * from './types'; +export * from './utils/normalization'; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/rule_execution_log_client.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/rule_execution_log_client.ts index 26b36c367bda6..16bdb338d2168 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/rule_execution_log_client.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/rule_execution_log_client.ts @@ -19,6 +19,7 @@ import { LogStatusChangeArgs, UpdateExecutionLogArgs, } from './types'; +import { truncateMessage } from './utils/normalization'; export interface RuleExecutionLogClientArgs { ruleDataService: IRuleDataPluginService; @@ -51,7 +52,16 @@ export class RuleExecutionLogClient implements IRuleExecutionLogClient { } public async update(args: UpdateExecutionLogArgs) { - return this.client.update(args); + const { lastFailureMessage, lastSuccessMessage, ...restAttributes } = args.attributes; + + return this.client.update({ + ...args, + attributes: { + lastFailureMessage: truncateMessage(lastFailureMessage), + lastSuccessMessage: truncateMessage(lastSuccessMessage), + ...restAttributes, + }, + }); } public async delete(id: string) { @@ -63,6 +73,10 @@ export class RuleExecutionLogClient implements IRuleExecutionLogClient { } public async logStatusChange(args: LogStatusChangeArgs) { - return this.client.logStatusChange(args); + const message = args.message ? truncateMessage(args.message) : args.message; + return this.client.logStatusChange({ + ...args, + message, + }); } } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/utils/normalization.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/utils/normalization.ts new file mode 100644 index 0000000000000..baaee9446eee3 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/utils/normalization.ts @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { take, toString, truncate, uniq } from 'lodash'; + +// When we write rule execution status updates to `siem-detection-engine-rule-status` saved objects +// or to event log, we write success and failure messages as well. Those messages are built from +// N errors collected during the "big loop" in the Detection Engine, where N can be very large. +// When N is large the resulting message strings are so large that these documents are up to 26MB. +// These large documents may cause migrations to fail because a batch of 1000 documents easily +// exceed Elasticsearch's `http.max_content_length` which defaults to 100mb. +// In order to fix that, we need to truncate those messages to an adequate MAX length. +// https://github.com/elastic/kibana/pull/112257 + +const MAX_MESSAGE_LENGTH = 10240; +const MAX_LIST_LENGTH = 20; + +export const truncateMessage = (value: unknown): string | undefined => { + if (value === undefined) { + return value; + } + + const str = toString(value); + return truncate(str, { length: MAX_MESSAGE_LENGTH }); +}; + +export const truncateMessageList = (list: string[]): string[] => { + const deduplicatedList = uniq(list); + return take(deduplicatedList, MAX_LIST_LENGTH); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/saved_object_mappings.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/saved_object_mappings.ts index 813e800f34ce2..d347fccf6b77b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/saved_object_mappings.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/saved_object_mappings.ts @@ -5,7 +5,8 @@ * 2.0. */ -import { SavedObjectsType } from '../../../../../../../src/core/server'; +import { SavedObjectsType, SavedObjectMigrationFn } from 'kibana/server'; +import { truncateMessage } from '../rule_execution_log'; export const ruleStatusSavedObjectType = 'siem-detection-engine-rule-status'; @@ -47,11 +48,28 @@ export const ruleStatusSavedObjectMappings: SavedObjectsType['mappings'] = { }, }; +const truncateMessageFields: SavedObjectMigrationFn> = (doc) => { + const { lastFailureMessage, lastSuccessMessage, ...restAttributes } = doc.attributes; + + return { + ...doc, + attributes: { + lastFailureMessage: truncateMessage(lastFailureMessage), + lastSuccessMessage: truncateMessage(lastSuccessMessage), + ...restAttributes, + }, + references: doc.references ?? [], + }; +}; + export const type: SavedObjectsType = { name: ruleStatusSavedObjectType, hidden: false, namespaceType: 'single', mappings: ruleStatusSavedObjectMappings, + migrations: { + '7.15.2': truncateMessageFields, + }, }; export const ruleAssetSavedObjectType = 'security-rule'; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts index 444a087821a95..6db8a0e20a759 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -70,8 +70,8 @@ import { wrapSequencesFactory } from './wrap_sequences_factory'; import { ConfigType } from '../../../config'; import { ExperimentalFeatures } from '../../../../common/experimental_features'; import { injectReferences, extractReferences } from './saved_object_references'; -import { RuleExecutionLogClient } from '../rule_execution_log/rule_execution_log_client'; import { IRuleDataPluginService } from '../rule_execution_log/types'; +import { RuleExecutionLogClient, truncateMessageList } from '../rule_execution_log'; export const signalRulesAlertType = ({ logger, @@ -362,7 +362,9 @@ export const signalRulesAlertType = ({ throw new Error(`unknown rule type ${type}`); } if (result.warningMessages.length) { - const warningMessage = buildRuleMessage(result.warningMessages.join()); + const warningMessage = buildRuleMessage( + truncateMessageList(result.warningMessages).join() + ); await ruleStatusService.partialFailure(warningMessage); } @@ -427,7 +429,7 @@ export const signalRulesAlertType = ({ } else { const errorMessage = buildRuleMessage( 'Bulk Indexing of signals failed:', - result.errors.join() + truncateMessageList(result.errors).join() ); logger.error(errorMessage); await ruleStatusService.error(errorMessage, {