From 2322cda3a402ab2b56a8a365f6f0f16d9ffd5bc0 Mon Sep 17 00:00:00 2001 From: Georgii Gorbachev Date: Thu, 14 Oct 2021 17:40:23 +0200 Subject: [PATCH] [Security Solution][Detections] Truncate lastFailureMessage for siem-detection-engine-rule-status documents (#112257) **Ticket:** https://github.com/elastic/kibana/issues/109815 ## Summary **Background:** `siem-detection-engine-rule-status` documents stores the `lastFailureMessage` a string which is indexed as `type: "text"` but some failure messages are so large that these documents are up to 26MB. These large documents cause migrations to fail because a batch of 1000 documents easily exceed Elasticsearch's `http.max_content_length` which defaults to 100mb. This PR truncates `lastFailureMessage` and `lastSuccessMessage` in the following cases: 1. When we write new or update existing status SOs: - The lists of errors/warnings are deduped -> truncated to max `20` items -> joined to a string - The resulting strings are truncated to max `10240` characters 2. When we migrate `siem-detection-engine-rule-status` SOs to 7.15.2: - The two message fields are truncated to max `10240` characters ### Checklist Delete any items that are not applicable to this PR. - [ ] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios --- .../rule_execution_log/index.ts | 10 ++++++ .../rule_execution_log_client.ts | 18 ++++++++-- .../rule_execution_log/utils/normalization.ts | 34 +++++++++++++++++++ .../create_security_rule_type_wrapper.ts | 8 +++-- .../rules/saved_object_mappings.ts | 20 ++++++++++- .../signals/signal_rule_alert_type.ts | 8 +++-- 6 files changed, 89 insertions(+), 9 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/index.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/utils/normalization.ts diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/index.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/index.ts new file mode 100644 index 000000000000..5c7d9a875056 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/index.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './rule_execution_log_client'; +export * from './types'; +export * from './utils/normalization'; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/rule_execution_log_client.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/rule_execution_log_client.ts index 2d773fc35cce..7ae2f179f969 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/rule_execution_log_client.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/rule_execution_log_client.ts @@ -18,6 +18,7 @@ import { UpdateExecutionLogArgs, UnderlyingLogClient, } from './types'; +import { truncateMessage } from './utils/normalization'; export interface RuleExecutionLogClientArgs { savedObjectsClient: SavedObjectsClientContract; @@ -52,7 +53,16 @@ export class RuleExecutionLogClient implements IRuleExecutionLogClient { } public async update(args: UpdateExecutionLogArgs) { - return this.client.update(args); + const { lastFailureMessage, lastSuccessMessage, ...restAttributes } = args.attributes; + + return this.client.update({ + ...args, + attributes: { + lastFailureMessage: truncateMessage(lastFailureMessage), + lastSuccessMessage: truncateMessage(lastSuccessMessage), + ...restAttributes, + }, + }); } public async delete(id: string) { @@ -64,6 +74,10 @@ export class RuleExecutionLogClient implements IRuleExecutionLogClient { } public async logStatusChange(args: LogStatusChangeArgs) { - return this.client.logStatusChange(args); + const message = args.message ? truncateMessage(args.message) : args.message; + return this.client.logStatusChange({ + ...args, + message, + }); } } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/utils/normalization.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/utils/normalization.ts new file mode 100644 index 000000000000..baaee9446eee --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/utils/normalization.ts @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { take, toString, truncate, uniq } from 'lodash'; + +// When we write rule execution status updates to `siem-detection-engine-rule-status` saved objects +// or to event log, we write success and failure messages as well. Those messages are built from +// N errors collected during the "big loop" in the Detection Engine, where N can be very large. +// When N is large the resulting message strings are so large that these documents are up to 26MB. +// These large documents may cause migrations to fail because a batch of 1000 documents easily +// exceed Elasticsearch's `http.max_content_length` which defaults to 100mb. +// In order to fix that, we need to truncate those messages to an adequate MAX length. +// https://github.com/elastic/kibana/pull/112257 + +const MAX_MESSAGE_LENGTH = 10240; +const MAX_LIST_LENGTH = 20; + +export const truncateMessage = (value: unknown): string | undefined => { + if (value === undefined) { + return value; + } + + const str = toString(value); + return truncate(str, { length: MAX_MESSAGE_LENGTH }); +}; + +export const truncateMessageList = (list: string[]): string[] => { + const deduplicatedList = uniq(list); + return take(deduplicatedList, MAX_LIST_LENGTH); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_wrapper.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_wrapper.ts index b037e572f21b..77981d92b2ba 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_wrapper.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_wrapper.ts @@ -38,7 +38,7 @@ import { import { getNotificationResultsLink } from '../notifications/utils'; import { createResultObject } from './utils'; import { bulkCreateFactory, wrapHitsFactory, wrapSequencesFactory } from './factories'; -import { RuleExecutionLogClient } from '../rule_execution_log/rule_execution_log_client'; +import { RuleExecutionLogClient, truncateMessageList } from '../rule_execution_log'; import { RuleExecutionStatus } from '../../../../common/detection_engine/schemas/common/schemas'; import { scheduleThrottledNotificationActions } from '../notifications/schedule_throttle_notification_actions'; import { AlertAttributes } from '../signals/types'; @@ -282,7 +282,9 @@ export const createSecurityRuleTypeWrapper: CreateSecurityRuleTypeWrapper = } if (result.warningMessages.length) { - const warningMessage = buildRuleMessage(result.warningMessages.join()); + const warningMessage = buildRuleMessage( + truncateMessageList(result.warningMessages).join() + ); await ruleStatusClient.logStatusChange({ ...basicLogArguments, newStatus: RuleExecutionStatus['partial failure'], @@ -372,7 +374,7 @@ export const createSecurityRuleTypeWrapper: CreateSecurityRuleTypeWrapper = } else { const errorMessage = buildRuleMessage( 'Bulk Indexing of signals failed:', - result.errors.join() + truncateMessageList(result.errors).join() ); logger.error(errorMessage); await ruleStatusClient.logStatusChange({ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/saved_object_mappings.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/saved_object_mappings.ts index 813e800f34ce..d347fccf6b77 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/saved_object_mappings.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/saved_object_mappings.ts @@ -5,7 +5,8 @@ * 2.0. */ -import { SavedObjectsType } from '../../../../../../../src/core/server'; +import { SavedObjectsType, SavedObjectMigrationFn } from 'kibana/server'; +import { truncateMessage } from '../rule_execution_log'; export const ruleStatusSavedObjectType = 'siem-detection-engine-rule-status'; @@ -47,11 +48,28 @@ export const ruleStatusSavedObjectMappings: SavedObjectsType['mappings'] = { }, }; +const truncateMessageFields: SavedObjectMigrationFn> = (doc) => { + const { lastFailureMessage, lastSuccessMessage, ...restAttributes } = doc.attributes; + + return { + ...doc, + attributes: { + lastFailureMessage: truncateMessage(lastFailureMessage), + lastSuccessMessage: truncateMessage(lastSuccessMessage), + ...restAttributes, + }, + references: doc.references ?? [], + }; +}; + export const type: SavedObjectsType = { name: ruleStatusSavedObjectType, hidden: false, namespaceType: 'single', mappings: ruleStatusSavedObjectMappings, + migrations: { + '7.15.2': truncateMessageFields, + }, }; export const ruleAssetSavedObjectType = 'security-rule'; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts index 1e3a8a513c4a..2094264cbf15 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -69,7 +69,7 @@ import { wrapSequencesFactory } from './wrap_sequences_factory'; import { ConfigType } from '../../../config'; import { ExperimentalFeatures } from '../../../../common/experimental_features'; import { injectReferences, extractReferences } from './saved_object_references'; -import { RuleExecutionLogClient } from '../rule_execution_log/rule_execution_log_client'; +import { RuleExecutionLogClient, truncateMessageList } from '../rule_execution_log'; import { RuleExecutionStatus } from '../../../../common/detection_engine/schemas/common/schemas'; import { scheduleThrottledNotificationActions } from '../notifications/schedule_throttle_notification_actions'; import { IEventLogService } from '../../../../../event_log/server'; @@ -384,7 +384,9 @@ export const signalRulesAlertType = ({ throw new Error(`unknown rule type ${type}`); } if (result.warningMessages.length) { - const warningMessage = buildRuleMessage(result.warningMessages.join()); + const warningMessage = buildRuleMessage( + truncateMessageList(result.warningMessages).join() + ); await ruleStatusClient.logStatusChange({ ...basicLogArguments, newStatus: RuleExecutionStatus['partial failure'], @@ -471,7 +473,7 @@ export const signalRulesAlertType = ({ } else { const errorMessage = buildRuleMessage( 'Bulk Indexing of signals failed:', - result.errors.join() + truncateMessageList(result.errors).join() ); logger.error(errorMessage); await ruleStatusClient.logStatusChange({