From be7f6cff3c39884aeffe6890022d5ddbfee8686b Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com> Date: Mon, 4 Dec 2023 13:07:28 +0000 Subject: [PATCH] [Security Solution][Detection Engine] adds alerts Suppression to threshold rule (#171423) ## Summary - addresses milestone 1 of https://github.com/elastic/security-team/issues/7773 epic - adds alerts suppression capabilities to threshold rule type - to enable alerts suppression for threshold rule type use experimental feature flag `alertSuppressionForThresholdRuleEnabled` in kibana.yml ``` xpack.securitySolution.enableExperimental: - alertSuppressionForThresholdRuleEnabled ``` - similarly to query rule Platinum license is required ### UI Few changes in comparison with custom query alerts suppression 1. Suppress by fields removed, since suppression is performed on Threshold Groups By fields 2. Instead, we show checkbox - so user can opt-in for alert suppression (either by selected threshold fields or w/o any) 3. Only time interval is radio button is available, suppression in rule execution is disabled(Threshold rule itself 'suppress' by grouping during rule execution) Demo video, shows suppression on interval when users select threshold group by fields and when do not https://github.com/elastic/kibana/assets/92328789/7dc476ad-0d0f-4e40-8042-d4dd552759d9
Suppression is enabled, threshold fields selected Screenshot 2023-11-27 at 16 44 04
Suppression is not enabled, threshold fields selected Screenshot 2023-11-27 at 16 44 27
Suppression is not enabled, threshold fields not selected Screenshot 2023-11-27 at 16 44 42
### Checklist - [x] Functional changes are hidden behind a feature flag Feature flag `alertSuppressionForThresholdRuleEnabled` - [x] Functional changes are covered with a test plan and automated tests. Test plan in progress(cc @vgomez-el), unit/ftr/cypress tests added to cover alert suppression functionality added - [x] Stability of new and changed tests is verified using the [Flaky Test Runner](https://ci-stats.kibana.dev/trigger_flaky_test_runner). [FTR ESS & Serverless tests](https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/4057) [Cypress ESS](https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/4058) [Cypress Serverless](https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/4059) - [ ] Comprehensive manual testing is done by two engineers: the PR author and one of the PR reviewers. Changes are tested in both ESS and Serverless. - [x] Mapping changes are accompanied by a technical design document. It can be a GitHub issue or an RFC explaining the changes. The design document is shared with and approved by the appropriate teams and individual stakeholders. Existing AlertSuppression schema field is used for Threshold rule, similarly to Query. But only `duration` field is applicable and required - [x] Functional changes are communicated to the Docs team. A ticket or PR is opened in https://github.com/elastic/security-docs. The following information is included: any feature flags used, affected environments (Serverless, ESS, or both). https://github.com/elastic/security-docs/issues/4315 --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- .../rule_schema/common_attributes.gen.ts | 6 + .../rule_schema/common_attributes.schema.yaml | 16 + .../rule_schema/rule_request_schema.test.ts | 49 ++ .../model/rule_schema/rule_schemas.gen.ts | 6 +- .../rule_schema/rule_schemas.schema.yaml | 2 + .../query_attributes.gen.ts | 8 +- .../query_attributes.schema.yaml | 20 +- .../threshold_attributes.gen.ts | 7 + .../threshold_attributes.schema.yaml | 8 + .../common/experimental_features.ts | 5 + .../pages/rule_creation/helpers.ts | 4 + .../pages/rule_creation/index.tsx | 4 + .../pages/rule_editing/index.tsx | 2 + .../rule_details/rule_definition_section.tsx | 74 +- .../components/rules_table/__mocks__/mock.ts | 1 + .../components/rules/duration_input/index.tsx | 3 +- .../rules/step_about_rule/index.test.tsx | 1 + .../rules/step_define_rule/index.test.tsx | 2 + .../rules/step_define_rule/index.tsx | 246 ++++-- .../rules/step_define_rule/schema.tsx | 4 + .../rules/step_define_rule/translations.tsx | 46 + .../detection_engine/rules/helpers.test.tsx | 1 + .../pages/detection_engine/rules/helpers.tsx | 15 +- .../pages/detection_engine/rules/types.ts | 3 +- .../pages/detection_engine/rules/utils.ts | 1 + .../bulk_actions/rule_params_modifier.ts | 2 +- .../normalization/rule_converters.test.ts | 17 + .../normalization/rule_converters.ts | 7 + .../rule_schema/model/rule_schemas.ts | 2 + .../create_security_rule_type_wrapper.ts | 1 + .../group_and_bulk_create.ts | 2 +- ...bulk_create_suppressed_threshold_alerts.ts | 102 +++ .../bulk_create_threshold_signals.ts | 71 +- .../threshold/create_threshold_alert_type.ts | 6 +- .../rule_types/threshold/threshold.test.ts | 9 + .../rule_types/threshold/threshold.ts | 97 ++- .../rule_types/threshold/utils.test.ts | 33 +- .../rule_types/threshold/utils.ts | 22 + .../wrap_suppressed_threshold_alerts.ts | 127 +++ .../lib/detection_engine/rule_types/types.ts | 1 + .../bulk_create_with_suppression.ts | 10 +- .../get_threshold_rule_for_signal_testing.ts | 1 + .../config/ess/config.base.ts | 1 + .../configs/serverless.config.ts | 3 + .../execution_logic/index.ts | 1 + .../execution_logic/threshold.ts | 66 +- .../threshold_alert_suppression.ts | 787 ++++++++++++++++++ .../keyword_mixed_with_const.ts | 1 + .../get_threshold_rule_for_alert_testing.ts | 1 + .../test/security_solution_cypress/config.ts | 1 + .../rule_creation/threshold_rule.cy.ts | 177 ++-- .../threshold_rule_ess_basic.cy.ts | 40 + ...threshold_rule_serverless_essentials.cy.ts | 44 + .../rule_edit/threshold_rule.cy.ts | 115 +++ .../prebuilt_rules_preview.cy.ts | 9 +- .../cypress/screens/create_new_rule.ts | 10 + .../cypress/screens/rule_details.ts | 2 + .../cypress/tasks/api_calls/licensing.ts | 15 + .../cypress/tasks/create_new_rule.ts | 33 + .../cypress/tsconfig.json | 3 +- .../serverless_config.ts | 3 + 61 files changed, 2043 insertions(+), 313 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/bulk_create_suppressed_threshold_alerts.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/wrap_suppressed_threshold_alerts.ts rename x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/{query/alert_suppression => utils}/bulk_create_with_suppression.ts (91%) create mode 100644 x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_execution_logic/execution_logic/threshold_alert_suppression.ts create mode 100644 x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_creation/threshold_rule_ess_basic.cy.ts create mode 100644 x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_creation/threshold_rule_serverless_essentials.cy.ts create mode 100644 x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_edit/threshold_rule.cy.ts create mode 100644 x-pack/test/security_solution_cypress/cypress/tasks/api_calls/licensing.ts diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/common_attributes.gen.ts b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/common_attributes.gen.ts index 4b9001dc35c00..11298104fc84a 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/common_attributes.gen.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/common_attributes.gen.ts @@ -432,3 +432,9 @@ export const RuleExceptionList = z.object({ */ namespace_type: z.enum(['agnostic', 'single']), }); + +export type AlertSuppressionDuration = z.infer; +export const AlertSuppressionDuration = z.object({ + value: z.number().int().min(1), + unit: z.enum(['s', 'm', 'h']), +}); diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/common_attributes.schema.yaml b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/common_attributes.schema.yaml index 047394c5843c7..35149d92f0c43 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/common_attributes.schema.yaml +++ b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/common_attributes.schema.yaml @@ -471,3 +471,19 @@ components: - list_id - type - namespace_type + + AlertSuppressionDuration: + type: object + properties: + value: + type: integer + minimum: 1 + unit: + type: string + enum: + - s + - m + - h + required: + - value + - unit diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_request_schema.test.ts b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_request_schema.test.ts index 3dfc4a294965f..961bc915b6010 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_request_schema.test.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_request_schema.test.ts @@ -1212,4 +1212,53 @@ describe('rules schema', () => { expect(stringifyZodError(result.error)).toEqual('investigation_fields.field_names: Required'); }); }); + + describe('alerts suppression', () => { + test('should drop suppression fields apart from duration for "threshold" rule type', () => { + const payload = { + ...getCreateThresholdRulesSchemaMock(), + alert_suppression: { + group_by: ['host.name'], + duration: { value: 5, unit: 'm' }, + missing_field_strategy: 'suppress', + }, + }; + + const result = RuleCreateProps.safeParse(payload); + expectParseSuccess(result); + expect(result.data).toEqual({ + ...payload, + alert_suppression: { + duration: { value: 5, unit: 'm' }, + }, + }); + }); + test('should validate only suppression duration for "threshold" rule type', () => { + const payload = { + ...getCreateThresholdRulesSchemaMock(), + alert_suppression: { + duration: { value: 5, unit: 'm' }, + }, + }; + + const result = RuleCreateProps.safeParse(payload); + expectParseSuccess(result); + expect(result.data).toEqual(payload); + }); + test('should throw error if alert suppression duration is absent for "threshold" rule type', () => { + const payload = { + ...getCreateThresholdRulesSchemaMock(), + alert_suppression: { + group_by: ['host.name'], + missing_field_strategy: 'suppress', + }, + }; + + const result = RuleCreateProps.safeParse(payload); + expectParseError(result); + expect(stringifyZodError(result.error)).toMatchInlineSnapshot( + `"alert_suppression.duration: Required"` + ); + }); + }); }); diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.gen.ts b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.gen.ts index 1051f59feac0a..0744b3fb57b5c 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.gen.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.gen.ts @@ -69,7 +69,10 @@ import { } from './specific_attributes/eql_attributes.gen'; import { ResponseAction } from '../rule_response_actions/response_actions.gen'; import { AlertSuppression } from './specific_attributes/query_attributes.gen'; -import { Threshold } from './specific_attributes/threshold_attributes.gen'; +import { + Threshold, + ThresholdAlertSuppression, +} from './specific_attributes/threshold_attributes.gen'; import { ThreatQuery, ThreatMapping, @@ -353,6 +356,7 @@ export const ThresholdRuleOptionalFields = z.object({ data_view_id: DataViewId.optional(), filters: RuleFilterArray.optional(), saved_id: SavedQueryId.optional(), + alert_suppression: ThresholdAlertSuppression.optional(), }); export type ThresholdRuleDefaultableFields = z.infer; diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.schema.yaml b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.schema.yaml index a916e7e4da224..67666aab5d95a 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.schema.yaml +++ b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.schema.yaml @@ -508,6 +508,8 @@ components: $ref: './common_attributes.schema.yaml#/components/schemas/RuleFilterArray' saved_id: $ref: './common_attributes.schema.yaml#/components/schemas/SavedQueryId' + alert_suppression: + $ref: './specific_attributes/threshold_attributes.schema.yaml#/components/schemas/ThresholdAlertSuppression' ThresholdRuleDefaultableFields: type: object diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/specific_attributes/query_attributes.gen.ts b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/specific_attributes/query_attributes.gen.ts index a21edeeb6831f..cca23efb4979a 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/specific_attributes/query_attributes.gen.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/specific_attributes/query_attributes.gen.ts @@ -12,6 +12,8 @@ import { z } from 'zod'; * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator. */ +import { AlertSuppressionDuration } from '../common_attributes.gen'; + /** * Describes how alerts will be generated for documents with missing suppress by fields: doNotSuppress - per each document a separate alert will be created @@ -28,12 +30,6 @@ export const AlertSuppressionMissingFieldsStrategyEnum = AlertSuppressionMissing export type AlertSuppressionGroupBy = z.infer; export const AlertSuppressionGroupBy = z.array(z.string()).min(1).max(3); -export type AlertSuppressionDuration = z.infer; -export const AlertSuppressionDuration = z.object({ - value: z.number().int().min(1), - unit: z.enum(['s', 'm', 'h']), -}); - export type AlertSuppression = z.infer; export const AlertSuppression = z.object({ group_by: AlertSuppressionGroupBy, diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/specific_attributes/query_attributes.schema.yaml b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/specific_attributes/query_attributes.schema.yaml index 36581c44e3d35..e47f5ed3b6ab3 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/specific_attributes/query_attributes.schema.yaml +++ b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/specific_attributes/query_attributes.schema.yaml @@ -23,29 +23,13 @@ components: minItems: 1 maxItems: 3 - AlertSuppressionDuration: - type: object - properties: - value: - type: integer - minimum: 1 - unit: - type: string - enum: - - s - - m - - h - required: - - value - - unit - AlertSuppression: type: object properties: group_by: $ref: '#/components/schemas/AlertSuppressionGroupBy' duration: - $ref: '#/components/schemas/AlertSuppressionDuration' + $ref: '../common_attributes.schema.yaml#/components/schemas/AlertSuppressionDuration' missing_fields_strategy: $ref: '#/components/schemas/AlertSuppressionMissingFieldsStrategy' required: @@ -57,7 +41,7 @@ components: groupBy: $ref: '#/components/schemas/AlertSuppressionGroupBy' duration: - $ref: '#/components/schemas/AlertSuppressionDuration' + $ref: '../common_attributes.schema.yaml#/components/schemas/AlertSuppressionDuration' missingFieldsStrategy: $ref: '#/components/schemas/AlertSuppressionMissingFieldsStrategy' required: diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/specific_attributes/threshold_attributes.gen.ts b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/specific_attributes/threshold_attributes.gen.ts index 46a48dae05abf..3f3fcbc7af736 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/specific_attributes/threshold_attributes.gen.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/specific_attributes/threshold_attributes.gen.ts @@ -12,6 +12,8 @@ import { z } from 'zod'; * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator. */ +import { AlertSuppressionDuration } from '../common_attributes.gen'; + export type ThresholdCardinality = z.infer; export const ThresholdCardinality = z.array( z.object({ @@ -58,3 +60,8 @@ export const ThresholdWithCardinality = z.object({ value: ThresholdValue, cardinality: ThresholdCardinality, }); + +export type ThresholdAlertSuppression = z.infer; +export const ThresholdAlertSuppression = z.object({ + duration: AlertSuppressionDuration, +}); diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/specific_attributes/threshold_attributes.schema.yaml b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/specific_attributes/threshold_attributes.schema.yaml index 4be7e45ba1012..206feaf7a01f5 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/specific_attributes/threshold_attributes.schema.yaml +++ b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/specific_attributes/threshold_attributes.schema.yaml @@ -78,3 +78,11 @@ components: - field - value - cardinality + + ThresholdAlertSuppression: + type: object + properties: + duration: + $ref: '../common_attributes.schema.yaml#/components/schemas/AlertSuppressionDuration' + required: + - duration \ No newline at end of file diff --git a/x-pack/plugins/security_solution/common/experimental_features.ts b/x-pack/plugins/security_solution/common/experimental_features.ts index 689ad118c9ca9..301ac9abf3636 100644 --- a/x-pack/plugins/security_solution/common/experimental_features.ts +++ b/x-pack/plugins/security_solution/common/experimental_features.ts @@ -115,6 +115,11 @@ export const allowedExperimentalValues = Object.freeze({ */ protectionUpdatesEnabled: true, + /** + * Enables alerts suppression for threshold rules + */ + alertSuppressionForThresholdRuleEnabled: false, + /** * Disables the timeline save tour. * This flag is used to disable the tour in cypress tests. diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_creation/helpers.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_creation/helpers.ts index 1c73a81b4d26e..983aba55c8165 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_creation/helpers.ts +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_creation/helpers.ts @@ -434,6 +434,9 @@ export const formatDefineStepData = (defineStepData: DefineStepRule): DefineStep ] : [], }, + ...(ruleFields.enableThresholdSuppression && { + alert_suppression: { duration: ruleFields.groupByDuration }, + }), }), } : isThreatMatchFields(ruleFields) @@ -505,6 +508,7 @@ export const formatDefineStepData = (defineStepData: DefineStepRule): DefineStep saved_id: ruleFields.queryBar.saved_id, }), }; + return { ...baseFields, ...typeFields, diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_creation/index.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_creation/index.tsx index ff6e1d201186b..d7f0b135979fe 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_creation/index.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_creation/index.tsx @@ -523,6 +523,8 @@ const CreateRulePageComponent: React.FC = () => { shouldLoadQueryDynamically={defineStepData.shouldLoadQueryDynamically} queryBarTitle={defineStepData.queryBar.title} queryBarSavedId={defineStepData.queryBar.saved_id} + thresholdFields={defineStepData.threshold.field} + enableThresholdSuppression={defineStepData.enableThresholdSuppression} /> { memoDefineStepReadOnly, setEqlOptionsSelected, threatIndicesConfig, + defineStepData.threshold.field, + defineStepData.enableThresholdSuppression, ] ); const memoDefineStepExtraAction = useMemo( diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_editing/index.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_editing/index.tsx index 60824648b18e0..94ce2442b236a 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_editing/index.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_editing/index.tsx @@ -261,6 +261,8 @@ const EditRulePageComponent: FC<{ rule: RuleResponse }> = ({ rule }) => { shouldLoadQueryDynamically={defineStepData.shouldLoadQueryDynamically} queryBarTitle={defineStepData.queryBar.title} queryBarSavedId={defineStepData.queryBar.saved_id} + thresholdFields={defineStepData.threshold.field} + enableThresholdSuppression={defineStepData.enableThresholdSuppression} /> )} diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_management/components/rule_details/rule_definition_section.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_management/components/rule_details/rule_definition_section.tsx index 92d0d9a5f60ed..eeacce21d3e8d 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_management/components/rule_details/rule_definition_section.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_management/components/rule_details/rule_definition_section.tsx @@ -55,6 +55,8 @@ import { TechnicalPreviewBadge } from '../../../../detections/components/rules/t import { BadgeList } from './badge_list'; import { DESCRIPTION_LIST_COLUMN_WIDTHS } from './constants'; import * as i18n from './translations'; +import { useIsExperimentalFeatureEnabled } from '../../../../common/hooks/use_experimental_features'; +import type { ExperimentalFeatures } from '../../../../../common/experimental_features'; interface SavedQueryNameProps { savedQueryName: string; @@ -427,7 +429,8 @@ const HistoryWindowSize = ({ historyWindowStart }: HistoryWindowSizeProps) => { const prepareDefinitionSectionListItems = ( rule: Partial, isInteractive: boolean, - savedQuery?: SavedQuery + savedQuery: SavedQuery | undefined, + { alertSuppressionForThresholdRuleEnabled }: Partial ): EuiDescriptionListProps['listItems'] => { const definitionSectionListItems: EuiDescriptionListProps['listItems'] = []; @@ -658,36 +661,42 @@ const prepareDefinitionSectionListItems = ( } if ('alert_suppression' in rule && rule.alert_suppression) { - definitionSectionListItems.push({ - title: ( - - - - ), - description: , - }); + if ('group_by' in rule.alert_suppression) { + definitionSectionListItems.push({ + title: ( + + + + ), + description: , + }); + } - definitionSectionListItems.push({ - title: ( - - - - ), - description: , - }); + if (rule.type !== 'threshold' || alertSuppressionForThresholdRuleEnabled) { + definitionSectionListItems.push({ + title: ( + + + + ), + description: , + }); + } - definitionSectionListItems.push({ - title: ( - - - - ), - description: ( - - ), - }); + if ('missing_fields_strategy' in rule.alert_suppression) { + definitionSectionListItems.push({ + title: ( + + + + ), + description: ( + + ), + }); + } } if ('new_terms_fields' in rule && rule.new_terms_fields && rule.new_terms_fields.length > 0) { @@ -733,10 +742,15 @@ export const RuleDefinitionSection = ({ ruleType: rule.type, }); + const alertSuppressionForThresholdRuleEnabled = useIsExperimentalFeatureEnabled( + 'alertSuppressionForThresholdRuleEnabled' + ); + const definitionSectionListItems = prepareDefinitionSectionListItems( rule, isInteractive, - savedQuery + savedQuery, + { alertSuppressionForThresholdRuleEnabled } ); return ( diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/__mocks__/mock.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/__mocks__/mock.ts index 86c0e7a1dd133..5d1d75296b08d 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/__mocks__/mock.ts +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/__mocks__/mock.ts @@ -241,6 +241,7 @@ export const mockDefineStepRule = (): DefineStepRule => ({ unit: 'm', value: 5, }, + enableThresholdSuppression: false, }); export const mockScheduleStepRule = (): ScheduleStepRule => ({ diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/duration_input/index.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/duration_input/index.tsx index 2133a10c38b8b..5163be74eed67 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/duration_input/index.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/duration_input/index.tsx @@ -85,6 +85,7 @@ const DurationInputComponent: React.FC = ({ { value: 'm', text: I18n.MINUTES }, { value: 'h', text: I18n.HOURS }, ], + ...props }: DurationInputProps) => { const { isInvalid, errorMessage } = getFieldValidityAndErrorMessage(durationValueField); const { value: durationValue, setValue: setDurationValue } = durationValueField; @@ -106,7 +107,7 @@ const DurationInputComponent: React.FC = ({ ); // EUI missing some props - const rest = { disabled: isDisabled }; + const rest = { disabled: isDisabled, ...props }; return ( diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/step_about_rule/index.test.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/step_about_rule/index.test.tsx index a9805bf71d3ce..ca8ef7ea56440 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/step_about_rule/index.test.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/step_about_rule/index.test.tsx @@ -69,6 +69,7 @@ export const stepDefineStepMLRule: DefineStepRule = { newTermsFields: ['host.ip'], historyWindowSize: '7d', shouldLoadQueryDynamically: false, + enableThresholdSuppression: false, }; describe('StepAboutRuleComponent', () => { diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.test.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.test.tsx index fe4adf6d0e79b..b38f10994eb13 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.test.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.test.tsx @@ -318,6 +318,8 @@ describe('StepDefineRule', () => { shouldLoadQueryDynamically={stepDefineDefaultValue.shouldLoadQueryDynamically} queryBarTitle="" queryBarSavedId="" + thresholdFields={[]} + enableThresholdSuppression={false} /> ); }; diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx index 3fb473204feea..e132f7e8a54c6 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx @@ -16,6 +16,7 @@ import { EuiButtonGroup, EuiText, EuiRadioGroup, + EuiToolTip, } from '@elastic/eui'; import type { FC } from 'react'; import React, { memo, useCallback, useState, useEffect, useMemo, useRef } from 'react'; @@ -64,7 +65,7 @@ import { isEqlRule, isNewTermsRule, isThreatMatchRule, - isThresholdRule, + isThresholdRule as getIsThresholdRule, isQueryRule, isEsqlRule, } from '../../../../../common/detection_engine/utils'; @@ -82,6 +83,7 @@ import { useLicense } from '../../../../common/hooks/use_license'; import { AlertSuppressionMissingFieldsStrategyEnum } from '../../../../../common/api/detection_engine/model/rule_schema'; import { DurationInput } from '../duration_input'; import { MINIMUM_LICENSE_FOR_SUPPRESSION } from '../../../../../common/detection_engine/constants'; +import { useIsExperimentalFeatureEnabled } from '../../../../common/hooks/use_experimental_features'; const CommonUseField = getUseField({ component: Field }); @@ -109,6 +111,8 @@ interface StepDefineRuleProps extends RuleStepProps { shouldLoadQueryDynamically: boolean; queryBarTitle: string | undefined; queryBarSavedId: string | null | undefined; + thresholdFields: string[] | undefined; + enableThresholdSuppression: boolean; } interface StepDefineRuleReadOnlyProps { @@ -166,6 +170,8 @@ const StepDefineRuleComponent: FC = ({ shouldLoadQueryDynamically, queryBarTitle, queryBarSavedId, + thresholdFields, + enableThresholdSuppression, }) => { const mlCapabilities = useMlCapabilities(); const [openTimelineSearch, setOpenTimelineSearch] = useState(false); @@ -175,6 +181,13 @@ const StepDefineRuleComponent: FC = ({ const esqlQueryRef = useRef(undefined); + const isAlertSuppressionForThresholdRuleFeatureEnabled = useIsExperimentalFeatureEnabled( + 'alertSuppressionForThresholdRuleEnabled' + ); + const isAlertSuppressionLicenseValid = license.isAtLeast(MINIMUM_LICENSE_FOR_SUPPRESSION); + + const isThresholdRule = getIsThresholdRule(ruleType); + const { getFields, reset, setFieldValue } = form; const setRuleTypeCallback = useSetFieldValueWithCallback({ @@ -350,6 +363,15 @@ const StepDefineRuleComponent: FC = ({ } }, [ruleType, previousRuleType, getFields]); + /** + * for threshold rule suppression only time interval suppression mode is available + */ + useEffect(() => { + if (isThresholdRule) { + form.setFieldValue('groupByRadioSelection', GroupByOptions.PerTimePeriod); + } + }, [isThresholdRule, form]); + // if saved query failed to load: // - reset shouldLoadFormDynamically to false, as non existent query cannot be used for loading and execution const handleSavedQueryError = useCallback(() => { @@ -413,31 +435,70 @@ const StepDefineRuleComponent: FC = ({ ] ); + /** + * Component that allows selection of suppression intervals disabled: + * - if suppression license is not valid(i.e. less than platinum) + * - or for not threshold rule - when groupBy fields not selected + */ + const isGroupByChildrenDisabled = + !isAlertSuppressionLicenseValid || isThresholdRule ? false : !groupByFields?.length; + + /** + * Per rule execution radio option is disabled + * - if suppression license is not valid(i.e. less than platinum) + * - always disabled for threshold rule + */ + const isPerRuleExecutionDisabled = !isAlertSuppressionLicenseValid || isThresholdRule; + + /** + * Per time period execution radio option is disabled + * - if suppression license is not valid(i.e. less than platinum) + * - disabled for threshold rule when enabled suppression is not checked + */ + const isPerTimePeriodDisabled = + !isAlertSuppressionLicenseValid || (isThresholdRule && !enableThresholdSuppression); + + /** + * Suppression duration is disabled when + * - if suppression license is not valid(i.e. less than platinum) + * - when suppression by rule execution is selected in radio button + * - whe threshold suppression is not enabled and no group by fields selected + * */ + const isDurationDisabled = + !isAlertSuppressionLicenseValid || (!enableThresholdSuppression && groupByFields?.length === 0); + const GroupByChildren = useCallback( ({ groupByRadioSelection, groupByDurationUnit, groupByDurationValue }) => ( + <> {i18n.ALERT_SUPPRESSION_PER_RULE_EXECUTION} + + ), + disabled: isPerRuleExecutionDisabled, }, { id: GroupByOptions.PerTimePeriod, + disabled: isPerTimePeriodDisabled, label: ( <> - {`Per time period`} + {i18n.ALERT_SUPPRESSION_PER_TIME_PERIOD} = ({ data-test-subj="groupByDurationOptions" /> ), - [license, groupByFields] + [ + isThresholdRule, + isDurationDisabled, + isPerTimePeriodDisabled, + isPerRuleExecutionDisabled, + isGroupByChildrenDisabled, + ] ); - const AlertsSuppressionMissingFields = useCallback( + const AlertSuppressionMissingFields = useCallback( ({ suppressionMissingFields }) => ( = ({ data-test-subj="suppressionMissingFieldsOptions" /> ), - [license, groupByFields] + [isAlertSuppressionLicenseValid, groupByFields] ); const dataViewIndexPatternToggleButtonOptions: EuiButtonGroupOptionProps[] = useMemo( @@ -743,6 +806,9 @@ const StepDefineRuleComponent: FC = ({ [isUpdateView, mlCapabilities] ); + const isAlertSuppressionEnabled = + isQueryRule(ruleType) || (isThresholdRule && isAlertSuppressionForThresholdRuleFeatureEnabled); + return ( <> @@ -827,65 +893,6 @@ const StepDefineRuleComponent: FC = ({ )} - - - - - - - {GroupByChildren} - - - - - {i18n.ALERT_SUPPRESSION_MISSING_FIELDS_FORM_ROW_LABEL} - - } - fullWidth - > - - {AlertsSuppressionMissingFields} - - - <> = ({ @@ -971,6 +978,89 @@ const StepDefineRuleComponent: FC = ({ /> + + + + + + + + + + + + + + + {GroupByChildren} + + + + + {i18n.ALERT_SUPPRESSION_MISSING_FIELDS_FORM_ROW_LABEL} + + } + fullWidth + > + + {AlertSuppressionMissingFields} + + + = { type: FIELD_TYPES.CHECKBOX, defaultValue: false, }, + enableThresholdSuppression: { + type: FIELD_TYPES.CHECKBOX, + defaultValue: false, + }, }; diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/translations.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/translations.tsx index bb289d0356bce..9206b5820f697 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/translations.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/translations.tsx @@ -5,7 +5,9 @@ * 2.0. */ +import React from 'react'; import { i18n } from '@kbn/i18n'; +import { FormattedMessage } from '@kbn/i18n-react'; export const CUSTOM_QUERY_REQUIRED = i18n.translate( 'xpack.securitySolution.detectionEngine.createRule.stepDefineRule.customQueryFieldRequiredError', @@ -192,3 +194,47 @@ export const GROUP_BY_FIELD_LICENSE_WARNING = i18n.translate( defaultMessage: 'Alert suppression is enabled with Platinum license or above', } ); + +export const ENABLE_THRESHOLD_SUPPRESSION_LICENSE_WARNING = i18n.translate( + 'xpack.securitySolution.detectionEngine.createRule.stepDefineRule.enableThresholdSuppression.licenseWarning', + { + defaultMessage: 'Alert suppression is enabled with Platinum license or above', + } +); + +export const ALERT_SUPPRESSION_PER_RULE_EXECUTION = i18n.translate( + 'xpack.securitySolution.detectionEngine.createRule.stepDefineRule.alertSuppressionOptions.perRuleExecutionLabel', + { + defaultMessage: 'Per rule execution', + } +); + +export const ALERT_SUPPRESSION_PER_TIME_PERIOD = i18n.translate( + 'xpack.securitySolution.detectionEngine.createRule.stepDefineRule.alertSuppressionOptions.perTimePeriodLabel', + { + defaultMessage: 'Per time period', + } +); + +export const THRESHOLD_SUPPRESSION_PER_RULE_EXECUTION_WARNING = i18n.translate( + 'xpack.securitySolution.detectionEngine.createRule.stepDefineRule.Su.perRuleExecutionWarning', + { + defaultMessage: 'Per rule execution option is not available for Threshold rule type', + } +); + +export const getEnableThresholdSuppressionLabel = (fields: string[] | undefined) => + fields?.length ? ( + {fields.join(', ')} }} + /> + ) : ( + i18n.translate( + 'xpack.securitySolution.detectionEngine.createRule.stepDefineRule.enableThresholdSuppressionLabel', + { + defaultMessage: 'Suppress alerts (Technical Preview)', + } + ) + ); diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.test.tsx b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.test.tsx index b401e2a1fe944..2ffedcdc55568 100644 --- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.test.tsx +++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.test.tsx @@ -125,6 +125,7 @@ describe('rule helpers', () => { newTermsFields: ['host.name'], historyWindowSize: '7d', suppressionMissingFields: expect.any(String), + enableThresholdSuppression: false, }; const aboutRuleStepData: AboutStepRule = { diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.tsx b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.tsx index 072649d52a6a6..2feba6edc5ea9 100644 --- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.tsx +++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.tsx @@ -156,7 +156,12 @@ export const getDefineStepsData = (rule: RuleResponse): DefineStepRule => ({ ? convertHistoryStartToSize(rule.history_window_start) : '7d', shouldLoadQueryDynamically: Boolean(rule.type === 'saved_query' && rule.saved_id), - groupByFields: ('alert_suppression' in rule && rule.alert_suppression?.group_by) || [], + groupByFields: + ('alert_suppression' in rule && + rule.alert_suppression && + 'group_by' in rule.alert_suppression && + rule.alert_suppression.group_by) || + [], groupByRadioSelection: 'alert_suppression' in rule && rule.alert_suppression?.duration ? GroupByOptions.PerTimePeriod @@ -166,8 +171,14 @@ export const getDefineStepsData = (rule: RuleResponse): DefineStepRule => ({ unit: 'm', }, suppressionMissingFields: - ('alert_suppression' in rule && rule.alert_suppression?.missing_fields_strategy) || + ('alert_suppression' in rule && + rule.alert_suppression && + 'missing_fields_strategy' in rule.alert_suppression && + rule.alert_suppression.missing_fields_strategy) || DEFAULT_SUPPRESSION_MISSING_FIELDS_STRATEGY, + enableThresholdSuppression: Boolean( + 'alert_suppression' in rule && rule.alert_suppression?.duration + ), }); export const convertHistoryStartToSize = (relativeTime: string) => { diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/types.ts b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/types.ts index 8b9fb30a4c1ba..6ad03d95ddccf 100644 --- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/types.ts +++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/types.ts @@ -158,6 +158,7 @@ export interface DefineStepRule { groupByRadioSelection: GroupByOptions; groupByDuration: Duration; suppressionMissingFields?: AlertSuppressionMissingFieldsStrategy; + enableThresholdSuppression: boolean; } export interface QueryDefineStep { @@ -174,7 +175,7 @@ export interface QueryDefineStep { export interface Duration { value: number; - unit: string; + unit: 's' | 'm' | 'h'; } export interface ScheduleStepRule { diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/utils.ts b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/utils.ts index 56076f54b4817..9e54856b7b28c 100644 --- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/utils.ts +++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/utils.ts @@ -73,6 +73,7 @@ export const stepDefineDefaultValue: DefineStepRule = { unit: 'm', }, suppressionMissingFields: DEFAULT_SUPPRESSION_MISSING_FIELDS_STRATEGY, + enableThresholdSuppression: false, }; export const stepAboutDefaultValue: AboutStepRule = { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/bulk_actions/rule_params_modifier.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/bulk_actions/rule_params_modifier.ts index 2994d2bf7f157..6bfdfcf394aac 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/bulk_actions/rule_params_modifier.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/bulk_actions/rule_params_modifier.ts @@ -203,7 +203,7 @@ export const ruleParamsModifier = ( if (!isActionSkipped) { isParamsUpdateSkipped = false; } - return { ...acc, ...ruleParams }; + return { ...acc, ...ruleParams } as RuleAlertType['params']; }, existingRuleParams); // increment version even if actions are empty, as attributes can be modified as well outside of ruleParamsModifier diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/normalization/rule_converters.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/normalization/rule_converters.test.ts index d03fe3587ce3b..29a67dc1da16b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/normalization/rule_converters.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/normalization/rule_converters.test.ts @@ -158,6 +158,23 @@ describe('rule_converters', () => { ); }); + test('should accept threshold alerts suppression params', () => { + const patchParams = { + alert_suppression: { + duration: { value: 4, unit: 'h' as const }, + }, + }; + const rule = getThresholdRuleParams(); + const patchedParams = patchTypeSpecificSnakeToCamel(patchParams, rule); + expect(patchedParams).toEqual( + expect.objectContaining({ + alertSuppression: { + duration: { value: 4, unit: 'h' }, + }, + }) + ); + }); + test('should accept machine learning params when existing rule type is machine learning', () => { const patchParams = { anomaly_threshold: 5, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/normalization/rule_converters.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/normalization/rule_converters.ts index b185a91b7ad37..2402e9fdcdf33 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/normalization/rule_converters.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/normalization/rule_converters.ts @@ -177,6 +177,9 @@ export const typeSpecificSnakeToCamel = ( filters: params.filters, savedId: params.saved_id, threshold: normalizeThresholdObject(params.threshold), + alertSuppression: params.alert_suppression?.duration + ? { duration: params.alert_suppression.duration } + : undefined, }; } case 'machine_learning': { @@ -310,6 +313,7 @@ const patchThresholdParams = ( threshold: params.threshold ? normalizeThresholdObject(params.threshold) : existingRule.threshold, + alertSuppression: params.alert_suppression ?? existingRule.alertSuppression, }; }; @@ -616,6 +620,9 @@ export const typeSpecificCamelToSnake = ( filters: params.filters, saved_id: params.savedId, threshold: params.threshold, + alert_suppression: params.alertSuppression?.duration + ? { duration: params.alertSuppression?.duration } + : undefined, }; } case 'machine_learning': { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_schema/model/rule_schemas.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_schema/model/rule_schemas.ts index cf124a3b775d3..c3075aa24af5b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_schema/model/rule_schemas.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_schema/model/rule_schemas.ts @@ -77,6 +77,7 @@ import { TiebreakerField, TimestampField, AlertSuppressionCamel, + ThresholdAlertSuppression, ThresholdNormalized, AnomalyThreshold, HistoryWindowStart, @@ -237,6 +238,7 @@ export const ThresholdSpecificRuleParams = z.object({ savedId: SavedQueryId.optional(), threshold: ThresholdNormalized, dataViewId: DataViewId.optional(), + alertSuppression: ThresholdAlertSuppression.optional(), }); export type ThresholdRuleParams = BaseRuleParams & ThresholdSpecificRuleParams; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_wrapper.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_wrapper.ts index 9620998722600..371f9601a8465 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_wrapper.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_wrapper.ts @@ -422,6 +422,7 @@ export const createSecurityRuleTypeWrapper: CreateSecurityRuleTypeWrapper = alertWithSuppression, refreshOnIndexingAlerts: refresh, publicBaseUrl, + experimentalFeatures, }, }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/alert_suppression/group_and_bulk_create.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/alert_suppression/group_and_bulk_create.ts index a336c717224d2..ced44553192e6 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/alert_suppression/group_and_bulk_create.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/alert_suppression/group_and_bulk_create.ts @@ -23,7 +23,7 @@ import { wrapSuppressedAlerts } from './wrap_suppressed_alerts'; import { buildGroupByFieldAggregation } from './build_group_by_field_aggregation'; import type { EventGroupingMultiBucketAggregationResult } from './build_group_by_field_aggregation'; import { singleSearchAfter } from '../../utils/single_search_after'; -import { bulkCreateWithSuppression } from './bulk_create_with_suppression'; +import { bulkCreateWithSuppression } from '../../utils/bulk_create_with_suppression'; import type { UnifiedQueryRuleParams } from '../../../rule_schema'; import type { BuildReasonMessage } from '../../utils/reason_formatters'; import { AlertSuppressionMissingFieldsStrategyEnum } from '../../../../../../common/api/detection_engine/model/rule_schema'; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/bulk_create_suppressed_threshold_alerts.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/bulk_create_suppressed_threshold_alerts.ts new file mode 100644 index 0000000000000..b2ec6fbc909d3 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/bulk_create_suppressed_threshold_alerts.ts @@ -0,0 +1,102 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { + AlertInstanceContext, + AlertInstanceState, + RuleExecutorServices, +} from '@kbn/alerting-plugin/server'; +import type { SuppressionFieldsLatest } from '@kbn/rule-registry-plugin/common/schemas'; +import type { SearchHit } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; + +import { buildReasonMessageForThresholdAlert } from '../utils/reason_formatters'; +import type { ThresholdBucket } from './types'; +import type { RunOpts } from '../types'; +import type { CompleteRule, ThresholdRuleParams } from '../../rule_schema'; +import type { BaseFieldsLatest } from '../../../../../common/api/detection_engine/model/alerts'; +import type { IRuleExecutionLogForExecutors } from '../../rule_monitoring'; +import { bulkCreateWithSuppression } from '../utils/bulk_create_with_suppression'; +import type { GenericBulkCreateResponse } from '../utils/bulk_create_with_suppression'; +import { wrapSuppressedThresholdALerts } from './wrap_suppressed_threshold_alerts'; +import { transformBulkCreatedItemsToHits } from './utils'; + +interface BulkCreateSuppressedThresholdAlertsParams { + buckets: ThresholdBucket[]; + completeRule: CompleteRule; + services: RuleExecutorServices; + inputIndexPattern: string[]; + startedAt: Date; + from: Date; + to: Date; + ruleExecutionLogger: IRuleExecutionLogForExecutors; + spaceId: string; + runOpts: RunOpts; +} + +/** + * wraps threshold alerts and creates them using bulkCreateWithSuppression utility + * returns {@link GenericBulkCreateResponse} + * and unsuppressed alerts, needed to create correct threshold history + */ +export const bulkCreateSuppressedThresholdAlerts = async ({ + buckets, + completeRule, + services, + inputIndexPattern, + startedAt, + from, + to, + ruleExecutionLogger, + spaceId, + runOpts, +}: BulkCreateSuppressedThresholdAlertsParams): Promise<{ + bulkCreateResult: GenericBulkCreateResponse; + unsuppressedAlerts: Array>; +}> => { + const ruleParams = completeRule.ruleParams; + const suppressionDuration = runOpts.completeRule.ruleParams.alertSuppression?.duration; + if (!suppressionDuration) { + throw Error('Suppression duration can not be empty'); + } + + const suppressionWindow = `now-${suppressionDuration.value}${suppressionDuration.unit}`; + + const wrappedAlerts = wrapSuppressedThresholdALerts({ + buckets, + spaceId, + completeRule, + mergeStrategy: runOpts.mergeStrategy, + indicesToQuery: runOpts.inputIndex, + buildReasonMessage: buildReasonMessageForThresholdAlert, + alertTimestampOverride: runOpts.alertTimestampOverride, + ruleExecutionLogger, + publicBaseUrl: runOpts.publicBaseUrl, + inputIndex: inputIndexPattern.join(','), + startedAt, + from, + to, + suppressionWindow, + threshold: ruleParams.threshold, + }); + + const bulkCreateResult = await bulkCreateWithSuppression({ + alertWithSuppression: runOpts.alertWithSuppression, + ruleExecutionLogger: runOpts.ruleExecutionLogger, + wrappedDocs: wrappedAlerts, + services, + suppressionWindow, + alertTimestampOverride: runOpts.alertTimestampOverride, + }); + + return { + bulkCreateResult, + // if there errors we going to use created items only + unsuppressedAlerts: bulkCreateResult.errors.length + ? transformBulkCreatedItemsToHits(bulkCreateResult.createdItems) + : wrappedAlerts, + }; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/bulk_create_threshold_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/bulk_create_threshold_signals.ts index 318ac5bf562b0..0034be0bf74b7 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/bulk_create_threshold_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/bulk_create_threshold_signals.ts @@ -38,6 +38,46 @@ interface BulkCreateThresholdSignalsParams { ruleExecutionLogger: IRuleExecutionLogForExecutors; } +export const transformBucketIntoHit = ( + bucket: ThresholdBucket, + inputIndex: string, + startedAt: Date, + from: Date, + threshold: ThresholdNormalized, + ruleId: string +) => { + // In case of absent threshold fields, `bucket.key` will be an empty string. Note that `Object.values('')` is `[]`, + // so the below logic works in either case (whether `terms` or `composite`). + return { + _index: inputIndex, + _id: calculateThresholdSignalUuid( + ruleId, + startedAt, + threshold.field, + Object.values(bucket.key).sort().join(',') + ), + _source: { + [TIMESTAMP]: bucket.max_timestamp.value_as_string, + ...bucket.key, + threshold_result: { + cardinality: threshold.cardinality?.length + ? [ + { + field: threshold.cardinality[0].field, + value: bucket.cardinality_count?.value, + }, + ] + : undefined, + count: bucket.doc_count, + from: bucket.min_timestamp.value_as_string + ? new Date(bucket.min_timestamp.value_as_string) + : from, + terms: Object.entries(bucket.key).map(([key, val]) => ({ field: key, value: val })), + }, + }, + }; +}; + export const getTransformedHits = ( buckets: ThresholdBucket[], inputIndex: string, @@ -47,36 +87,7 @@ export const getTransformedHits = ( ruleId: string ) => buckets.map((bucket, i) => { - // In case of absent threshold fields, `bucket.key` will be an empty string. Note that `Object.values('')` is `[]`, - // so the below logic works in either case (whether `terms` or `composite`). - return { - _index: inputIndex, - _id: calculateThresholdSignalUuid( - ruleId, - startedAt, - threshold.field, - Object.values(bucket.key).sort().join(',') - ), - _source: { - [TIMESTAMP]: bucket.max_timestamp.value_as_string, - ...bucket.key, - threshold_result: { - cardinality: threshold.cardinality?.length - ? [ - { - field: threshold.cardinality[0].field, - value: bucket.cardinality_count?.value, - }, - ] - : undefined, - count: bucket.doc_count, - from: bucket.min_timestamp.value_as_string - ? new Date(bucket.min_timestamp.value_as_string) - : from, - terms: Object.entries(bucket.key).map(([key, val]) => ({ field: key, value: val })), - }, - }, - }; + return transformBucketIntoHit(bucket, inputIndex, startedAt, from, threshold, ruleId); }); export const bulkCreateThresholdSignals = async ( diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/create_threshold_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/create_threshold_alert_type.ts index 4297a24fa8bd3..40eec8e10a808 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/create_threshold_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/create_threshold_alert_type.ts @@ -19,7 +19,7 @@ import { validateIndexPatterns } from '../utils'; export const createThresholdAlertType = ( createOptions: CreateRuleOptions ): SecurityAlertType => { - const { version } = createOptions; + const { version, licensing } = createOptions; return { id: THRESHOLD_RULE_TYPE_ID, name: 'Threshold Rule', @@ -76,6 +76,7 @@ export const createThresholdAlertType = ( services, startedAt, state, + spaceId, } = execOptions; const result = await thresholdExecutor({ completeRule, @@ -96,6 +97,9 @@ export const createThresholdAlertType = ( exceptionFilter, unprocessedExceptions, inputIndexFields, + spaceId, + runOpts: execOptions.runOpts, + licensing, }); return result; }, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/threshold.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/threshold.test.ts index 08f86f4f4b7ba..4cd366722a279 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/threshold.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/threshold.test.ts @@ -9,6 +9,7 @@ import dateMath from '@kbn/datemath'; import { elasticsearchClientMock } from '@kbn/core-elasticsearch-client-server-mocks'; import { getExceptionListItemSchemaMock } from '@kbn/lists-plugin/common/schemas/response/exception_list_item_schema.mock'; import type { RuleExecutorServicesMock } from '@kbn/alerting-plugin/server/mocks'; +import { licensingMock } from '@kbn/licensing-plugin/server/mocks'; import { alertsMock } from '@kbn/alerting-plugin/server/mocks'; import { thresholdExecutor } from './threshold'; import { getThresholdRuleParams, getCompleteRuleMock } from '../../rule_schema/mocks'; @@ -18,6 +19,7 @@ import type { ThresholdRuleParams } from '../../rule_schema'; import { createRuleDataClientMock } from '@kbn/rule-registry-plugin/server/rule_data_client/rule_data_client.mock'; import { TIMESTAMP } from '@kbn/rule-data-utils'; import { ruleExecutionLogMock } from '../../rule_monitoring/mocks'; +import type { RunOpts } from '../types'; describe('threshold_executor', () => { let alertServices: RuleExecutorServicesMock; @@ -31,6 +33,7 @@ describe('threshold_executor', () => { to: dateMath.parse(params.to)!, maxSignals: params.maxSignals, }; + const licensing = licensingMock.createSetup(); beforeEach(() => { alertServices = alertsMock.createRuleExecutorServices(); @@ -104,6 +107,9 @@ describe('threshold_executor', () => { exceptionFilter: undefined, unprocessedExceptions: [], inputIndexFields: [], + spaceId: 'default', + runOpts: {} as RunOpts, + licensing, }); expect(response.state).toEqual({ initialized: true, @@ -166,6 +172,9 @@ describe('threshold_executor', () => { exceptionFilter: undefined, unprocessedExceptions: [getExceptionListItemSchemaMock()], inputIndexFields: [], + spaceId: 'default', + runOpts: {} as RunOpts, + licensing, }); expect(result.warningMessages).toEqual([ `The following exceptions won't be applied to rule execution: ${ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/threshold.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/threshold.ts index 416d5b5a53f4f..41ede6563524c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/threshold.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/threshold.ts @@ -6,9 +6,11 @@ */ import { isEmpty } from 'lodash'; -import type { SearchHit } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import { firstValueFrom } from 'rxjs'; + import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import type { ExceptionListItemSchema } from '@kbn/securitysolution-io-ts-list-types'; +import type { LicensingPluginSetup } from '@kbn/licensing-plugin/server'; import type { AlertInstanceContext, @@ -23,14 +25,18 @@ import { bulkCreateThresholdSignals } from './bulk_create_threshold_signals'; import { findThresholdSignals } from './find_threshold_signals'; import { getThresholdBucketFilters } from './get_threshold_bucket_filters'; import { getThresholdSignalHistory } from './get_threshold_signal_history'; +import { bulkCreateSuppressedThresholdAlerts } from './bulk_create_suppressed_threshold_alerts'; +import type { GenericBulkCreateResponse } from '../utils/bulk_create_with_suppression'; +import type { BaseFieldsLatest } from '../../../../../common/api/detection_engine/model/alerts'; import type { BulkCreate, RuleRangeTuple, SearchAfterAndBulkCreateReturnType, WrapHits, + RunOpts, } from '../types'; -import type { ThresholdAlertState } from './types'; +import type { ThresholdAlertState, ThresholdSignalHistory } from './types'; import { addToSearchAfterReturn, createSearchAfterReturnType, @@ -39,7 +45,7 @@ import { import { withSecuritySpan } from '../../../../utils/with_security_span'; import { buildThresholdSignalHistory } from './build_signal_history'; import type { IRuleExecutionLogForExecutors } from '../../rule_monitoring'; -import { getSignalHistory } from './utils'; +import { getSignalHistory, transformBulkCreatedItemsToHits } from './utils'; export const thresholdExecutor = async ({ inputIndex, @@ -60,6 +66,9 @@ export const thresholdExecutor = async ({ exceptionFilter, unprocessedExceptions, inputIndexFields, + spaceId, + runOpts, + licensing, }: { inputIndex: string[]; runtimeMappings: estypes.MappingRuntimeFields | undefined; @@ -79,6 +88,9 @@ export const thresholdExecutor = async ({ exceptionFilter: Filter | undefined; unprocessedExceptions: ExceptionListItemSchema[]; inputIndexFields: DataViewFieldBase[]; + spaceId: string; + runOpts: RunOpts; + licensing: LicensingPluginSetup; }): Promise => { const result = createSearchAfterReturnType(); const ruleParams = completeRule.ruleParams; @@ -89,6 +101,9 @@ export const thresholdExecutor = async ({ result.warningMessages.push(exceptionsWarning); } + const license = await firstValueFrom(licensing.license$); + const hasPlatinumLicense = license.hasAtLeast('platinum'); + // Get state or build initial state (on upgrade) const { signalHistory, searchErrors: previousSearchErrors } = state.initialized ? { signalHistory: state.signalHistory, searchErrors: [] } @@ -136,20 +151,53 @@ export const thresholdExecutor = async ({ aggregatableTimestampField, }); - const createResult = await bulkCreateThresholdSignals({ - buckets, - completeRule, - filter: esFilter, - services, - inputIndexPattern: inputIndex, - signalsIndex: ruleParams.outputIndex, - startedAt, - from: tuple.from.toDate(), - signalHistory: validSignalHistory, - bulkCreate, - wrapHits, - ruleExecutionLogger, - }); + const alertSuppression = completeRule.ruleParams.alertSuppression; + + let createResult: GenericBulkCreateResponse; + let newSignalHistory: ThresholdSignalHistory; + + if ( + alertSuppression?.duration && + runOpts?.experimentalFeatures?.alertSuppressionForThresholdRuleEnabled && + hasPlatinumLicense + ) { + const suppressedResults = await bulkCreateSuppressedThresholdAlerts({ + buckets, + completeRule, + services, + inputIndexPattern: inputIndex, + startedAt, + from: tuple.from.toDate(), + to: tuple.to.toDate(), + ruleExecutionLogger, + spaceId, + runOpts, + }); + createResult = suppressedResults.bulkCreateResult; + + newSignalHistory = buildThresholdSignalHistory({ + alerts: suppressedResults.unsuppressedAlerts, + }); + } else { + createResult = await bulkCreateThresholdSignals({ + buckets, + completeRule, + filter: esFilter, + services, + inputIndexPattern: inputIndex, + signalsIndex: ruleParams.outputIndex, + startedAt, + from: tuple.from.toDate(), + signalHistory: validSignalHistory, + bulkCreate, + wrapHits, + ruleExecutionLogger, + }); + + newSignalHistory = buildThresholdSignalHistory({ + alerts: transformBulkCreatedItemsToHits(createResult.createdItems), + }); + } addToSearchAfterReturn({ current: result, @@ -161,21 +209,6 @@ export const thresholdExecutor = async ({ result.warningMessages.push(...warnings); result.searchAfterTimes = searchDurations; - const createdAlerts = createResult.createdItems.map((alert) => { - const { _id, _index, ...source } = alert; - return { - _id, - _index, - _source: { - ...source, - }, - } as SearchHit; - }); - - const newSignalHistory = buildThresholdSignalHistory({ - alerts: createdAlerts, - }); - return { ...result, state: { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/utils.test.ts index 0323f3263a92a..2a54c3b0e156f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/utils.test.ts @@ -8,7 +8,12 @@ import dateMath from '@kbn/datemath'; import { getThresholdRuleParams } from '../../rule_schema/mocks'; -import { calculateThresholdSignalUuid, getSignalHistory, getThresholdTermsHash } from './utils'; +import { + calculateThresholdSignalUuid, + getSignalHistory, + getThresholdTermsHash, + transformBulkCreatedItemsToHits, +} from './utils'; describe('threshold utils', () => { describe('calcualteThresholdSignalUuid', () => { @@ -66,4 +71,30 @@ describe('threshold utils', () => { expect(validSignalHistory[hashTwo]).toBe(state.signalHistory[hashTwo]); }); }); + + describe('transformBulkCreatedItemsToHits', () => { + it('should correctly transform bulk created items to hit', () => { + expect( + transformBulkCreatedItemsToHits([ + { + _id: 'test-1', + _index: 'logs-*', + rule: { + name: 'test', + }, + }, + ] as unknown as Parameters[number]) + ).toEqual([ + { + _id: 'test-1', + _index: 'logs-*', + _source: { + rule: { + name: 'test', + }, + }, + }, + ]); + }); + }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/utils.ts index 52685190950f1..c73becbdd8f57 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/utils.ts @@ -8,6 +8,9 @@ import type { estypes } from '@elastic/elasticsearch'; import { createHash } from 'crypto'; import { v5 as uuidv5 } from 'uuid'; + +import type { AlertWithCommonFieldsLatest } from '@kbn/rule-registry-plugin/common/schemas'; +import type { BaseFieldsLatest } from '../../../../../common/api/detection_engine/model/alerts'; import type { ThresholdNormalized, ThresholdWithCardinality, @@ -94,3 +97,22 @@ export const searchResultHasAggs = < >( obj: SignalSearchResponse> ): obj is T => obj?.aggregations != null; + +/** + * transforms documents returned from bulk creation into Hit formatting + * basically, moving all fields(apart from _id & _index) from root node to _source property + * { _id: 1, _index: "logs", field1, field2 } => { _id: 1, _index: "logs", _source: { field1, field2 } } + */ +export const transformBulkCreatedItemsToHits = ( + items: Array & { _id: string; _index: string }> +) => + items.map((alert) => { + const { _id, _index, ...source } = alert; + return { + _id, + _index, + _source: { + ...source, + }, + }; + }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/wrap_suppressed_threshold_alerts.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/wrap_suppressed_threshold_alerts.ts new file mode 100644 index 0000000000000..a3df4faa14c56 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/wrap_suppressed_threshold_alerts.ts @@ -0,0 +1,127 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import objectHash from 'object-hash'; + +import type { SuppressionFieldsLatest } from '@kbn/rule-registry-plugin/common/schemas'; +import { + ALERT_SUPPRESSION_DOCS_COUNT, + ALERT_INSTANCE_ID, + ALERT_SUPPRESSION_TERMS, + ALERT_SUPPRESSION_START, + ALERT_SUPPRESSION_END, + TIMESTAMP, +} from '@kbn/rule-data-utils'; + +import type { + BaseFieldsLatest, + WrappedFieldsLatest, +} from '../../../../../common/api/detection_engine/model/alerts'; +import type { ConfigType } from '../../../../config'; +import type { CompleteRule, ThresholdRuleParams } from '../../rule_schema'; +import type { IRuleExecutionLogForExecutors } from '../../rule_monitoring'; +import { buildBulkBody } from '../factories/utils/build_bulk_body'; + +import type { ThresholdBucket } from './types'; +import type { BuildReasonMessage } from '../utils/reason_formatters'; +import { transformBucketIntoHit } from './bulk_create_threshold_signals'; +import type { ThresholdNormalized } from '../../../../../common/api/detection_engine/model/rule_schema'; + +/** + * wraps suppressed threshold alerts + * first, transforms aggregation threshold buckets to hits + * creates instanceId hash, which is used to search suppressed on time interval alerts + * populates alert's suppression fields + */ +export const wrapSuppressedThresholdALerts = ({ + buckets, + spaceId, + completeRule, + mergeStrategy, + indicesToQuery, + buildReasonMessage, + alertTimestampOverride, + ruleExecutionLogger, + publicBaseUrl, + inputIndex, + startedAt, + from, + to, + threshold, +}: { + buckets: ThresholdBucket[]; + spaceId: string; + completeRule: CompleteRule; + mergeStrategy: ConfigType['alertMergeStrategy']; + indicesToQuery: string[]; + buildReasonMessage: BuildReasonMessage; + alertTimestampOverride: Date | undefined; + ruleExecutionLogger: IRuleExecutionLogForExecutors; + publicBaseUrl: string | undefined; + inputIndex: string; + startedAt: Date; + from: Date; + to: Date; + suppressionWindow: string; + threshold: ThresholdNormalized; +}): Array> => { + return buckets.map((bucket) => { + const hit = transformBucketIntoHit( + bucket, + inputIndex, + startedAt, + from, + threshold, + completeRule.ruleParams.ruleId + ); + + const suppressedValues = Object.entries(bucket.key) + .map(([key, value]) => value) + .sort((a, b) => a.localeCompare(b)); + + const id = objectHash([ + hit._index, + hit._id, + `${spaceId}:${completeRule.alertId}`, + suppressedValues, + ]); + + const instanceId = objectHash([suppressedValues, completeRule.alertId, spaceId]); + + const baseAlert: BaseFieldsLatest = buildBulkBody( + spaceId, + completeRule, + hit, + mergeStrategy, + [], + true, + buildReasonMessage, + indicesToQuery, + alertTimestampOverride, + ruleExecutionLogger, + id, + publicBaseUrl + ); + // suppression start/end equals to alert timestamp, since we suppress alerts for threshold rule type, not documents as for query rule type + const suppressionTime = new Date(baseAlert[TIMESTAMP]); + return { + _id: id, + _index: '', + _source: { + ...baseAlert, + [ALERT_SUPPRESSION_TERMS]: Object.entries(bucket.key).map(([field, value]) => ({ + field, + value, + })), + [ALERT_SUPPRESSION_START]: suppressionTime, + [ALERT_SUPPRESSION_END]: suppressionTime, + [ALERT_SUPPRESSION_DOCS_COUNT]: 0, + [ALERT_INSTANCE_ID]: instanceId, + }, + }; + }); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts index c9159c1739c37..8e91f48038845 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts @@ -100,6 +100,7 @@ export interface RunOpts { refreshOnIndexingAlerts: RefreshTypes; publicBaseUrl: string | undefined; inputIndexFields: DataViewFieldBase[]; + experimentalFeatures?: ExperimentalFeatures; } export type SecurityAlertType< diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/alert_suppression/bulk_create_with_suppression.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_with_suppression.ts similarity index 91% rename from x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/alert_suppression/bulk_create_with_suppression.ts rename to x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_with_suppression.ts index 231387f9c004a..7b03211b574b0 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/alert_suppression/bulk_create_with_suppression.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_with_suppression.ts @@ -13,14 +13,14 @@ import type { AlertWithCommonFieldsLatest, SuppressionFieldsLatest, } from '@kbn/rule-registry-plugin/common/schemas'; -import type { IRuleExecutionLogForExecutors } from '../../../rule_monitoring'; -import { makeFloatString } from '../../utils/utils'; +import type { IRuleExecutionLogForExecutors } from '../../rule_monitoring'; +import { makeFloatString } from './utils'; import type { BaseFieldsLatest, WrappedFieldsLatest, -} from '../../../../../../common/api/detection_engine/model/alerts'; -import type { RuleServices } from '../../types'; -import { createEnrichEventsFunction } from '../../utils/enrichments'; +} from '../../../../../common/api/detection_engine/model/alerts'; +import type { RuleServices } from '../types'; +import { createEnrichEventsFunction } from './enrichments'; export interface GenericBulkCreateResponse { success: boolean; diff --git a/x-pack/test/detection_engine_api_integration/utils/get_threshold_rule_for_signal_testing.ts b/x-pack/test/detection_engine_api_integration/utils/get_threshold_rule_for_signal_testing.ts index 7dcfcc469f1be..f2cb502e12a2e 100644 --- a/x-pack/test/detection_engine_api_integration/utils/get_threshold_rule_for_signal_testing.ts +++ b/x-pack/test/detection_engine_api_integration/utils/get_threshold_rule_for_signal_testing.ts @@ -28,4 +28,5 @@ export const getThresholdRuleForSignalTesting = ( field: 'process.name', value: 21, }, + alert_suppression: undefined, }); diff --git a/x-pack/test/security_solution_api_integration/config/ess/config.base.ts b/x-pack/test/security_solution_api_integration/config/ess/config.base.ts index b4fbdba6de4c4..b7f08d5180bbe 100644 --- a/x-pack/test/security_solution_api_integration/config/ess/config.base.ts +++ b/x-pack/test/security_solution_api_integration/config/ess/config.base.ts @@ -81,6 +81,7 @@ export function createTestConfig(options: CreateTestConfigOptions, testFiles?: s 'previewTelemetryUrlEnabled', 'riskScoringPersistence', 'riskScoringRoutesEnabled', + 'alertSuppressionForThresholdRuleEnabled', ])}`, '--xpack.task_manager.poll_interval=1000', `--xpack.actions.preconfigured=${JSON.stringify({ diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_execution_logic/configs/serverless.config.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_execution_logic/configs/serverless.config.ts index 7bcb663699d68..c01c3a74e61cf 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_execution_logic/configs/serverless.config.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_execution_logic/configs/serverless.config.ts @@ -16,5 +16,8 @@ export default createTestConfig({ 'testing_ignored.constant', '/testing_regex*/', ])}`, // See tests within the file "ignore_fields.ts" which use these values in "alertIgnoreFields" + `--xpack.securitySolution.enableExperimental=${JSON.stringify([ + 'alertSuppressionForThresholdRuleEnabled', + ])}`, ], }); diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_execution_logic/execution_logic/index.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_execution_logic/execution_logic/index.ts index 36a249304c7e6..f164857d9bd8f 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_execution_logic/execution_logic/index.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_execution_logic/execution_logic/index.ts @@ -16,6 +16,7 @@ export default ({ loadTestFile }: FtrProviderContext): void => { loadTestFile(require.resolve('./saved_query')); loadTestFile(require.resolve('./threat_match')); loadTestFile(require.resolve('./threshold')); + loadTestFile(require.resolve('./threshold_alert_suppression')); loadTestFile(require.resolve('./non_ecs_fields')); loadTestFile(require.resolve('./query')); }); diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_execution_logic/execution_logic/threshold.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_execution_logic/execution_logic/threshold.ts index 81203c3d48a28..5edee29c02dc6 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_execution_logic/execution_logic/threshold.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_execution_logic/execution_logic/threshold.ts @@ -5,7 +5,8 @@ * 2.0. */ -import expect from '@kbn/expect'; +import expect from 'expect'; + import { ALERT_REASON, ALERT_RULE_UUID, @@ -15,6 +16,7 @@ import { import { ThresholdRuleCreateProps } from '@kbn/security-solution-plugin/common/api/detection_engine'; import { Ancestor } from '@kbn/security-solution-plugin/server/lib/detection_engine/rule_types/types'; + import { ALERT_ANCESTORS, ALERT_DEPTH, @@ -63,13 +65,13 @@ export default ({ getService }: FtrProviderContext) => { }; const createdRule = await createRule(supertest, log, rule); const alerts = await getOpenAlerts(supertest, log, es, createdRule); - expect(alerts.hits.hits.length).eql(1); + expect(alerts.hits.hits.length).toEqual(1); const fullAlert = alerts.hits.hits[0]._source; if (!fullAlert) { - return expect(fullAlert).to.be.ok(); + return expect(fullAlert).toBeTruthy(); } const eventIds = (fullAlert?.[ALERT_ANCESTORS] as Ancestor[]).map((event) => event.id); - expect(fullAlert).eql({ + expect(fullAlert).toEqual({ ...fullAlert, 'host.id': '8cc95778cce5407c809480e8e32ad76b', [EVENT_KIND]: 'signal', @@ -109,7 +111,7 @@ export default ({ getService }: FtrProviderContext) => { max_signals: 5, }; const { logs } = await previewRule({ supertest, rule }); - expect(logs[0].warnings).contain(getMaxAlertsWarning()); + expect(logs[0].warnings).toContain(getMaxAlertsWarning()); }); it("doesn't generate max alerts warning when circuit breaker is met but not exceeded", async () => { @@ -122,7 +124,7 @@ export default ({ getService }: FtrProviderContext) => { max_signals: 7, }; const { logs } = await previewRule({ supertest, rule }); - expect(logs[0].warnings).not.contain(getMaxAlertsWarning()); + expect(logs[0].warnings).not.toContain(getMaxAlertsWarning()); }); it('generates 2 alerts from Threshold rules when threshold is met', async () => { @@ -135,7 +137,7 @@ export default ({ getService }: FtrProviderContext) => { }; const { previewId } = await previewRule({ supertest, rule }); const previewAlerts = await getPreviewAlerts({ es, previewId }); - expect(previewAlerts.length).eql(2); + expect(previewAlerts.length).toEqual(2); }); it('applies the provided query before bucketing ', async () => { @@ -149,7 +151,7 @@ export default ({ getService }: FtrProviderContext) => { }; const { previewId } = await previewRule({ supertest, rule }); const previewAlerts = await getPreviewAlerts({ es, previewId }); - expect(previewAlerts.length).eql(1); + expect(previewAlerts.length).toEqual(1); }); it('generates no alerts from Threshold rules when threshold is met and cardinality is not met', async () => { @@ -168,7 +170,7 @@ export default ({ getService }: FtrProviderContext) => { }; const { previewId } = await previewRule({ supertest, rule }); const previewAlerts = await getPreviewAlerts({ es, previewId }); - expect(previewAlerts.length).eql(0); + expect(previewAlerts.length).toEqual(0); }); it('generates no alerts from Threshold rules when cardinality is met and threshold is not met', async () => { @@ -187,7 +189,7 @@ export default ({ getService }: FtrProviderContext) => { }; const { previewId } = await previewRule({ supertest, rule }); const previewAlerts = await getPreviewAlerts({ es, previewId }); - expect(previewAlerts.length).eql(0); + expect(previewAlerts.length).toEqual(0); }); it('generates alerts from Threshold rules when threshold and cardinality are both met', async () => { @@ -206,13 +208,13 @@ export default ({ getService }: FtrProviderContext) => { }; const { previewId } = await previewRule({ supertest, rule }); const previewAlerts = await getPreviewAlerts({ es, previewId }); - expect(previewAlerts.length).eql(1); + expect(previewAlerts.length).toEqual(1); const fullAlert = previewAlerts[0]._source; if (!fullAlert) { - return expect(fullAlert).to.be.ok(); + return expect(fullAlert).toBeTruthy(); } const eventIds = (fullAlert?.[ALERT_ANCESTORS] as Ancestor[]).map((event) => event.id); - expect(fullAlert).eql({ + expect(fullAlert).toEqual({ ...fullAlert, 'host.id': '8cc95778cce5407c809480e8e32ad76b', [EVENT_KIND]: 'signal', @@ -258,7 +260,7 @@ export default ({ getService }: FtrProviderContext) => { }; const { previewId } = await previewRule({ supertest, rule }); const previewAlerts = await getPreviewAlerts({ es, previewId }); - expect(previewAlerts.length).eql(0); + expect(previewAlerts.length).toEqual(0); }); it('generates alerts from Threshold rules when bucketing by multiple fields', async () => { @@ -271,13 +273,13 @@ export default ({ getService }: FtrProviderContext) => { }; const { previewId } = await previewRule({ supertest, rule }); const previewAlerts = await getPreviewAlerts({ es, previewId }); - expect(previewAlerts.length).eql(1); + expect(previewAlerts.length).toEqual(1); const fullAlert = previewAlerts[0]._source; if (!fullAlert) { - return expect(fullAlert).to.be.ok(); + return expect(fullAlert).toBeTruthy(); } const eventIds = (fullAlert[ALERT_ANCESTORS] as Ancestor[]).map((event) => event.id); - expect(fullAlert).eql({ + expect(fullAlert).toEqual({ ...fullAlert, 'event.module': 'system', 'host.id': '2ab45fc1c41e4c84bbd02202a7e5761f', @@ -329,7 +331,7 @@ export default ({ getService }: FtrProviderContext) => { }; const createdRule = await createRule(supertest, log, rule); const alerts = await getOpenAlerts(supertest, log, es, createdRule); - expect(alerts.hits.hits.length).eql(1); + expect(alerts.hits.hits.length).toEqual(1); }); describe('Timestamp override and fallback', async () => { @@ -356,19 +358,19 @@ export default ({ getService }: FtrProviderContext) => { }; const { previewId } = await previewRule({ supertest, rule }); const previewAlerts = await getPreviewAlerts({ es, previewId }); - expect(previewAlerts.length).eql(4); + expect(previewAlerts.length).toEqual(4); for (const hit of previewAlerts) { const originalTime = hit._source?.[ALERT_ORIGINAL_TIME]; const hostName = hit._source?.['host.name']; if (hostName === 'host-1') { - expect(originalTime).eql('2020-12-16T15:15:18.570Z'); + expect(originalTime).toEqual('2020-12-16T15:15:18.570Z'); } else if (hostName === 'host-2') { - expect(originalTime).eql('2020-12-16T15:16:18.570Z'); + expect(originalTime).toEqual('2020-12-16T15:16:18.570Z'); } else if (hostName === 'host-3') { - expect(originalTime).eql('2020-12-16T16:15:18.570Z'); + expect(originalTime).toEqual('2020-12-16T16:15:18.570Z'); } else { - expect(originalTime).eql('2020-12-16T16:16:18.570Z'); + expect(originalTime).toEqual('2020-12-16T16:16:18.570Z'); } } }); @@ -384,19 +386,19 @@ export default ({ getService }: FtrProviderContext) => { }; const { previewId } = await previewRule({ supertest, rule }); const previewAlerts = await getPreviewAlerts({ es, previewId }); - expect(previewAlerts.length).eql(4); + expect(previewAlerts.length).toEqual(4); for (const hit of previewAlerts) { const originalTime = hit._source?.[ALERT_ORIGINAL_TIME]; const hostName = hit._source?.['host.name']; if (hostName === 'host-1') { - expect(originalTime).eql('2020-12-16T15:15:18.570Z'); + expect(originalTime).toEqual('2020-12-16T15:15:18.570Z'); } else if (hostName === 'host-2') { - expect(originalTime).eql('2020-12-16T15:16:18.570Z'); + expect(originalTime).toEqual('2020-12-16T15:16:18.570Z'); } else if (hostName === 'host-3') { - expect(originalTime).eql('2020-12-16T16:15:18.570Z'); + expect(originalTime).toEqual('2020-12-16T16:15:18.570Z'); } else { - expect(originalTime).eql('2020-12-16T16:16:18.570Z'); + expect(originalTime).toEqual('2020-12-16T16:16:18.570Z'); } } }); @@ -422,10 +424,10 @@ export default ({ getService }: FtrProviderContext) => { const { previewId } = await previewRule({ supertest, rule }); const previewAlerts = await getPreviewAlerts({ es, previewId, sort: ['host.name'] }); - expect(previewAlerts[0]?._source?.host?.risk?.calculated_level).to.eql('Low'); - expect(previewAlerts[0]?._source?.host?.risk?.calculated_score_norm).to.eql(20); - expect(previewAlerts[1]?._source?.host?.risk?.calculated_level).to.eql('Critical'); - expect(previewAlerts[1]?._source?.host?.risk?.calculated_score_norm).to.eql(96); + expect(previewAlerts[0]?._source?.host?.risk?.calculated_level).toEqual('Low'); + expect(previewAlerts[0]?._source?.host?.risk?.calculated_score_norm).toEqual(20); + expect(previewAlerts[1]?._source?.host?.risk?.calculated_level).toEqual('Critical'); + expect(previewAlerts[1]?._source?.host?.risk?.calculated_score_norm).toEqual(96); }); }); }); diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_execution_logic/execution_logic/threshold_alert_suppression.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_execution_logic/execution_logic/threshold_alert_suppression.ts new file mode 100644 index 0000000000000..59ab5185f6ab6 --- /dev/null +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_execution_logic/execution_logic/threshold_alert_suppression.ts @@ -0,0 +1,787 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { v4 as uuidv4 } from 'uuid'; +import expect from 'expect'; + +import { + ALERT_SUPPRESSION_START, + ALERT_SUPPRESSION_END, + ALERT_SUPPRESSION_DOCS_COUNT, + ALERT_SUPPRESSION_TERMS, + ALERT_LAST_DETECTED, + TIMESTAMP, +} from '@kbn/rule-data-utils'; + +import { DETECTION_ENGINE_SIGNALS_STATUS_URL as DETECTION_ENGINE_ALERTS_STATUS_URL } from '@kbn/security-solution-plugin/common/constants'; + +import { ThresholdRuleCreateProps } from '@kbn/security-solution-plugin/common/api/detection_engine'; +import { RuleExecutionStatusEnum } from '@kbn/security-solution-plugin/common/api/detection_engine/rule_monitoring'; + +import { ALERT_ORIGINAL_TIME } from '@kbn/security-solution-plugin/common/field_maps/field_names'; +import { + createRule, + getOpenAlerts, + getPreviewAlerts, + getThresholdRuleForAlertTesting, + previewRule, + patchRule, + setAlertStatus, + dataGeneratorFactory, +} from '../../../utils'; +import { FtrProviderContext } from '../../../../../ftr_provider_context'; + +export default ({ getService }: FtrProviderContext) => { + const supertest = getService('supertest'); + const esArchiver = getService('esArchiver'); + const es = getService('es'); + const log = getService('log'); + + describe('@ess @serverless Threshold type rules, alert suppression', () => { + const { indexListOfDocuments, indexGeneratedDocuments } = dataGeneratorFactory({ + es, + index: 'ecs_compliant', + log, + }); + + before(async () => { + await esArchiver.load('x-pack/test/functional/es_archives/security_solution/ecs_compliant'); + }); + + after(async () => { + await esArchiver.unload('x-pack/test/functional/es_archives/security_solution/ecs_compliant'); + }); + + it('should update an alert using real rule executions', async () => { + const id = uuidv4(); + const firstTimestamp = new Date().toISOString(); + const firstDocument = { + id, + '@timestamp': firstTimestamp, + agent: { + name: 'agent-1', + }, + }; + await indexListOfDocuments([firstDocument, firstDocument]); + + const rule: ThresholdRuleCreateProps = { + ...getThresholdRuleForAlertTesting(['ecs_compliant']), + query: `id:${id}`, + threshold: { + field: ['agent.name'], + value: 2, + }, + alert_suppression: { + duration: { + value: 300, + unit: 'm', + }, + }, + from: 'now-35m', + interval: '30m', + }; + const createdRule = await createRule(supertest, log, rule); + const alerts = await getOpenAlerts(supertest, log, es, createdRule); + expect(alerts.hits.hits.length).toEqual(1); + + // suppression start equal to alert timestamp + const suppressionStart = alerts.hits.hits[0]._source?.[TIMESTAMP]; + + expect(alerts.hits.hits[0]._source).toEqual( + expect.objectContaining({ + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'agent.name', + value: 'agent-1', + }, + ], + // suppression boundaries equal to alert time, since no alert been suppressed + [ALERT_SUPPRESSION_START]: suppressionStart, + [ALERT_SUPPRESSION_END]: suppressionStart, + [ALERT_ORIGINAL_TIME]: firstTimestamp, + [TIMESTAMP]: suppressionStart, + [ALERT_SUPPRESSION_DOCS_COUNT]: 0, + }) + ); + + const secondTimestamp = new Date().toISOString(); + const secondDocument = { + id, + '@timestamp': secondTimestamp, + agent: { + name: 'agent-1', + }, + }; + // Add a new document, then disable and re-enable to trigger another rule run. The second doc should + // trigger an update to the existing alert without changing the timestamp + await indexListOfDocuments([secondDocument, secondDocument]); + await patchRule(supertest, log, { id: createdRule.id, enabled: false }); + await patchRule(supertest, log, { id: createdRule.id, enabled: true }); + const afterTimestamp = new Date(); + const secondAlerts = await getOpenAlerts( + supertest, + log, + es, + createdRule, + RuleExecutionStatusEnum.succeeded, + undefined, + afterTimestamp + ); + expect(secondAlerts.hits.hits.length).toEqual(1); + expect(secondAlerts.hits.hits[0]._source).toEqual( + expect.objectContaining({ + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'agent.name', + value: 'agent-1', + }, + ], + [ALERT_ORIGINAL_TIME]: firstTimestamp, // timestamp is the same + [ALERT_SUPPRESSION_START]: suppressionStart, // suppression start is the same + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }) + ); + // suppression end value should be greater than second document timestamp, but lesser than current time + const suppressionEnd = new Date( + secondAlerts.hits.hits[0]._source?.[ALERT_SUPPRESSION_END] as string + ).getTime(); + expect(suppressionEnd).toBeLessThan(new Date().getTime()); + expect(suppressionEnd).toBeGreaterThan(new Date(secondTimestamp).getDate()); + }); + + it('should NOT suppress and update an alert if the alert is closed', async () => { + const id = uuidv4(); + const firstTimestamp = new Date().toISOString(); + const firstDocument = { + id, + '@timestamp': firstTimestamp, + agent: { + name: 'agent-1', + }, + }; + await indexListOfDocuments([firstDocument, firstDocument]); + + const rule: ThresholdRuleCreateProps = { + ...getThresholdRuleForAlertTesting(['ecs_compliant']), + query: `id:${id}`, + threshold: { + field: ['agent.name'], + value: 2, + }, + alert_suppression: { + duration: { + value: 300, + unit: 'm', + }, + }, + from: 'now-35m', + interval: '30m', + }; + const createdRule = await createRule(supertest, log, rule); + const alerts = await getOpenAlerts(supertest, log, es, createdRule); + + // Close the alert. Subsequent rule executions should ignore this closed alert + // for suppression purposes. + const alertIds = alerts.hits.hits.map((alert) => alert._id); + await supertest + .post(DETECTION_ENGINE_ALERTS_STATUS_URL) + .set('kbn-xsrf', 'true') + .send(setAlertStatus({ alertIds, status: 'closed' })) + .expect(200); + + const secondTimestamp = new Date().toISOString(); + const secondDocument = { + id, + '@timestamp': secondTimestamp, + agent: { + name: 'agent-1', + }, + }; + // Add new documents, then disable and re-enable to trigger another rule run. The second doc should + // trigger a new alert since the first one is now closed. + await indexListOfDocuments([secondDocument, secondDocument]); + await patchRule(supertest, log, { id: createdRule.id, enabled: false }); + await patchRule(supertest, log, { id: createdRule.id, enabled: true }); + const afterTimestamp = new Date(); + const secondAlerts = await getOpenAlerts( + supertest, + log, + es, + createdRule, + RuleExecutionStatusEnum.succeeded, + undefined, + afterTimestamp + ); + expect(secondAlerts.hits.hits.length).toEqual(2); + expect(alerts.hits.hits[0]._source).toEqual( + expect.objectContaining({ + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'agent.name', + value: 'agent-1', + }, + ], + [ALERT_ORIGINAL_TIME]: firstTimestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 0, + }) + ); + expect(secondAlerts.hits.hits[1]._source).toEqual( + expect.objectContaining({ + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'agent.name', + value: 'agent-1', + }, + ], + [ALERT_ORIGINAL_TIME]: secondTimestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 0, + }) + ); + }); + + it('should generate an alert per rule run when duration is less than rule interval', async () => { + const id = uuidv4(); + const timestamp = '2020-10-28T05:45:00.000Z'; + const firstRunDoc = { + id, + '@timestamp': timestamp, + agent: { + name: 'agent-1', + }, + }; + const secondRunDoc = { + ...firstRunDoc, + '@timestamp': '2020-10-28T06:15:00.000Z', + }; + + await indexListOfDocuments([ + firstRunDoc, + firstRunDoc, + firstRunDoc, + secondRunDoc, + secondRunDoc, + ]); + + const rule: ThresholdRuleCreateProps = { + ...getThresholdRuleForAlertTesting(['ecs_compliant']), + query: `id:${id}`, + threshold: { + field: ['agent.name'], + value: 2, + }, + alert_suppression: { + duration: { + value: 20, + unit: 'm', + }, + }, + from: 'now-35m', + interval: '30m', + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T06:30:00.000Z'), + invocationCount: 2, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: [ALERT_ORIGINAL_TIME], + }); + expect(previewAlerts.length).toBe(2); + expect(previewAlerts[0]._source).toEqual( + expect.objectContaining({ + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'agent.name', + value: 'agent-1', + }, + ], + [TIMESTAMP]: '2020-10-28T06:00:00.000Z', + [ALERT_SUPPRESSION_START]: '2020-10-28T06:00:00.000Z', + [ALERT_SUPPRESSION_END]: '2020-10-28T06:00:00.000Z', + [ALERT_SUPPRESSION_DOCS_COUNT]: 0, + }) + ); + + expect(previewAlerts[1]._source).toEqual( + expect.objectContaining({ + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'agent.name', + value: 'agent-1', + }, + ], + [TIMESTAMP]: '2020-10-28T06:30:00.000Z', + [ALERT_SUPPRESSION_START]: '2020-10-28T06:30:00.000Z', + [ALERT_SUPPRESSION_END]: '2020-10-28T06:30:00.000Z', + [ALERT_SUPPRESSION_DOCS_COUNT]: 0, + }) + ); + }); + + it('should update an existing alert in the time window that covers 2 rule executions', async () => { + const id = uuidv4(); + const timestamp = '2020-10-28T05:45:00.000Z'; + const firstRunDoc = { + id, + '@timestamp': timestamp, + agent: { + name: 'agent-1', + }, + }; + const secondRunDoc = { + ...firstRunDoc, + '@timestamp': '2020-10-28T06:15:00.000Z', + }; + + await indexListOfDocuments([ + firstRunDoc, + firstRunDoc, + firstRunDoc, + secondRunDoc, + secondRunDoc, + ]); + + const rule: ThresholdRuleCreateProps = { + ...getThresholdRuleForAlertTesting(['ecs_compliant']), + query: `id:${id}`, + threshold: { + field: ['agent.name'], + value: 1, + }, + alert_suppression: { + duration: { + value: 2, + unit: 'h', + }, + }, + from: 'now-35m', + interval: '30m', + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T06:30:00.000Z'), + invocationCount: 2, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: [ALERT_ORIGINAL_TIME], + }); + expect(previewAlerts.length).toBe(1); + expect(previewAlerts[0]._source).toEqual({ + ...previewAlerts[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'agent.name', + value: 'agent-1', + }, + ], + [TIMESTAMP]: '2020-10-28T06:00:00.000Z', + [ALERT_LAST_DETECTED]: '2020-10-28T06:30:00.000Z', // Note: ALERT_LAST_DETECTED gets updated, timestamp does not + [ALERT_ORIGINAL_TIME]: timestamp, + [ALERT_SUPPRESSION_START]: '2020-10-28T06:00:00.000Z', + [ALERT_SUPPRESSION_END]: '2020-10-28T06:30:00.000Z', + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }); + }); + + it('should update an existing alert in the time window that covers 3 rule executions', async () => { + const id = uuidv4(); + const timestamp = '2020-10-28T05:45:00.000Z'; + const firstRunDoc = { + id, + '@timestamp': timestamp, + agent: { + name: 'agent-1', + }, + }; + const secondRunDoc = { + ...firstRunDoc, + '@timestamp': '2020-10-28T06:15:00.000Z', + }; + const thirdRunDoc = { + ...firstRunDoc, + '@timestamp': '2020-10-28T06:45:00.000Z', + }; + + await indexListOfDocuments([ + firstRunDoc, + firstRunDoc, + firstRunDoc, + secondRunDoc, + secondRunDoc, + thirdRunDoc, + ]); + + const rule: ThresholdRuleCreateProps = { + ...getThresholdRuleForAlertTesting(['ecs_compliant']), + query: `id:${id}`, + threshold: { + field: ['agent.name'], + value: 1, + }, + alert_suppression: { + duration: { + value: 2, + unit: 'h', + }, + }, + from: 'now-35m', + interval: '30m', + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T07:00:00.000Z'), + invocationCount: 3, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: [ALERT_ORIGINAL_TIME], + }); + expect(previewAlerts.length).toEqual(1); + expect(previewAlerts[0]._source).toEqual({ + ...previewAlerts[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'agent.name', + value: 'agent-1', + }, + ], + [TIMESTAMP]: '2020-10-28T06:00:00.000Z', + [ALERT_LAST_DETECTED]: '2020-10-28T07:00:00.000Z', // Note: ALERT_LAST_DETECTED gets updated, timestamp does not + [ALERT_ORIGINAL_TIME]: timestamp, + [ALERT_SUPPRESSION_START]: '2020-10-28T06:00:00.000Z', + [ALERT_SUPPRESSION_END]: '2020-10-28T07:00:00.000Z', + [ALERT_SUPPRESSION_DOCS_COUNT]: 2, + }); + }); + + // needed to ensure threshold history works correctly for suppressed alerts + // history should be updated from events in suppressed alerts, not existing alert + // so subsequent rule runs won't trigger false positives + it('should not generate false positives suppressed alerts when threshold history is present', async () => { + const id = uuidv4(); + const timestamp = '2020-10-28T05:45:00.000Z'; + const firstRunDoc = { + id, + '@timestamp': timestamp, + agent: { + name: 'agent-1', + }, + }; + const secondRunDoc = { + ...firstRunDoc, + '@timestamp': '2020-10-28T06:15:00.000Z', + }; + + await indexListOfDocuments([ + firstRunDoc, + firstRunDoc, + firstRunDoc, + secondRunDoc, + secondRunDoc, + ]); + + const rule: ThresholdRuleCreateProps = { + ...getThresholdRuleForAlertTesting(['ecs_compliant']), + query: `id:${id}`, + threshold: { + field: ['agent.name'], + value: 2, + }, + alert_suppression: { + duration: { + value: 2, + unit: 'h', + }, + }, + from: 'now-60m', + interval: '30m', + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T07:00:00.000Z'), + invocationCount: 3, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: [ALERT_ORIGINAL_TIME], + }); + expect(previewAlerts.length).toEqual(1); + expect(previewAlerts[0]._source).toEqual({ + ...previewAlerts[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'agent.name', + value: 'agent-1', + }, + ], + [TIMESTAMP]: '2020-10-28T06:00:00.000Z', + [ALERT_LAST_DETECTED]: '2020-10-28T06:30:00.000Z', // Note: ALERT_LAST_DETECTED gets updated, timestamp does not + [ALERT_ORIGINAL_TIME]: timestamp, + [ALERT_SUPPRESSION_START]: '2020-10-28T06:00:00.000Z', + [ALERT_SUPPRESSION_END]: '2020-10-28T06:30:00.000Z', + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, // only one suppressed alert as expected + }); + }); + + it('should update the correct alerts based on multi values threshold.field', async () => { + const id = uuidv4(); + const timestamp = '2020-10-28T05:45:00.000Z'; + const firstRunDocA = { + id, + '@timestamp': timestamp, + agent: { + name: 'agent-1', + type: 'auditbeat', + }, + }; + const firstRunDocF = { + ...firstRunDocA, + agent: { + name: 'agent-1', + type: 'filebeat', + }, + }; + const firstRunDocAgent2 = { + ...firstRunDocA, + agent: { + name: 'agent-2', + type: 'auditbeat', + }, + }; + + const secondRunDocA = { + ...firstRunDocA, + '@timestamp': '2020-10-28T06:15:00.000Z', + }; + + await indexListOfDocuments([ + firstRunDocA, + firstRunDocA, + firstRunDocF, + firstRunDocF, + firstRunDocAgent2, + secondRunDocA, + secondRunDocA, + secondRunDocA, + ]); + + const rule: ThresholdRuleCreateProps = { + ...getThresholdRuleForAlertTesting(['ecs_compliant']), + query: `id:${id}`, + threshold: { + field: ['agent.name', 'agent.type'], + value: 2, + }, + alert_suppression: { + duration: { + value: 2, + unit: 'h', + }, + }, + from: 'now-35m', + interval: '30m', + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T06:30:00.000Z'), + invocationCount: 2, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: ['agent.type', ALERT_ORIGINAL_TIME], + }); + // 2 alert should be generated: + // 1. for pair 'agent-1', 'auditbeat' - suppressed + // 2. for pair 'agent-1', 'filebeat' - not suppressed + expect(previewAlerts.length).toEqual(2); + expect(previewAlerts[0]._source).toEqual({ + ...previewAlerts[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'agent.name', + value: 'agent-1', + }, + { + field: 'agent.type', + value: 'auditbeat', + }, + ], + [TIMESTAMP]: '2020-10-28T06:00:00.000Z', + [ALERT_LAST_DETECTED]: '2020-10-28T06:30:00.000Z', // Note: ALERT_LAST_DETECTED gets updated, timestamp does not + [ALERT_ORIGINAL_TIME]: timestamp, + [ALERT_SUPPRESSION_START]: '2020-10-28T06:00:00.000Z', + [ALERT_SUPPRESSION_END]: '2020-10-28T06:30:00.000Z', + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }); + expect(previewAlerts[1]._source).toEqual({ + ...previewAlerts[1]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'agent.name', + value: 'agent-1', + }, + { + field: 'agent.type', + value: 'filebeat', + }, + ], + [TIMESTAMP]: '2020-10-28T06:00:00.000Z', + [ALERT_LAST_DETECTED]: '2020-10-28T06:00:00.000Z', + [ALERT_ORIGINAL_TIME]: timestamp, + [ALERT_SUPPRESSION_START]: '2020-10-28T06:00:00.000Z', + [ALERT_SUPPRESSION_END]: '2020-10-28T06:00:00.000Z', + [ALERT_SUPPRESSION_DOCS_COUNT]: 0, // no suppressed alerts + }); + }); + + it('should correctly suppress when using a timestamp override', async () => { + const id = uuidv4(); + const timestamp = '2020-10-28T05:45:00.000Z'; + const docWithoutOverride = { + id, + '@timestamp': timestamp, + agent: { + name: 'agent-1', + }, + }; + const docWithOverride = { + ...docWithoutOverride, + // This doc simulates a very late arriving doc + '@timestamp': '2020-10-28T03:00:00.000Z', + event: { + ingested: '2020-10-28T06:10:00.000Z', + }, + }; + await indexListOfDocuments([docWithoutOverride, docWithOverride]); + + const rule: ThresholdRuleCreateProps = { + ...getThresholdRuleForAlertTesting(['ecs_compliant']), + query: `id:${id}`, + threshold: { + field: ['agent.name'], + value: 1, + }, + alert_suppression: { + duration: { + value: 300, + unit: 'm', + }, + }, + from: 'now-35m', + interval: '30m', + timestamp_override: 'event.ingested', + }; + // 1 alert should be suppressed, based on event.ingested value of a document + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T06:30:00.000Z'), + invocationCount: 2, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: ['agent.name', ALERT_ORIGINAL_TIME], + }); + expect(previewAlerts.length).toEqual(1); + expect(previewAlerts[0]._source).toEqual({ + ...previewAlerts[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'agent.name', + value: 'agent-1', + }, + ], + [ALERT_ORIGINAL_TIME]: timestamp, + [ALERT_SUPPRESSION_START]: '2020-10-28T06:00:00.000Z', + [ALERT_SUPPRESSION_END]: '2020-10-28T06:30:00.000Z', + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }); + }); + + it('should generate and update up to max_signals alerts', async () => { + const id = uuidv4(); + const timestamp = '2020-10-28T05:45:00.000Z'; + const laterTimestamp = '2020-10-28T06:10:00.000Z'; + + await Promise.all( + [timestamp, laterTimestamp].map((t) => + indexGeneratedDocuments({ + docsCount: 150, + seed: (index) => ({ + id, + '@timestamp': t, + agent: { + name: `agent-${index}`, + }, + }), + }) + ) + ); + + const rule: ThresholdRuleCreateProps = { + ...getThresholdRuleForAlertTesting(['ecs_compliant']), + query: `id:${id}`, + threshold: { + field: ['agent.name'], + value: 1, + }, + alert_suppression: { + duration: { + value: 300, + unit: 'm', + }, + }, + from: 'now-35m', + interval: '30m', + max_signals: 150, + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T06:30:00.000Z'), + invocationCount: 2, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + size: 1000, + sort: ['agent.name', ALERT_ORIGINAL_TIME], + }); + expect(previewAlerts.length).toEqual(150); + expect(previewAlerts[0]._source).toEqual( + expect.objectContaining({ + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'agent.name', + value: 'agent-0', + }, + ], + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }) + ); + }); + }); +}; diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_execution_logic/keyword_family/keyword_mixed_with_const.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_execution_logic/keyword_family/keyword_mixed_with_const.ts index 1353c0b6ca5ed..dd4310d4bb069 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_execution_logic/keyword_family/keyword_mixed_with_const.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_execution_logic/keyword_family/keyword_mixed_with_const.ts @@ -139,6 +139,7 @@ export default ({ getService }: FtrProviderContext) => { field: 'event.dataset', value: 1, }, + alert_suppression: undefined, }; const { id } = await createRule(supertest, log, rule); await waitForRuleSuccess({ supertest, log, id }); diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/get_threshold_rule_for_alert_testing.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/get_threshold_rule_for_alert_testing.ts index f66e342a7431c..a64aa04981c3a 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/get_threshold_rule_for_alert_testing.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/get_threshold_rule_for_alert_testing.ts @@ -28,4 +28,5 @@ export const getThresholdRuleForAlertTesting = ( field: 'process.name', value: 21, }, + alert_suppression: undefined, }); diff --git a/x-pack/test/security_solution_cypress/config.ts b/x-pack/test/security_solution_cypress/config.ts index fb34362f7fb9b..4fe61b660f1a4 100644 --- a/x-pack/test/security_solution_cypress/config.ts +++ b/x-pack/test/security_solution_cypress/config.ts @@ -46,6 +46,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) { '--xpack.ruleRegistry.unsafe.legacyMultiTenancy.enabled=true', `--xpack.securitySolution.enableExperimental=${JSON.stringify([ 'chartEmbeddablesEnabled', + 'alertSuppressionForThresholdRuleEnabled', ])}`, // mock cloud to enable the guided onboarding tour in e2e tests '--xpack.cloud.id=test', diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_creation/threshold_rule.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_creation/threshold_rule.cy.ts index 62fdb9121c9c5..c81b93bc5757b 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_creation/threshold_rule.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_creation/threshold_rule.cy.ts @@ -41,13 +41,22 @@ import { TAGS_DETAILS, THRESHOLD_DETAILS, TIMELINE_TEMPLATE_DETAILS, + SUPPRESS_FOR_DETAILS, } from '../../../screens/rule_details'; -import { getDetails, waitForTheRuleToBeExecuted } from '../../../tasks/rule_details'; +import { + getDetails, + waitForTheRuleToBeExecuted, + assertDetailsNotExist, +} from '../../../tasks/rule_details'; import { expectNumberOfRules, goToRuleDetailsOf } from '../../../tasks/alerts_detection_rules'; import { deleteAlertsAndRules } from '../../../tasks/api_calls/common'; import { createAndEnableRule, + createRuleWithoutEnabling, + fillAboutRuleMinimumAndContinue, + enablesAndPopulatesThresholdSuppression, + skipScheduleRuleAction, fillAboutRuleAndContinue, fillDefineThresholdRuleAndContinue, fillScheduleRuleAndContinue, @@ -59,75 +68,107 @@ import { visit } from '../../../tasks/navigation'; import { openRuleManagementPageViaBreadcrumbs } from '../../../tasks/rules_management'; import { CREATE_RULE_URL } from '../../../urls/navigation'; -describe('Threshold rules', { tags: ['@ess', '@serverless'] }, () => { - const rule = getNewThresholdRule(); - const expectedUrls = rule.references?.join(''); - const expectedFalsePositives = rule.false_positives?.join(''); - const expectedTags = rule.tags?.join(''); - const mitreAttack = rule.threat; - const expectedMitre = formatMitreAttackDescription(mitreAttack ?? []); - - beforeEach(() => { - deleteAlertsAndRules(); - login(); - visit(CREATE_RULE_URL); - }); - - it('Creates and enables a new threshold rule', () => { - selectThresholdRuleType(); - fillDefineThresholdRuleAndContinue(rule); - fillAboutRuleAndContinue(rule); - fillScheduleRuleAndContinue(rule); - createAndEnableRule(); - openRuleManagementPageViaBreadcrumbs(); - - cy.get(CUSTOM_RULES_BTN).should('have.text', 'Custom rules (1)'); - - expectNumberOfRules(RULES_MANAGEMENT_TABLE, 1); - - cy.get(RULE_NAME).should('have.text', rule.name); - cy.get(RISK_SCORE).should('have.text', rule.risk_score); - cy.get(SEVERITY).should('have.text', 'High'); - cy.get(RULE_SWITCH).should('have.attr', 'aria-checked', 'true'); - - goToRuleDetailsOf(rule.name); - - cy.get(RULE_NAME_HEADER).should('contain', `${rule.name}`); - cy.get(ABOUT_RULE_DESCRIPTION).should('have.text', rule.description); - cy.get(ABOUT_DETAILS).within(() => { - getDetails(SEVERITY_DETAILS).should('have.text', 'High'); - getDetails(RISK_SCORE_DETAILS).should('have.text', rule.risk_score); - getDetails(REFERENCE_URLS_DETAILS).should((details) => { - expect(removeExternalLinkText(details.text())).equal(expectedUrls); +describe( + 'Threshold rules', + { + tags: ['@ess', '@serverless'], + env: { + ftrConfig: { + enableExperimental: ['alertSuppressionForThresholdRuleEnabled'], + }, + }, + }, + () => { + const rule = getNewThresholdRule(); + const expectedUrls = rule.references?.join(''); + const expectedFalsePositives = rule.false_positives?.join(''); + const expectedTags = rule.tags?.join(''); + const mitreAttack = rule.threat; + const expectedMitre = formatMitreAttackDescription(mitreAttack ?? []); + + beforeEach(() => { + deleteAlertsAndRules(); + login(); + visit(CREATE_RULE_URL); + }); + + it('Creates and enables a new threshold rule', () => { + selectThresholdRuleType(); + fillDefineThresholdRuleAndContinue(rule); + fillAboutRuleAndContinue(rule); + fillScheduleRuleAndContinue(rule); + createAndEnableRule(); + openRuleManagementPageViaBreadcrumbs(); + + cy.get(CUSTOM_RULES_BTN).should('have.text', 'Custom rules (1)'); + + expectNumberOfRules(RULES_MANAGEMENT_TABLE, 1); + + cy.get(RULE_NAME).should('have.text', rule.name); + cy.get(RISK_SCORE).should('have.text', rule.risk_score); + cy.get(SEVERITY).should('have.text', 'High'); + cy.get(RULE_SWITCH).should('have.attr', 'aria-checked', 'true'); + + goToRuleDetailsOf(rule.name); + + cy.get(RULE_NAME_HEADER).should('contain', `${rule.name}`); + cy.get(ABOUT_RULE_DESCRIPTION).should('have.text', rule.description); + cy.get(ABOUT_DETAILS).within(() => { + getDetails(SEVERITY_DETAILS).should('have.text', 'High'); + getDetails(RISK_SCORE_DETAILS).should('have.text', rule.risk_score); + getDetails(REFERENCE_URLS_DETAILS).should((details) => { + expect(removeExternalLinkText(details.text())).equal(expectedUrls); + }); + getDetails(FALSE_POSITIVES_DETAILS).should('have.text', expectedFalsePositives); + getDetails(MITRE_ATTACK_DETAILS).should((mitre) => { + expect(removeExternalLinkText(mitre.text())).equal(expectedMitre); + }); + getDetails(TAGS_DETAILS).should('have.text', expectedTags); }); - getDetails(FALSE_POSITIVES_DETAILS).should('have.text', expectedFalsePositives); - getDetails(MITRE_ATTACK_DETAILS).should((mitre) => { - expect(removeExternalLinkText(mitre.text())).equal(expectedMitre); + cy.get(INVESTIGATION_NOTES_TOGGLE).click(); + cy.get(ABOUT_INVESTIGATION_NOTES).should('have.text', INVESTIGATION_NOTES_MARKDOWN); + cy.get(DEFINITION_DETAILS).within(() => { + getDetails(INDEX_PATTERNS_DETAILS).should('have.text', getIndexPatterns().join('')); + getDetails(CUSTOM_QUERY_DETAILS).should('have.text', rule.query); + getDetails(RULE_TYPE_DETAILS).should('have.text', 'Threshold'); + getDetails(TIMELINE_TEMPLATE_DETAILS).should('have.text', 'None'); + getDetails(THRESHOLD_DETAILS).should( + 'have.text', + `Results aggregated by ${rule.threshold.field} >= ${rule.threshold.value}` + ); + assertDetailsNotExist(SUPPRESS_FOR_DETAILS); }); - getDetails(TAGS_DETAILS).should('have.text', expectedTags); - }); - cy.get(INVESTIGATION_NOTES_TOGGLE).click(); - cy.get(ABOUT_INVESTIGATION_NOTES).should('have.text', INVESTIGATION_NOTES_MARKDOWN); - cy.get(DEFINITION_DETAILS).within(() => { - getDetails(INDEX_PATTERNS_DETAILS).should('have.text', getIndexPatterns().join('')); - getDetails(CUSTOM_QUERY_DETAILS).should('have.text', rule.query); - getDetails(RULE_TYPE_DETAILS).should('have.text', 'Threshold'); - getDetails(TIMELINE_TEMPLATE_DETAILS).should('have.text', 'None'); - getDetails(THRESHOLD_DETAILS).should( - 'have.text', - `Results aggregated by ${rule.threshold.field} >= ${rule.threshold.value}` - ); - }); - cy.get(SCHEDULE_DETAILS).within(() => { - getDetails(RUNS_EVERY_DETAILS).should('have.text', `${rule.interval}`); - const humanizedDuration = getHumanizedDuration(rule.from ?? 'now-6m', rule.interval ?? '5m'); - getDetails(ADDITIONAL_LOOK_BACK_DETAILS).should('have.text', `${humanizedDuration}`); + cy.get(SCHEDULE_DETAILS).within(() => { + getDetails(RUNS_EVERY_DETAILS).should('have.text', `${rule.interval}`); + const humanizedDuration = getHumanizedDuration( + rule.from ?? 'now-6m', + rule.interval ?? '5m' + ); + getDetails(ADDITIONAL_LOOK_BACK_DETAILS).should('have.text', `${humanizedDuration}`); + }); + + waitForTheRuleToBeExecuted(); + waitForAlertsToPopulate(); + + cy.get(ALERTS_COUNT).should(($count) => expect(+$count.text().split(' ')[0]).to.be.lt(100)); + cy.get(ALERT_GRID_CELL).contains(rule.name); }); - waitForTheRuleToBeExecuted(); - waitForAlertsToPopulate(); + it('Creates a new threshold rule with suppression enabled', () => { + selectThresholdRuleType(); - cy.get(ALERTS_COUNT).should(($count) => expect(+$count.text().split(' ')[0]).to.be.lt(100)); - cy.get(ALERT_GRID_CELL).contains(rule.name); - }); -}); + enablesAndPopulatesThresholdSuppression(5, 'h'); + fillDefineThresholdRuleAndContinue(rule); + + fillAboutRuleMinimumAndContinue(rule); + skipScheduleRuleAction(); + createRuleWithoutEnabling(); + openRuleManagementPageViaBreadcrumbs(); + goToRuleDetailsOf(rule.name); + + cy.get(DEFINITION_DETAILS).within(() => { + getDetails(SUPPRESS_FOR_DETAILS).should('have.text', '5h'); + }); + }); + } +); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_creation/threshold_rule_ess_basic.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_creation/threshold_rule_ess_basic.cy.ts new file mode 100644 index 0000000000000..2c8d5879834e1 --- /dev/null +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_creation/threshold_rule_ess_basic.cy.ts @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + ALERT_SUPPRESSION_DURATION_INPUT, + THRESHOLD_ENABLE_SUPPRESSION_CHECKBOX, +} from '../../../screens/create_new_rule'; + +import { deleteAlertsAndRules } from '../../../tasks/api_calls/common'; +import { startBasicLicense } from '../../../tasks/api_calls/licensing'; +import { selectThresholdRuleType } from '../../../tasks/create_new_rule'; +import { login } from '../../../tasks/login'; +import { visit } from '../../../tasks/navigation'; +import { CREATE_RULE_URL } from '../../../urls/navigation'; +import { TOOLTIP } from '../../../screens/common'; + +describe('Threshold rules, ESS basic license', { tags: ['@ess'] }, () => { + beforeEach(() => { + deleteAlertsAndRules(); + login(); + visit(CREATE_RULE_URL); + startBasicLicense(); + }); + + it('Alert suppression is disabled for basic license', () => { + selectThresholdRuleType(); + + cy.get(THRESHOLD_ENABLE_SUPPRESSION_CHECKBOX).should('be.disabled'); + cy.get(THRESHOLD_ENABLE_SUPPRESSION_CHECKBOX).parent().trigger('mouseover'); + // Platinum license is required, tooltip on disabled alert suppression checkbox should tell this + cy.get(TOOLTIP).contains('Platinum license'); + + cy.get(ALERT_SUPPRESSION_DURATION_INPUT).eq(0).should('be.disabled'); + cy.get(ALERT_SUPPRESSION_DURATION_INPUT).eq(1).should('be.disabled'); + }); +}); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_creation/threshold_rule_serverless_essentials.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_creation/threshold_rule_serverless_essentials.cy.ts new file mode 100644 index 0000000000000..ddeda8c0a2ff8 --- /dev/null +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_creation/threshold_rule_serverless_essentials.cy.ts @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { THRESHOLD_ENABLE_SUPPRESSION_CHECKBOX } from '../../../screens/create_new_rule'; + +import { deleteAlertsAndRules } from '../../../tasks/api_calls/common'; +import { selectThresholdRuleType } from '../../../tasks/create_new_rule'; +import { login } from '../../../tasks/login'; +import { visit } from '../../../tasks/navigation'; +import { CREATE_RULE_URL } from '../../../urls/navigation'; + +describe( + 'Threshold rules, Serverless essentials license', + { + tags: ['@serverless'], + + env: { + ftrConfig: { + productTypes: [ + { product_line: 'security', product_tier: 'essentials' }, + { product_line: 'endpoint', product_tier: 'essentials' }, + ], + enableExperimental: ['alertSuppressionForThresholdRuleEnabled'], + }, + }, + }, + () => { + beforeEach(() => { + deleteAlertsAndRules(); + login(); + visit(CREATE_RULE_URL); + }); + + it('Alert suppression is enabled for essentials', () => { + selectThresholdRuleType(); + + cy.get(THRESHOLD_ENABLE_SUPPRESSION_CHECKBOX).should('be.enabled'); + }); + } +); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_edit/threshold_rule.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_edit/threshold_rule.cy.ts new file mode 100644 index 0000000000000..2e249bb8f5195 --- /dev/null +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_edit/threshold_rule.cy.ts @@ -0,0 +1,115 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { getNewThresholdRule } from '../../../objects/rule'; + +import { + SUPPRESS_FOR_DETAILS, + DETAILS_TITLE, + SUPPRESS_BY_DETAILS, + SUPPRESS_MISSING_FIELD, +} from '../../../screens/rule_details'; + +import { + ALERT_SUPPRESSION_DURATION_INPUT, + THRESHOLD_ENABLE_SUPPRESSION_CHECKBOX, + ALERT_SUPPRESSION_DURATION_PER_RULE_EXECUTION, + ALERT_SUPPRESSION_DURATION_PER_TIME_INTERVAL, + ALERT_SUPPRESSION_FIELDS, +} from '../../../screens/create_new_rule'; + +import { createRule } from '../../../tasks/api_calls/rules'; + +import { RULES_MANAGEMENT_URL } from '../../../urls/rules_management'; +import { getDetails, assertDetailsNotExist } from '../../../tasks/rule_details'; +import { deleteAlertsAndRules } from '../../../tasks/api_calls/common'; +import { login } from '../../../tasks/login'; + +import { editFirstRule } from '../../../tasks/alerts_detection_rules'; + +import { saveEditedRule, goBackToRuleDetails } from '../../../tasks/edit_rule'; +import { enablesAndPopulatesThresholdSuppression } from '../../../tasks/create_new_rule'; +import { visit } from '../../../tasks/navigation'; + +const rule = getNewThresholdRule(); + +describe( + 'Detection threshold rules, edit', + { + tags: ['@ess', '@serverless'], + env: { + ftrConfig: { + enableExperimental: ['alertSuppressionForThresholdRuleEnabled'], + }, + }, + }, + () => { + describe('without suppression', () => { + beforeEach(() => { + login(); + deleteAlertsAndRules(); + createRule(rule); + }); + + it('enables suppression on time interval', () => { + visit(RULES_MANAGEMENT_URL); + editFirstRule(); + + // suppression fields are hidden since threshold fields used for suppression + cy.get(ALERT_SUPPRESSION_FIELDS).should('not.be.visible'); + + enablesAndPopulatesThresholdSuppression(60, 'm'); + + saveEditedRule(); + + // ensure typed interval is displayed on details page + getDetails(SUPPRESS_FOR_DETAILS).should('have.text', '60m'); + // suppression functionality should be under Tech Preview + cy.contains(DETAILS_TITLE, SUPPRESS_FOR_DETAILS).contains('Technical Preview'); + + // the rest of suppress properties do not exist for threshold rule + assertDetailsNotExist(SUPPRESS_BY_DETAILS); + assertDetailsNotExist(SUPPRESS_MISSING_FIELD); + }); + }); + + describe('with suppression enabled', () => { + beforeEach(() => { + login(); + deleteAlertsAndRules(); + createRule({ ...rule, alert_suppression: { duration: { value: 360, unit: 's' } } }); + }); + + it('displays suppress options correctly on edit form', () => { + visit(RULES_MANAGEMENT_URL); + editFirstRule(); + + cy.get(ALERT_SUPPRESSION_DURATION_PER_TIME_INTERVAL) + .should('be.enabled') + .should('be.checked'); + cy.get(ALERT_SUPPRESSION_DURATION_PER_RULE_EXECUTION) + .should('be.disabled') + .should('not.be.checked'); + + // ensures enable suppression checkbox is checked and suppression options displayed correctly + cy.get(THRESHOLD_ENABLE_SUPPRESSION_CHECKBOX).should('be.enabled').should('be.checked'); + cy.get(ALERT_SUPPRESSION_DURATION_INPUT) + .eq(0) + .should('be.enabled') + .should('have.value', 360); + cy.get(ALERT_SUPPRESSION_DURATION_INPUT) + .eq(1) + .should('be.enabled') + .should('have.value', 's'); + + goBackToRuleDetails(); + // no changes on rule details page + getDetails(SUPPRESS_FOR_DETAILS).should('have.text', '360s'); + }); + }); + } +); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/prebuilt_rules_preview.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/prebuilt_rules_preview.cy.ts index 81f37b7760df2..a8349a0410c63 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/prebuilt_rules_preview.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/prebuilt_rules_preview.cy.ts @@ -167,8 +167,8 @@ describe('Detection rules, Prebuilt Rules Installation and Update workflow', () name: 'Custom query index pattern rule', rule_id: 'custom_query_index_pattern_rule', ...(commonProperties as Record), - type: 'query', ...queryProperties, + type: 'query', index: ['winlogbeat-*', 'logs-endpoint.events.*'], alert_suppression: { group_by: [ @@ -216,8 +216,8 @@ describe('Detection rules, Prebuilt Rules Installation and Update workflow', () name: 'Threshold index pattern rule', rule_id: 'threshold_index_pattern_rule', ...commonProperties, - type: 'threshold', ...queryProperties, + type: 'threshold', language: 'lucene', index: ['winlogbeat-*', 'logs-endpoint.events.*'], threshold: { @@ -228,6 +228,7 @@ describe('Detection rules, Prebuilt Rules Installation and Update workflow', () value: 200, cardinality: [{ field: 'Ransomware.score', value: 3 }], }, + alert_suppression: undefined, }); const EQL_INDEX_PATTERN_RULE = createRuleAssetSavedObject({ @@ -245,8 +246,8 @@ describe('Detection rules, Prebuilt Rules Installation and Update workflow', () name: 'Threat match index pattern rule', rule_id: 'threat_match_index_pattern_rule', ...commonProperties, - type: 'threat_match', ...queryProperties, + type: 'threat_match', language: 'lucene', index: ['winlogbeat-*', 'logs-endpoint.events.*'], filters, @@ -285,8 +286,8 @@ describe('Detection rules, Prebuilt Rules Installation and Update workflow', () name: 'New terms index pattern rule', rule_id: 'new_terms_index_pattern_rule', ...commonProperties, - type: 'new_terms', ...queryProperties, + type: 'new_terms', query: '_id: *', new_terms_fields: ['Endpoint.policy.applied.id', 'Memory_protection.unique_key_v1'], history_window_start: 'now-9d', diff --git a/x-pack/test/security_solution_cypress/cypress/screens/create_new_rule.ts b/x-pack/test/security_solution_cypress/cypress/screens/create_new_rule.ts index 17e129813c866..c7fbe341eb700 100644 --- a/x-pack/test/security_solution_cypress/cypress/screens/create_new_rule.ts +++ b/x-pack/test/security_solution_cypress/cypress/screens/create_new_rule.ts @@ -25,6 +25,16 @@ export const ALERT_SUPPRESSION_FIELDS = export const ALERT_SUPPRESSION_DURATION_OPTIONS = '[data-test-subj="alertSuppressionDuration"] [data-test-subj="groupByDurationOptions"]'; +export const ALERT_SUPPRESSION_DURATION_PER_TIME_INTERVAL = `${ALERT_SUPPRESSION_DURATION_OPTIONS} #per-time-period`; + +export const ALERT_SUPPRESSION_DURATION_PER_RULE_EXECUTION = `${ALERT_SUPPRESSION_DURATION_OPTIONS} #per-rule-execution`; + +export const ALERT_SUPPRESSION_DURATION_INPUT = + '[data-test-subj="alertSuppressionDuration"] [data-test-subj="alertSuppressionDurationInput"]'; + +export const THRESHOLD_ENABLE_SUPPRESSION_CHECKBOX = + '[data-test-subj="detectionEngineStepDefineRuleThresholdEnableSuppression"] input'; + export const ANOMALY_THRESHOLD_INPUT = '[data-test-subj="anomalyThresholdSlider"] .euiFieldNumber'; export const ADVANCED_SETTINGS_BTN = '[data-test-subj="advancedSettings"] .euiAccordion__button'; diff --git a/x-pack/test/security_solution_cypress/cypress/screens/rule_details.ts b/x-pack/test/security_solution_cypress/cypress/screens/rule_details.ts index fd98e38f9cc32..f627ae6546182 100644 --- a/x-pack/test/security_solution_cypress/cypress/screens/rule_details.ts +++ b/x-pack/test/security_solution_cypress/cypress/screens/rule_details.ts @@ -128,6 +128,8 @@ export const SUPPRESS_BY_DETAILS = 'Suppress alerts by'; export const SUPPRESS_FOR_DETAILS = 'Suppress alerts for'; +export const SUPPRESS_MISSING_FIELD = 'If a suppression field is missing'; + export const TIMELINE_FIELD = (field: string) => { return `[data-test-subj="formatted-field-${field}"]`; }; diff --git a/x-pack/test/security_solution_cypress/cypress/tasks/api_calls/licensing.ts b/x-pack/test/security_solution_cypress/cypress/tasks/api_calls/licensing.ts new file mode 100644 index 0000000000000..c8e85cd9f9a65 --- /dev/null +++ b/x-pack/test/security_solution_cypress/cypress/tasks/api_calls/licensing.ts @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { API_BASE_PATH } from '@kbn/license-management-plugin/common/constants'; + +export const startBasicLicense = () => + cy.request({ + headers: { 'kbn-xsrf': 'cypress-creds', 'x-elastic-internal-origin': 'security-solution' }, + method: 'POST', + url: `${API_BASE_PATH}/start_basic?acknowledge=true`, + }); diff --git a/x-pack/test/security_solution_cypress/cypress/tasks/create_new_rule.ts b/x-pack/test/security_solution_cypress/cypress/tasks/create_new_rule.ts index 9faf5fa81ec00..a5f411670ad38 100644 --- a/x-pack/test/security_solution_cypress/cypress/tasks/create_new_rule.ts +++ b/x-pack/test/security_solution_cypress/cypress/tasks/create_new_rule.ts @@ -31,6 +31,10 @@ import { convertHistoryStartToSize, getHumanizedDuration } from '../helpers/rule import { ABOUT_CONTINUE_BTN, + ALERT_SUPPRESSION_DURATION_INPUT, + THRESHOLD_ENABLE_SUPPRESSION_CHECKBOX, + ALERT_SUPPRESSION_DURATION_PER_RULE_EXECUTION, + ALERT_SUPPRESSION_DURATION_PER_TIME_INTERVAL, ABOUT_EDIT_TAB, ACTIONS_EDIT_TAB, ADD_FALSE_POSITIVE_BTN, @@ -426,6 +430,13 @@ export const fillScheduleRuleAndContinue = (rule: RuleCreateProps) => { cy.get(SCHEDULE_CONTINUE_BUTTON).click({ force: true }); }; +/** + * use default schedule options + */ +export const skipScheduleRuleAction = () => { + cy.get(SCHEDULE_CONTINUE_BUTTON).click(); +}; + export const fillFrom = (from: RuleIntervalFrom = ruleFields.ruleIntervalFrom) => { const value = from.slice(0, from.length - 1); const type = from.slice(from.length - 1); @@ -763,6 +774,28 @@ export const selectAndLoadSavedQuery = (queryName: string, queryValue: string) = cy.get(CUSTOM_QUERY_INPUT).should('have.value', queryValue); }; +export const enablesAndPopulatesThresholdSuppression = ( + interval: number, + timeUnit: 's' | 'm' | 'h' +) => { + // enable suppression is unchecked so the rest of suppression components are disabled + cy.get(ALERT_SUPPRESSION_DURATION_PER_TIME_INTERVAL).should('be.disabled').should('be.checked'); + cy.get(ALERT_SUPPRESSION_DURATION_PER_RULE_EXECUTION) + .should('be.disabled') + .should('not.be.checked'); + + // enables suppression for threshold rule + cy.get(THRESHOLD_ENABLE_SUPPRESSION_CHECKBOX).should('not.be.checked'); + cy.get(THRESHOLD_ENABLE_SUPPRESSION_CHECKBOX).siblings('label').click(); + + cy.get(ALERT_SUPPRESSION_DURATION_INPUT).first().type(`{selectall}${interval}`); + cy.get(ALERT_SUPPRESSION_DURATION_INPUT).eq(1).select(timeUnit); + + // rule execution radio option is disabled, per time interval becomes enabled when suppression enabled + cy.get(ALERT_SUPPRESSION_DURATION_PER_RULE_EXECUTION).should('be.disabled'); + cy.get(ALERT_SUPPRESSION_DURATION_PER_TIME_INTERVAL).should('be.enabled').should('be.checked'); +}; + export const checkLoadQueryDynamically = () => { cy.get(LOAD_QUERY_DYNAMICALLY_CHECKBOX).click({ force: true }); cy.get(LOAD_QUERY_DYNAMICALLY_CHECKBOX).should('be.checked'); diff --git a/x-pack/test/security_solution_cypress/cypress/tsconfig.json b/x-pack/test/security_solution_cypress/cypress/tsconfig.json index 3e3563fa2e97b..45b526793e98e 100644 --- a/x-pack/test/security_solution_cypress/cypress/tsconfig.json +++ b/x-pack/test/security_solution_cypress/cypress/tsconfig.json @@ -39,6 +39,7 @@ "@kbn/security-plugin", "@kbn/management-settings-ids", "@kbn/es-query", - "@kbn/ml-plugin" + "@kbn/ml-plugin", + "@kbn/license-management-plugin" ] } diff --git a/x-pack/test/security_solution_cypress/serverless_config.ts b/x-pack/test/security_solution_cypress/serverless_config.ts index d0ee1613f6e4c..8eb8d2efdefdc 100644 --- a/x-pack/test/security_solution_cypress/serverless_config.ts +++ b/x-pack/test/security_solution_cypress/serverless_config.ts @@ -34,6 +34,9 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) { { product_line: 'endpoint', product_tier: 'complete' }, { product_line: 'cloud', product_tier: 'complete' }, ])}`, + `--xpack.securitySolution.enableExperimental=${JSON.stringify([ + 'alertSuppressionForThresholdRuleEnabled', + ])}`, ], }, testRunner: SecuritySolutionConfigurableCypressTestRunner,