From dc6c976336688a6825d36dddad459ced8859c734 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com> Date: Thu, 4 Jan 2024 10:09:41 +0000 Subject: [PATCH 01/28] move alert suppression schema to common (#174240) ## Summary Summarize your PR. If it involves visual changes include a screenshot or gif. ### Checklist Delete any items that are not applicable to this PR. - [ ] Any text added follows [EUI's writing guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses sentence case text and includes [i18n support](https://github.com/elastic/kibana/blob/main/packages/kbn-i18n/README.md) - [ ] [Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html) was added for features that require explanation or tutorials - [ ] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios - [ ] [Flaky Test Runner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1) was used on any tests changed - [ ] Any UI touched in this PR is usable by keyboard only (learn more about [keyboard accessibility](https://webaim.org/techniques/keyboard/)) - [ ] Any UI touched in this PR does not create any new axe failures (run axe in browser: [FF](https://addons.mozilla.org/en-US/firefox/addon/axe-devtools/), [Chrome](https://chrome.google.com/webstore/detail/axe-web-accessibility-tes/lhdoppojpmngadmnindnejefpokejbdd?hl=en-US)) - [ ] If a plugin configuration key changed, check if it needs to be allowlisted in the cloud and added to the [docker list](https://github.com/elastic/kibana/blob/main/src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker) - [ ] This renders correctly on smaller devices using a responsive layout. (You can test this [in your browser](https://www.browserstack.com/guide/responsive-testing-on-local-server)) - [ ] This was checked for [cross-browser compatibility](https://www.elastic.co/support/matrix#matrix_browsers) ### Risk Matrix Delete this section if it is not applicable to this PR. Before closing this PR, invite QA, stakeholders, and other developers to identify risks that should be tested prior to the change/feature release. When forming the risk matrix, consider some of the following examples and how they may potentially impact the change: | Risk | Probability | Severity | Mitigation/Notes | |---------------------------|-------------|----------|-------------------------| | Multiple Spaces—unexpected behavior in non-default Kibana Space. | Low | High | Integration tests will verify that all features are still supported in non-default Kibana Space and when user switches between spaces. | | Multiple nodes—Elasticsearch polling might have race conditions when multiple Kibana nodes are polling for the same tasks. | High | Low | Tasks are idempotent, so executing them multiple times will not result in logical error, but will degrade performance. To test for this case we add plenty of unit tests around this logic and document manual testing procedure. | | Code should gracefully handle cases when feature X or plugin Y are disabled. | Medium | High | Unit tests will verify that any feature flag or plugin combination still results in our service operational. | | [See more potential risk examples](https://github.com/elastic/kibana/blob/main/RISK_MATRIX.mdx) | ### For maintainers - [ ] This was checked for breaking API changes and was [labeled appropriately](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process) --- .../rule_schema/common_attributes.gen.ts | 30 ++++++++++++ .../rule_schema/common_attributes.schema.yaml | 41 ++++++++++++++++ .../model/rule_schema/rule_schemas.gen.ts | 2 +- .../rule_schema/rule_schemas.schema.yaml | 4 +- .../query_attributes.gen.ts | 45 ----------------- .../query_attributes.schema.yaml | 48 ------------------- 6 files changed, 74 insertions(+), 96 deletions(-) delete mode 100644 x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/specific_attributes/query_attributes.gen.ts delete mode 100644 x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/specific_attributes/query_attributes.schema.yaml diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/common_attributes.gen.ts b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/common_attributes.gen.ts index 11298104fc84a..aff87d49829b9 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/common_attributes.gen.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/common_attributes.gen.ts @@ -438,3 +438,33 @@ export const AlertSuppressionDuration = z.object({ value: z.number().int().min(1), unit: z.enum(['s', 'm', 'h']), }); + +/** + * Describes how alerts will be generated for documents with missing suppress by fields: +doNotSuppress - per each document a separate alert will be created +suppress - only alert will be created per suppress by bucket + */ +export type AlertSuppressionMissingFieldsStrategy = z.infer< + typeof AlertSuppressionMissingFieldsStrategy +>; +export const AlertSuppressionMissingFieldsStrategy = z.enum(['doNotSuppress', 'suppress']); +export type AlertSuppressionMissingFieldsStrategyEnum = + typeof AlertSuppressionMissingFieldsStrategy.enum; +export const AlertSuppressionMissingFieldsStrategyEnum = AlertSuppressionMissingFieldsStrategy.enum; + +export type AlertSuppressionGroupBy = z.infer; +export const AlertSuppressionGroupBy = z.array(z.string()).min(1).max(3); + +export type AlertSuppression = z.infer; +export const AlertSuppression = z.object({ + group_by: AlertSuppressionGroupBy, + duration: AlertSuppressionDuration.optional(), + missing_fields_strategy: AlertSuppressionMissingFieldsStrategy.optional(), +}); + +export type AlertSuppressionCamel = z.infer; +export const AlertSuppressionCamel = z.object({ + groupBy: AlertSuppressionGroupBy, + duration: AlertSuppressionDuration.optional(), + missingFieldsStrategy: AlertSuppressionMissingFieldsStrategy.optional(), +}); diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/common_attributes.schema.yaml b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/common_attributes.schema.yaml index 35149d92f0c43..1387072bf99e8 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/common_attributes.schema.yaml +++ b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/common_attributes.schema.yaml @@ -487,3 +487,44 @@ components: required: - value - unit + + AlertSuppressionMissingFieldsStrategy: + type: string + enum: + - doNotSuppress + - suppress + description: |- + Describes how alerts will be generated for documents with missing suppress by fields: + doNotSuppress - per each document a separate alert will be created + suppress - only alert will be created per suppress by bucket + + AlertSuppressionGroupBy: + type: array + items: + type: string + minItems: 1 + maxItems: 3 + + AlertSuppression: + type: object + properties: + group_by: + $ref: '#/components/schemas/AlertSuppressionGroupBy' + duration: + $ref: '#/components/schemas/AlertSuppressionDuration' + missing_fields_strategy: + $ref: '#/components/schemas/AlertSuppressionMissingFieldsStrategy' + required: + - group_by + + AlertSuppressionCamel: + type: object + properties: + groupBy: + $ref: '#/components/schemas/AlertSuppressionGroupBy' + duration: + $ref: '#/components/schemas/AlertSuppressionDuration' + missingFieldsStrategy: + $ref: '#/components/schemas/AlertSuppressionMissingFieldsStrategy' + required: + - groupBy \ No newline at end of file diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.gen.ts b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.gen.ts index 0744b3fb57b5c..a6a9b1e5d511c 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.gen.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.gen.ts @@ -59,6 +59,7 @@ import { DataViewId, RuleFilterArray, SavedQueryId, + AlertSuppression, KqlQueryLanguage, } from './common_attributes.gen'; import { RuleExecutionSummary } from '../../rule_monitoring/model/execution_summary.gen'; @@ -68,7 +69,6 @@ import { TimestampField, } from './specific_attributes/eql_attributes.gen'; import { ResponseAction } from '../rule_response_actions/response_actions.gen'; -import { AlertSuppression } from './specific_attributes/query_attributes.gen'; import { Threshold, ThresholdAlertSuppression, diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.schema.yaml b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.schema.yaml index 67666aab5d95a..763982248a367 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.schema.yaml +++ b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.schema.yaml @@ -346,7 +346,7 @@ components: items: $ref: '../rule_response_actions/response_actions.schema.yaml#/components/schemas/ResponseAction' alert_suppression: - $ref: './specific_attributes/query_attributes.schema.yaml#/components/schemas/AlertSuppression' + $ref: './common_attributes.schema.yaml#/components/schemas/AlertSuppression' QueryRuleDefaultableFields: type: object @@ -427,7 +427,7 @@ components: items: $ref: '../rule_response_actions/response_actions.schema.yaml#/components/schemas/ResponseAction' alert_suppression: - $ref: './specific_attributes/query_attributes.schema.yaml#/components/schemas/AlertSuppression' + $ref: './common_attributes.schema.yaml#/components/schemas/AlertSuppression' query: $ref: './common_attributes.schema.yaml#/components/schemas/RuleQuery' diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/specific_attributes/query_attributes.gen.ts b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/specific_attributes/query_attributes.gen.ts deleted file mode 100644 index cca23efb4979a..0000000000000 --- a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/specific_attributes/query_attributes.gen.ts +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { z } from 'zod'; - -/* - * NOTICE: Do not edit this file manually. - * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator. - */ - -import { AlertSuppressionDuration } from '../common_attributes.gen'; - -/** - * Describes how alerts will be generated for documents with missing suppress by fields: -doNotSuppress - per each document a separate alert will be created -suppress - only alert will be created per suppress by bucket - */ -export type AlertSuppressionMissingFieldsStrategy = z.infer< - typeof AlertSuppressionMissingFieldsStrategy ->; -export const AlertSuppressionMissingFieldsStrategy = z.enum(['doNotSuppress', 'suppress']); -export type AlertSuppressionMissingFieldsStrategyEnum = - typeof AlertSuppressionMissingFieldsStrategy.enum; -export const AlertSuppressionMissingFieldsStrategyEnum = AlertSuppressionMissingFieldsStrategy.enum; - -export type AlertSuppressionGroupBy = z.infer; -export const AlertSuppressionGroupBy = z.array(z.string()).min(1).max(3); - -export type AlertSuppression = z.infer; -export const AlertSuppression = z.object({ - group_by: AlertSuppressionGroupBy, - duration: AlertSuppressionDuration.optional(), - missing_fields_strategy: AlertSuppressionMissingFieldsStrategy.optional(), -}); - -export type AlertSuppressionCamel = z.infer; -export const AlertSuppressionCamel = z.object({ - groupBy: AlertSuppressionGroupBy, - duration: AlertSuppressionDuration.optional(), - missingFieldsStrategy: AlertSuppressionMissingFieldsStrategy.optional(), -}); diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/specific_attributes/query_attributes.schema.yaml b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/specific_attributes/query_attributes.schema.yaml deleted file mode 100644 index e47f5ed3b6ab3..0000000000000 --- a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/specific_attributes/query_attributes.schema.yaml +++ /dev/null @@ -1,48 +0,0 @@ -openapi: 3.0.0 -info: - title: Query Rule Attributes - version: 'not applicable' -paths: {} -components: - x-codegen-enabled: true - schemas: - AlertSuppressionMissingFieldsStrategy: - type: string - enum: - - doNotSuppress - - suppress - description: |- - Describes how alerts will be generated for documents with missing suppress by fields: - doNotSuppress - per each document a separate alert will be created - suppress - only alert will be created per suppress by bucket - - AlertSuppressionGroupBy: - type: array - items: - type: string - minItems: 1 - maxItems: 3 - - AlertSuppression: - type: object - properties: - group_by: - $ref: '#/components/schemas/AlertSuppressionGroupBy' - duration: - $ref: '../common_attributes.schema.yaml#/components/schemas/AlertSuppressionDuration' - missing_fields_strategy: - $ref: '#/components/schemas/AlertSuppressionMissingFieldsStrategy' - required: - - group_by - - AlertSuppressionCamel: - type: object - properties: - groupBy: - $ref: '#/components/schemas/AlertSuppressionGroupBy' - duration: - $ref: '../common_attributes.schema.yaml#/components/schemas/AlertSuppressionDuration' - missingFieldsStrategy: - $ref: '#/components/schemas/AlertSuppressionMissingFieldsStrategy' - required: - - groupBy From 7b4b18278f54b9358ef4b29295b16834c864dec7 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com> Date: Thu, 4 Jan 2024 10:56:53 +0000 Subject: [PATCH 02/28] Suppression/add schema to im eql (#174244) ## Summary Summarize your PR. If it involves visual changes include a screenshot or gif. ### Checklist Delete any items that are not applicable to this PR. - [ ] Any text added follows [EUI's writing guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses sentence case text and includes [i18n support](https://github.com/elastic/kibana/blob/main/packages/kbn-i18n/README.md) - [ ] [Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html) was added for features that require explanation or tutorials - [ ] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios - [ ] [Flaky Test Runner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1) was used on any tests changed - [ ] Any UI touched in this PR is usable by keyboard only (learn more about [keyboard accessibility](https://webaim.org/techniques/keyboard/)) - [ ] Any UI touched in this PR does not create any new axe failures (run axe in browser: [FF](https://addons.mozilla.org/en-US/firefox/addon/axe-devtools/), [Chrome](https://chrome.google.com/webstore/detail/axe-web-accessibility-tes/lhdoppojpmngadmnindnejefpokejbdd?hl=en-US)) - [ ] If a plugin configuration key changed, check if it needs to be allowlisted in the cloud and added to the [docker list](https://github.com/elastic/kibana/blob/main/src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker) - [ ] This renders correctly on smaller devices using a responsive layout. (You can test this [in your browser](https://www.browserstack.com/guide/responsive-testing-on-local-server)) - [ ] This was checked for [cross-browser compatibility](https://www.elastic.co/support/matrix#matrix_browsers) ### Risk Matrix Delete this section if it is not applicable to this PR. Before closing this PR, invite QA, stakeholders, and other developers to identify risks that should be tested prior to the change/feature release. When forming the risk matrix, consider some of the following examples and how they may potentially impact the change: | Risk | Probability | Severity | Mitigation/Notes | |---------------------------|-------------|----------|-------------------------| | Multiple Spaces—unexpected behavior in non-default Kibana Space. | Low | High | Integration tests will verify that all features are still supported in non-default Kibana Space and when user switches between spaces. | | Multiple nodes—Elasticsearch polling might have race conditions when multiple Kibana nodes are polling for the same tasks. | High | Low | Tasks are idempotent, so executing them multiple times will not result in logical error, but will degrade performance. To test for this case we add plenty of unit tests around this logic and document manual testing procedure. | | Code should gracefully handle cases when feature X or plugin Y are disabled. | Medium | High | Unit tests will verify that any feature flag or plugin combination still results in our service operational. | | [See more potential risk examples](https://github.com/elastic/kibana/blob/main/RISK_MATRIX.mdx) | ### For maintainers - [ ] This was checked for breaking API changes and was [labeled appropriately](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process) --- .../common/api/detection_engine/model/rule_schema/index.ts | 1 - .../detection_engine/model/rule_schema/rule_schemas.gen.ts | 3 +++ .../model/rule_schema/rule_schemas.schema.yaml | 6 +++++- .../model/rule_schema_legacy/query_attributes.ts | 2 +- .../rule_management/normalization/rule_converters.ts | 2 ++ .../lib/detection_engine/rule_schema/model/rule_schemas.ts | 2 ++ 6 files changed, 13 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/index.ts b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/index.ts index 1b4d506a80648..5bb393c1fd419 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/index.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/index.ts @@ -11,6 +11,5 @@ export * from './rule_schemas.gen'; export * from './specific_attributes/eql_attributes.gen'; export * from './specific_attributes/ml_attributes.gen'; export * from './specific_attributes/new_terms_attributes.gen'; -export * from './specific_attributes/query_attributes.gen'; export * from './specific_attributes/threat_match_attributes.gen'; export * from './specific_attributes/threshold_attributes.gen'; diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.gen.ts b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.gen.ts index a6a9b1e5d511c..530169c1d1da8 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.gen.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.gen.ts @@ -58,6 +58,7 @@ import { IndexPatternArray, DataViewId, RuleFilterArray, + AlertSuppression, SavedQueryId, AlertSuppression, KqlQueryLanguage, @@ -215,6 +216,7 @@ export const EqlOptionalFields = z.object({ event_category_override: EventCategoryOverride.optional(), tiebreaker_field: TiebreakerField.optional(), timestamp_field: TimestampField.optional(), + alert_suppression: AlertSuppression.optional(), }); export type EqlRuleCreateFields = z.infer; @@ -414,6 +416,7 @@ export const ThreatMatchRuleOptionalFields = z.object({ threat_language: KqlQueryLanguage.optional(), concurrent_searches: ConcurrentSearches.optional(), items_per_search: ItemsPerSearch.optional(), + alert_suppression: AlertSuppression.optional(), }); export type ThreatMatchRuleDefaultableFields = z.infer; diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.schema.yaml b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.schema.yaml index 763982248a367..172b04c92e884 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.schema.yaml +++ b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.schema.yaml @@ -279,7 +279,9 @@ components: $ref: './specific_attributes/eql_attributes.schema.yaml#/components/schemas/TiebreakerField' timestamp_field: $ref: './specific_attributes/eql_attributes.schema.yaml#/components/schemas/TimestampField' - + alert_suppression: + $ref: './common_attributes.schema.yaml#/components/schemas/AlertSuppression' + EqlRuleCreateFields: allOf: - $ref: '#/components/schemas/EqlRequiredFields' @@ -604,6 +606,8 @@ components: $ref: './specific_attributes/threat_match_attributes.schema.yaml#/components/schemas/ConcurrentSearches' items_per_search: $ref: './specific_attributes/threat_match_attributes.schema.yaml#/components/schemas/ItemsPerSearch' + alert_suppression: + $ref: './common_attributes.schema.yaml#/components/schemas/AlertSuppression' ThreatMatchRuleDefaultableFields: type: object diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema_legacy/query_attributes.ts b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema_legacy/query_attributes.ts index f6090383a3ce3..8ee40e7a507d4 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema_legacy/query_attributes.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema_legacy/query_attributes.ts @@ -11,7 +11,7 @@ import { PositiveIntegerGreaterThanZero, enumeration, } from '@kbn/securitysolution-io-ts-types'; -import { AlertSuppressionMissingFieldsStrategyEnum } from '../rule_schema/specific_attributes/query_attributes.gen'; +import { AlertSuppressionMissingFieldsStrategyEnum } from '../rule_schema/common_attributes.gen'; export type AlertSuppressionMissingFields = t.TypeOf; export const AlertSuppressionMissingFields = enumeration( diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/normalization/rule_converters.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/normalization/rule_converters.ts index 2402e9fdcdf33..545caf1507167 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/normalization/rule_converters.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/normalization/rule_converters.ts @@ -556,6 +556,7 @@ export const typeSpecificCamelToSnake = ( timestamp_field: params.timestampField, event_category_override: params.eventCategoryOverride, tiebreaker_field: params.tiebreakerField, + alert_suppression: convertAlertSuppressionToSnake(params.alertSuppression), }; } case 'esql': { @@ -582,6 +583,7 @@ export const typeSpecificCamelToSnake = ( threat_indicator_path: params.threatIndicatorPath, concurrent_searches: params.concurrentSearches, items_per_search: params.itemsPerSearch, + alert_suppression: convertAlertSuppressionToSnake(params.alertSuppression), }; } case 'query': { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_schema/model/rule_schemas.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_schema/model/rule_schemas.ts index c3075aa24af5b..2151260292841 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_schema/model/rule_schemas.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_schema/model/rule_schemas.ts @@ -153,6 +153,7 @@ export const EqlSpecificRuleParams = z.object({ eventCategoryOverride: EventCategoryOverride.optional(), timestampField: TimestampField.optional(), tiebreakerField: TiebreakerField.optional(), + alertSuppression: AlertSuppressionCamel.optional(), }); export type EqlRuleParams = BaseRuleParams & EqlSpecificRuleParams; @@ -185,6 +186,7 @@ export const ThreatSpecificRuleParams = z.object({ concurrentSearches: ConcurrentSearches.optional(), itemsPerSearch: ItemsPerSearch.optional(), dataViewId: DataViewId.optional(), + alertSuppression: AlertSuppressionCamel.optional(), }); export type ThreatRuleParams = BaseRuleParams & ThreatSpecificRuleParams; From 4f3632320574a5e3a189439cab55cab024aff71b Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com> Date: Thu, 4 Jan 2024 11:06:36 +0000 Subject: [PATCH 03/28] Update rule_schemas.gen.ts --- .../api/detection_engine/model/rule_schema/rule_schemas.gen.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.gen.ts b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.gen.ts index 530169c1d1da8..d5f614393e0f1 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.gen.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.gen.ts @@ -60,7 +60,6 @@ import { RuleFilterArray, AlertSuppression, SavedQueryId, - AlertSuppression, KqlQueryLanguage, } from './common_attributes.gen'; import { RuleExecutionSummary } from '../../rule_monitoring/model/execution_summary.gen'; From 918ed288bdef062b9a987b4f98399d8ce6d91025 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com> Date: Thu, 4 Jan 2024 17:32:26 +0000 Subject: [PATCH 04/28] introduce feature flags (#174293) ## Summary Summarize your PR. If it involves visual changes include a screenshot or gif. ### Checklist Delete any items that are not applicable to this PR. - [ ] Any text added follows [EUI's writing guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses sentence case text and includes [i18n support](https://github.com/elastic/kibana/blob/main/packages/kbn-i18n/README.md) - [ ] [Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html) was added for features that require explanation or tutorials - [ ] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios - [ ] [Flaky Test Runner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1) was used on any tests changed - [ ] Any UI touched in this PR is usable by keyboard only (learn more about [keyboard accessibility](https://webaim.org/techniques/keyboard/)) - [ ] Any UI touched in this PR does not create any new axe failures (run axe in browser: [FF](https://addons.mozilla.org/en-US/firefox/addon/axe-devtools/), [Chrome](https://chrome.google.com/webstore/detail/axe-web-accessibility-tes/lhdoppojpmngadmnindnejefpokejbdd?hl=en-US)) - [ ] If a plugin configuration key changed, check if it needs to be allowlisted in the cloud and added to the [docker list](https://github.com/elastic/kibana/blob/main/src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker) - [ ] This renders correctly on smaller devices using a responsive layout. (You can test this [in your browser](https://www.browserstack.com/guide/responsive-testing-on-local-server)) - [ ] This was checked for [cross-browser compatibility](https://www.elastic.co/support/matrix#matrix_browsers) ### Risk Matrix Delete this section if it is not applicable to this PR. Before closing this PR, invite QA, stakeholders, and other developers to identify risks that should be tested prior to the change/feature release. When forming the risk matrix, consider some of the following examples and how they may potentially impact the change: | Risk | Probability | Severity | Mitigation/Notes | |---------------------------|-------------|----------|-------------------------| | Multiple Spaces—unexpected behavior in non-default Kibana Space. | Low | High | Integration tests will verify that all features are still supported in non-default Kibana Space and when user switches between spaces. | | Multiple nodes—Elasticsearch polling might have race conditions when multiple Kibana nodes are polling for the same tasks. | High | Low | Tasks are idempotent, so executing them multiple times will not result in logical error, but will degrade performance. To test for this case we add plenty of unit tests around this logic and document manual testing procedure. | | Code should gracefully handle cases when feature X or plugin Y are disabled. | Medium | High | Unit tests will verify that any feature flag or plugin combination still results in our service operational. | | [See more potential risk examples](https://github.com/elastic/kibana/blob/main/RISK_MATRIX.mdx) | ### For maintainers - [ ] This was checked for breaking API changes and was [labeled appropriately](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process) --- .../common/experimental_features.ts | 10 ++++++ .../pages/rule_creation/helpers.ts | 32 +++++++++++-------- .../rule_details/rule_definition_section.tsx | 22 +++++++++++-- .../rules/step_define_rule/index.tsx | 20 +++++++++--- .../rules/step_define_rule/schema.tsx | 6 +++- .../normalization/rule_converters.ts | 2 ++ .../prebuilt_rules_preview.cy.ts | 2 ++ 7 files changed, 72 insertions(+), 22 deletions(-) diff --git a/x-pack/plugins/security_solution/common/experimental_features.ts b/x-pack/plugins/security_solution/common/experimental_features.ts index 2b34532e4e848..cd55fcce145e3 100644 --- a/x-pack/plugins/security_solution/common/experimental_features.ts +++ b/x-pack/plugins/security_solution/common/experimental_features.ts @@ -137,6 +137,16 @@ export const allowedExperimentalValues = Object.freeze({ */ riskEnginePrivilegesRouteEnabled: true, + /** + * Enables alerts suppression for indicator match rules + */ + alertSuppressionForIndicatorMatchRuleEnabled: false, + + /** + * Enables alerts suppression for EQL rules + */ + alertSuppressionForEqlRuleEnabled: false, + /* * Enables experimental Entity Analytics Asset Criticality feature */ diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_creation/helpers.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_creation/helpers.ts index 983aba55c8165..fcdbf8b5fb1dd 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_creation/helpers.ts +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_creation/helpers.ts @@ -407,6 +407,21 @@ export const formatDefineStepData = (defineStepData: DefineStepRule): DefineStep }), }; + const alertSuppressionFields = + ruleFields.groupByFields.length > 0 + ? { + alert_suppression: { + group_by: ruleFields.groupByFields, + duration: + ruleFields.groupByRadioSelection === GroupByOptions.PerTimePeriod + ? ruleFields.groupByDuration + : undefined, + missing_fields_strategy: + ruleFields.suppressionMissingFields || DEFAULT_SUPPRESSION_MISSING_FIELDS_STRATEGY, + }, + } + : {}; + const typeFields = isMlFields(ruleFields) ? { anomaly_threshold: ruleFields.anomalyThreshold, @@ -451,6 +466,7 @@ export const formatDefineStepData = (defineStepData: DefineStepRule): DefineStep threat_filters: ruleFields.threatQueryBar?.filters, threat_mapping: ruleFields.threatMapping, threat_language: ruleFields.threatQueryBar?.query?.language, + ...alertSuppressionFields, } : isEqlFields(ruleFields) ? { @@ -462,6 +478,7 @@ export const formatDefineStepData = (defineStepData: DefineStepRule): DefineStep timestamp_field: ruleFields.eqlOptions?.timestampField, event_category_override: ruleFields.eqlOptions?.eventCategoryField, tiebreaker_field: ruleFields.eqlOptions?.tiebreakerField, + ...alertSuppressionFields, } : isNewTermsFields(ruleFields) ? { @@ -478,20 +495,7 @@ export const formatDefineStepData = (defineStepData: DefineStepRule): DefineStep query: ruleFields.queryBar?.query?.query as string, } : { - ...(ruleFields.groupByFields.length > 0 - ? { - alert_suppression: { - group_by: ruleFields.groupByFields, - duration: - ruleFields.groupByRadioSelection === GroupByOptions.PerTimePeriod - ? ruleFields.groupByDuration - : undefined, - missing_fields_strategy: - ruleFields.suppressionMissingFields || - DEFAULT_SUPPRESSION_MISSING_FIELDS_STRATEGY, - }, - } - : {}), + ...alertSuppressionFields, index: ruleFields.index, filters: ruleFields.queryBar?.filters, language: ruleFields.queryBar?.query?.language, diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_management/components/rule_details/rule_definition_section.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_management/components/rule_details/rule_definition_section.tsx index cb37caa661fda..c137ddc6aa8c3 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_management/components/rule_details/rule_definition_section.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_management/components/rule_details/rule_definition_section.tsx @@ -54,6 +54,7 @@ import { TechnicalPreviewBadge } from '../../../../detections/components/rules/t import { BadgeList } from './badge_list'; import { DEFAULT_DESCRIPTION_LIST_COLUMN_WIDTHS } from './constants'; import * as i18n from './translations'; +import { useIsExperimentalFeatureEnabled } from '../../../../common/hooks/use_experimental_features'; import type { ExperimentalFeatures } from '../../../../../common/experimental_features'; interface SavedQueryNameProps { @@ -426,7 +427,10 @@ const prepareDefinitionSectionListItems = ( rule: Partial, isInteractive: boolean, savedQuery: SavedQuery | undefined, - experimentalFeatures?: Partial + { + alertSuppressionForEqlRuleEnabled, + alertSuppressionForIndicatorMatchRuleEnabled, + }: Partial = {} ): EuiDescriptionListProps['listItems'] => { const definitionSectionListItems: EuiDescriptionListProps['listItems'] = []; @@ -656,7 +660,11 @@ const prepareDefinitionSectionListItems = ( }); } - if ('alert_suppression' in rule && rule.alert_suppression) { + const isSuppressionEnabled = + (rule.type === 'eql' && alertSuppressionForEqlRuleEnabled) || + (rule.type === 'threat_match' && alertSuppressionForIndicatorMatchRuleEnabled); + + if ('alert_suppression' in rule && rule.alert_suppression && isSuppressionEnabled) { if ('group_by' in rule.alert_suppression) { definitionSectionListItems.push({ title: ( @@ -738,10 +746,18 @@ export const RuleDefinitionSection = ({ ruleType: rule.type, }); + const alertSuppressionForIndicatorMatchRuleEnabled = useIsExperimentalFeatureEnabled( + 'alertSuppressionForIndicatorMatchRuleEnabled' + ); + const alertSuppressionForEqlRuleEnabled = useIsExperimentalFeatureEnabled( + 'alertSuppressionForEqlRuleEnabled' + ); + const definitionSectionListItems = prepareDefinitionSectionListItems( rule, isInteractive, - savedQuery + savedQuery, + { alertSuppressionForIndicatorMatchRuleEnabled, alertSuppressionForEqlRuleEnabled } ); return ( diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx index 809fb97eb260c..0618c35d106b3 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx @@ -84,6 +84,7 @@ import { AlertSuppressionMissingFieldsStrategyEnum } from '../../../../../common import { DurationInput } from '../duration_input'; import { MINIMUM_LICENSE_FOR_SUPPRESSION } from '../../../../../common/detection_engine/constants'; import { useUpsellingMessage } from '../../../../common/hooks/use_upselling'; +import { useIsExperimentalFeatureEnabled } from '../../../../common/hooks/use_experimental_features'; const CommonUseField = getUseField({ component: Field }); @@ -181,6 +182,13 @@ const StepDefineRuleComponent: FC = ({ const esqlQueryRef = useRef(undefined); + const isAlertSuppressionForIndicatorMatchRuleEnabled = useIsExperimentalFeatureEnabled( + 'alertSuppressionForIndicatorMatchRuleEnabled' + ); + const isAlertSuppressionForEqlRuleEnabled = useIsExperimentalFeatureEnabled( + 'alertSuppressionForEqlRuleEnabled' + ); + const isAlertSuppressionLicenseValid = license.isAtLeast(MINIMUM_LICENSE_FOR_SUPPRESSION); const isThresholdRule = getIsThresholdRule(ruleType); @@ -804,7 +812,11 @@ const StepDefineRuleComponent: FC = ({ [isUpdateView, mlCapabilities] ); - const isAlertSuppressionEnabled = isQueryRule(ruleType) || isThresholdRule; + const isAlertSuppressionEnabled = + isQueryRule(ruleType) || + isThresholdRule || + (isAlertSuppressionForIndicatorMatchRuleEnabled && isThreatMatchRule(ruleType)) || + (isAlertSuppressionForEqlRuleEnabled && isEqlRule(ruleType)); return ( <> @@ -994,7 +1006,7 @@ const StepDefineRuleComponent: FC = ({ = ({ diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/schema.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/schema.tsx index 0d22c459705a9..c2441d4b3ca01 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/schema.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/schema.tsx @@ -17,6 +17,7 @@ import { customValidators, } from '../../../../common/components/threat_match/helpers'; import { + isEqlRule, isEsqlRule, isNewTermsRule, isQueryRule, @@ -603,7 +604,10 @@ export const schema: FormSchema = { ...args: Parameters ): ReturnType> | undefined => { const [{ formData }] = args; - const needsValidation = isQueryRule(formData.ruleType); + const needsValidation = + isQueryRule(formData.ruleType) || + isEqlRule(formData.ruleType) || + isThreatMatchRule(formData.ruleType); if (!needsValidation) { return; } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/normalization/rule_converters.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/normalization/rule_converters.ts index 545caf1507167..596b81dcbeca2 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/normalization/rule_converters.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/normalization/rule_converters.ts @@ -113,6 +113,7 @@ export const typeSpecificSnakeToCamel = ( timestampField: params.timestamp_field, eventCategoryOverride: params.event_category_override, tiebreakerField: params.tiebreaker_field, + alertSuppression: convertAlertSuppressionToCamel(params.alert_suppression), }; } case 'esql': { @@ -139,6 +140,7 @@ export const typeSpecificSnakeToCamel = ( threatIndicatorPath: params.threat_indicator_path ?? DEFAULT_INDICATOR_SOURCE_PATH, concurrentSearches: params.concurrent_searches, itemsPerSearch: params.items_per_search, + alertSuppression: convertAlertSuppressionToCamel(params.alert_suppression), }; } case 'query': { diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/prebuilt_rules_preview.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/prebuilt_rules_preview.cy.ts index a8349a0410c63..8f0d972f6cabf 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/prebuilt_rules_preview.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/prebuilt_rules_preview.cy.ts @@ -240,6 +240,7 @@ describe('Detection rules, Prebuilt Rules Installation and Update workflow', () query: 'process where process.name == "regsvr32.exe"', index: ['winlogbeat-*', 'logs-endpoint.events.*'], filters, + alert_suppression: undefined, }); const THREAT_MATCH_INDEX_PATTERN_RULE = createRuleAssetSavedObject({ @@ -280,6 +281,7 @@ describe('Detection rules, Prebuilt Rules Installation and Update workflow', () ], threat_language: 'kuery', threat_indicator_path: 'threat.indicator', + alert_suppression: undefined, }); const NEW_TERMS_INDEX_PATTERN_RULE = createRuleAssetSavedObject({ From 2dfa47c562bcc3f0531ef987b3a42d1b50ab1788 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com> Date: Fri, 5 Jan 2024 10:16:12 +0000 Subject: [PATCH 05/28] Update rule_definition_section.tsx --- .../components/rule_details/rule_definition_section.tsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_management/components/rule_details/rule_definition_section.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_management/components/rule_details/rule_definition_section.tsx index c137ddc6aa8c3..289e1cdb4d78e 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_management/components/rule_details/rule_definition_section.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_management/components/rule_details/rule_definition_section.tsx @@ -662,7 +662,8 @@ const prepareDefinitionSectionListItems = ( const isSuppressionEnabled = (rule.type === 'eql' && alertSuppressionForEqlRuleEnabled) || - (rule.type === 'threat_match' && alertSuppressionForIndicatorMatchRuleEnabled); + (rule.type === 'threat_match' && alertSuppressionForIndicatorMatchRuleEnabled) || + (rule.type && (['query', 'saved_query', 'threshold'] as Type[]).includes(rule.type)); if ('alert_suppression' in rule && rule.alert_suppression && isSuppressionEnabled) { if ('group_by' in rule.alert_suppression) { From 483f190adc89b1fab0bdc728ee2a080efa8fe0c8 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com> Date: Mon, 8 Jan 2024 14:08:21 +0000 Subject: [PATCH 06/28] [Security Solution][Detection Engine][Feature branch] adds Cypress tests fro IM suppression (#174344) ## Summary Adds tests by test plan **Scenario: Create rule with per rule execution suppression** Given rule create page When user select rule type And user adds suppress fields And user selects on rule execution suppression only And user saves rule Then on rule details page suppress by fields should be displayed And suppression on rule execution should be rendered **Scenario: Create rule with time interval suppression** Given rule create page When user select rule type And user adds suppress fields And user selects time interval suppression option And user selects do not suppress missing fields And user saves rule Then on rule details page suppress by fields should be displayed And time interval suppression should be displayed **Scenario: Edit rule with suppression** Given rule configured with suppression on rule interval When user edits rule And changes suppression from time interval to rule execution suppression Then on rule details page suppress by fields should be displayed And per rule execution suppression should be displayed **Scenario: Edit rule without suppression** Given rule without suppression When user edits rule And user adds suppress fields And user selects time interval suppression option And user saves rule Then on rule details page suppress by fields should be displayed And time interval suppression should be displayed **Scenario: Rule details with suppression** Given rule configured with suppression time interval When user views rule details page Then on rule details page suppress by fields should be displayed And time interval suppression should be displayed ### **License tests** **Scenario: Create rule with rule execution suppression on basic license (ESS only)** Given rule create page When user select rule type Then user sees suppression options disabled And upselling message displayed **Scenario: Create rule with rule execution suppression on Essentials tier (Serverless only)** Given rule create page When user select rule type And user adds suppress fields And user selects on rule execution suppression only And user saves rule Then on rule details page suppress by fields should be displayed And suppression on rule execution should be rendered **Scenario: Rule details with suppression on basic license (ESS only)** Given rule configured with suppression time interval When user views rule details page Then on rule details page suppress by fields should be displayed And time interval suppression should be displayed And upselling warning that suppression is not applied should be rendered on details labels --- ...rt_suppression_technical_preview_badge.tsx | 8 +- .../rules/description_step/index.test.tsx | 8 +- .../rules/description_step/index.tsx | 19 ++- .../test/security_solution_cypress/config.ts | 1 + .../indicator_match_rule_suppression.cy.ts | 135 ++++++++++++++++ ...tor_match_rule_suppression_ess_basic.cy.ts | 92 +++++++++++ ...le_suppression_serverless_essentials.cy.ts | 77 +++++++++ .../rule_edit/indicator_match_rule.cy.ts | 149 ++++++++++++++++++ .../cypress/objects/rule.ts | 1 + .../cypress/screens/create_new_rule.ts | 13 +- .../cypress/screens/rule_details.ts | 3 + .../cypress/tasks/create_new_rule.ts | 41 ++++- .../threat_indicator/mappings.json | 6 - .../serverless_config.ts | 3 + 14 files changed, 535 insertions(+), 21 deletions(-) create mode 100644 x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/indicator_match_rule_suppression.cy.ts create mode 100644 x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/indicator_match_rule_suppression_ess_basic.cy.ts create mode 100644 x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/indicator_match_rule_suppression_serverless_essentials.cy.ts create mode 100644 x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_edit/indicator_match_rule.cy.ts diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/description_step/alert_suppression_technical_preview_badge.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/description_step/alert_suppression_technical_preview_badge.tsx index 739fdfa6f5d1c..fcdbb6afb3301 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/description_step/alert_suppression_technical_preview_badge.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/description_step/alert_suppression_technical_preview_badge.tsx @@ -23,7 +23,13 @@ export const AlertSuppressionTechnicalPreviewBadge = ({ label }: TechnicalPrevie {alertSuppressionUpsellingMessage && ( - + )} diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/description_step/index.test.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/description_step/index.test.tsx index 113f81631a06a..19f2efa0a42b9 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/description_step/index.test.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/description_step/index.test.tsx @@ -537,13 +537,7 @@ describe('description_step', () => { }); describe('alert suppression', () => { - const ruleTypesWithoutSuppression: Type[] = [ - 'eql', - 'esql', - 'machine_learning', - 'new_terms', - 'threat_match', - ]; + const ruleTypesWithoutSuppression: Type[] = ['esql', 'machine_learning', 'new_terms']; const suppressionFields = { groupByDuration: { unit: 'm', diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/description_step/index.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/description_step/index.tsx index 883c01430e644..e78c004cb5c76 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/description_step/index.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/description_step/index.tsx @@ -56,7 +56,12 @@ import { THREAT_QUERY_LABEL } from './translations'; import { filterEmptyThreats } from '../../../../detection_engine/rule_creation_ui/pages/rule_creation/helpers'; import { useLicense } from '../../../../common/hooks/use_license'; import type { LicenseService } from '../../../../../common/license'; -import { isThresholdRule, isQueryRule } from '../../../../../common/detection_engine/utils'; +import { + isThresholdRule, + isQueryRule, + isThreatMatchRule, + isEqlRule, +} from '../../../../../common/detection_engine/utils'; const DescriptionListContainer = styled(EuiDescriptionList)` max-width: 600px; @@ -206,7 +211,8 @@ export const getDescriptionItem = ( return []; } else if (field === 'groupByFields') { const ruleType: Type = get('ruleType', data); - const ruleCanHaveGroupByFields = isQueryRule(ruleType); + const ruleCanHaveGroupByFields = + isQueryRule(ruleType) || isThreatMatchRule(ruleType) || isEqlRule(ruleType); if (!ruleCanHaveGroupByFields) { return []; } @@ -216,7 +222,11 @@ export const getDescriptionItem = ( return []; } else if (field === 'groupByDuration') { const ruleType: Type = get('ruleType', data); - const ruleCanHaveDuration = isQueryRule(ruleType) || isThresholdRule(ruleType); + const ruleCanHaveDuration = + isQueryRule(ruleType) || + isThresholdRule(ruleType) || + isThreatMatchRule(ruleType) || + isEqlRule(ruleType); if (!ruleCanHaveDuration) { return []; } @@ -239,7 +249,8 @@ export const getDescriptionItem = ( } } else if (field === 'suppressionMissingFields') { const ruleType: Type = get('ruleType', data); - const ruleCanHaveSuppressionMissingFields = isQueryRule(ruleType); + const ruleCanHaveSuppressionMissingFields = + isQueryRule(ruleType) || isThreatMatchRule(ruleType) || isEqlRule(ruleType); if (!ruleCanHaveSuppressionMissingFields) { return []; } diff --git a/x-pack/test/security_solution_cypress/config.ts b/x-pack/test/security_solution_cypress/config.ts index fb34362f7fb9b..d8207d622bb47 100644 --- a/x-pack/test/security_solution_cypress/config.ts +++ b/x-pack/test/security_solution_cypress/config.ts @@ -46,6 +46,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) { '--xpack.ruleRegistry.unsafe.legacyMultiTenancy.enabled=true', `--xpack.securitySolution.enableExperimental=${JSON.stringify([ 'chartEmbeddablesEnabled', + 'alertSuppressionForIndicatorMatchRuleEnabled', ])}`, // mock cloud to enable the guided onboarding tour in e2e tests '--xpack.cloud.id=test', diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/indicator_match_rule_suppression.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/indicator_match_rule_suppression.cy.ts new file mode 100644 index 0000000000000..4c95996841663 --- /dev/null +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/indicator_match_rule_suppression.cy.ts @@ -0,0 +1,135 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { getNewThreatIndicatorRule } from '../../../../objects/rule'; + +import { + DEFINITION_DETAILS, + SUPPRESS_FOR_DETAILS, + SUPPRESS_BY_DETAILS, + SUPPRESS_MISSING_FIELD, + DETAILS_TITLE, +} from '../../../../screens/rule_details'; + +import { + fillDefineIndicatorMatchRule, + fillAlertSuppressionFields, + selectIndicatorMatchType, + fillAboutRuleMinimumAndContinue, + createRuleWithoutEnabling, + skipScheduleRuleAction, + continueFromDefineStep, + selectAlertSuppressionPerInterval, + setAlertSuppressionDuration, + selectDoNotSuppressForMissingFields, +} from '../../../../tasks/create_new_rule'; + +import { login } from '../../../../tasks/login'; +import { visit } from '../../../../tasks/navigation'; +import { getDetails } from '../../../../tasks/rule_details'; +import { CREATE_RULE_URL } from '../../../../urls/navigation'; +import { deleteAlertsAndRules } from '../../../../tasks/api_calls/common'; + +const SUPPRESS_BY_FIELDS = ['myhash.mysha256', 'source.ip.keyword']; + +describe( + 'Detection rules, Indicator Match, Alert Suppression', + { + tags: ['@ess', '@serverless'], + env: { + ftrConfig: { + kbnServerArgs: [ + `--xpack.securitySolution.enableExperimental=${JSON.stringify([ + 'alertSuppressionForIndicatorMatchRuleEnabled', + ])}`, + ], + }, + }, + }, + () => { + const rule = getNewThreatIndicatorRule(); + beforeEach(() => { + cy.task('esArchiverLoad', { archiveName: 'threat_indicator' }); + cy.task('esArchiverLoad', { archiveName: 'suspicious_source_event' }); + deleteAlertsAndRules(); + login(); + visit(CREATE_RULE_URL); + }); + + it('creates rule with per rule execution suppression', () => { + selectIndicatorMatchType(); + fillDefineIndicatorMatchRule(rule); + + // selecting only suppression fields, the rest options would be default + fillAlertSuppressionFields(SUPPRESS_BY_FIELDS); + continueFromDefineStep(); + + // ensures details preview works correctly + cy.get(DEFINITION_DETAILS).within(() => { + getDetails(SUPPRESS_BY_DETAILS).should('have.text', SUPPRESS_BY_FIELDS.join('')); + getDetails(SUPPRESS_FOR_DETAILS).should('have.text', 'One rule execution'); + getDetails(SUPPRESS_MISSING_FIELD).should( + 'have.text', + 'Suppress and group alerts for events with missing fields' + ); + + // suppression functionality should be under Tech Preview + cy.contains(DETAILS_TITLE, SUPPRESS_FOR_DETAILS).contains('Technical Preview'); + }); + + fillAboutRuleMinimumAndContinue(rule); + skipScheduleRuleAction(); + createRuleWithoutEnabling(); + + cy.get(DEFINITION_DETAILS).within(() => { + getDetails(SUPPRESS_BY_DETAILS).should('have.text', SUPPRESS_BY_FIELDS.join('')); + getDetails(SUPPRESS_FOR_DETAILS).should('have.text', 'One rule execution'); + getDetails(SUPPRESS_MISSING_FIELD).should( + 'have.text', + 'Suppress and group alerts for events with missing fields' + ); + }); + }); + + it('creates rule rule with time interval suppression', () => { + const expectedSuppressByFields = SUPPRESS_BY_FIELDS.slice(0, 1); + + selectIndicatorMatchType(); + fillDefineIndicatorMatchRule(rule); + + // fill suppress by fields and select non-default suppression options + fillAlertSuppressionFields(expectedSuppressByFields); + selectAlertSuppressionPerInterval(); + setAlertSuppressionDuration(45, 'm'); + selectDoNotSuppressForMissingFields(); + continueFromDefineStep(); + + // ensures details preview works correctly + cy.get(DEFINITION_DETAILS).within(() => { + getDetails(SUPPRESS_BY_DETAILS).should('have.text', expectedSuppressByFields.join('')); + getDetails(SUPPRESS_FOR_DETAILS).should('have.text', '45m'); + getDetails(SUPPRESS_MISSING_FIELD).should( + 'have.text', + 'Do not suppress alerts for events with missing fields' + ); + }); + + fillAboutRuleMinimumAndContinue(rule); + skipScheduleRuleAction(); + createRuleWithoutEnabling(); + + cy.get(DEFINITION_DETAILS).within(() => { + getDetails(SUPPRESS_BY_DETAILS).should('have.text', expectedSuppressByFields.join('')); + getDetails(SUPPRESS_FOR_DETAILS).should('have.text', '45m'); + getDetails(SUPPRESS_MISSING_FIELD).should( + 'have.text', + 'Do not suppress alerts for events with missing fields' + ); + }); + }); + } +); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/indicator_match_rule_suppression_ess_basic.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/indicator_match_rule_suppression_ess_basic.cy.ts new file mode 100644 index 0000000000000..f3d724ec49a11 --- /dev/null +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/indicator_match_rule_suppression_ess_basic.cy.ts @@ -0,0 +1,92 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { getNewThreatIndicatorRule } from '../../../../objects/rule'; + +import { + ALERT_SUPPRESSION_FIELDS_INPUT, + ALERT_SUPPRESSION_FIELDS, +} from '../../../../screens/create_new_rule'; +import { + SUPPRESS_FOR_DETAILS, + DETAILS_TITLE, + SUPPRESS_BY_DETAILS, + SUPPRESS_MISSING_FIELD, + DEFINITION_DETAILS, + ALERT_SUPPRESSION_INSUFFICIENT_LICENSING_ICON, +} from '../../../../screens/rule_details'; + +import { selectIndicatorMatchType } from '../../../../tasks/create_new_rule'; +import { startBasicLicense } from '../../../../tasks/api_calls/licensing'; +import { createRule } from '../../../../tasks/api_calls/rules'; +import { login } from '../../../../tasks/login'; +import { visit } from '../../../../tasks/navigation'; +import { CREATE_RULE_URL } from '../../../../urls/navigation'; +import { TOOLTIP } from '../../../../screens/common'; +import { ruleDetailsUrl } from '../../../../urls/rule_details'; +import { getDetails } from '../../../../tasks/rule_details'; + +const SUPPRESS_BY_FIELDS = ['myhash.mysha256', 'source.ip.keyword']; + +describe( + 'Detection rules, Indicator Match, Alert Suppression', + { + tags: ['@ess'], + }, + () => { + describe('Create rule form', () => { + beforeEach(() => { + login(); + visit(CREATE_RULE_URL); + startBasicLicense(); + }); + + it('can not create rule with rule execution suppression on basic license', () => { + selectIndicatorMatchType(); + + cy.get(ALERT_SUPPRESSION_FIELDS_INPUT).should('be.disabled'); + cy.get(ALERT_SUPPRESSION_FIELDS).trigger('mouseover'); + + // Platinum license is required, tooltip on disabled alert suppression checkbox should tell this + cy.get(TOOLTIP).contains('Platinum license'); + }); + + it('shows upselling message on rule details with suppression on basic license', () => { + const rule = getNewThreatIndicatorRule(); + + createRule({ + ...rule, + alert_suppression: { + group_by: SUPPRESS_BY_FIELDS, + duration: { value: 360, unit: 's' }, + missing_fields_strategy: 'doNotSuppress', + }, + }).then((createdRule) => { + visit(ruleDetailsUrl(createdRule.body.id)); + + cy.get(DEFINITION_DETAILS).within(() => { + getDetails(SUPPRESS_BY_DETAILS).should('have.text', SUPPRESS_BY_FIELDS.join('')); + getDetails(SUPPRESS_FOR_DETAILS).should('have.text', '360s'); + getDetails(SUPPRESS_MISSING_FIELD).should( + 'have.text', + 'Do not suppress alerts for events with missing fields' + ); + + // suppression functionality should be under Tech Preview + cy.contains(DETAILS_TITLE, SUPPRESS_FOR_DETAILS).contains('Technical Preview'); + }); + + // Platinum license is required for configuration to apply + cy.get(ALERT_SUPPRESSION_INSUFFICIENT_LICENSING_ICON).eq(2).trigger('mouseover'); + cy.get(TOOLTIP).contains( + 'Alert suppression is configured but will not be applied due to insufficient licensing' + ); + }); + }); + }); + } +); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/indicator_match_rule_suppression_serverless_essentials.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/indicator_match_rule_suppression_serverless_essentials.cy.ts new file mode 100644 index 0000000000000..f5c845f344289 --- /dev/null +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/indicator_match_rule_suppression_serverless_essentials.cy.ts @@ -0,0 +1,77 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { getNewThreatIndicatorRule } from '../../../../objects/rule'; + +import { DEFINITION_DETAILS, SUPPRESS_BY_DETAILS } from '../../../../screens/rule_details'; + +import { + fillDefineIndicatorMatchRule, + fillAlertSuppressionFields, + selectIndicatorMatchType, + fillAboutRuleMinimumAndContinue, + createRuleWithoutEnabling, + skipScheduleRuleAction, + continueFromDefineStep, +} from '../../../../tasks/create_new_rule'; + +import { login } from '../../../../tasks/login'; +import { visit } from '../../../../tasks/navigation'; +import { getDetails } from '../../../../tasks/rule_details'; +import { CREATE_RULE_URL } from '../../../../urls/navigation'; +import { deleteAlertsAndRules } from '../../../../tasks/api_calls/common'; + +const SUPPRESS_BY_FIELDS = ['myhash.mysha256', 'source.ip.keyword']; + +describe( + 'Detection rules, Indicator Match, Alert Suppression', + { + tags: ['@serverless'], + env: { + ftrConfig: { + productTypes: [ + { product_line: 'security', product_tier: 'essentials' }, + { product_line: 'endpoint', product_tier: 'essentials' }, + ], + + kbnServerArgs: [ + `--xpack.securitySolution.enableExperimental=${JSON.stringify([ + 'alertSuppressionForIndicatorMatchRuleEnabled', + ])}`, + ], + }, + }, + }, + () => { + const rule = getNewThreatIndicatorRule(); + beforeEach(() => { + cy.task('esArchiverLoad', { archiveName: 'threat_indicator' }); + cy.task('esArchiverLoad', { archiveName: 'suspicious_source_event' }); + deleteAlertsAndRules(); + login(); + }); + + it('creates rule with per rule execution suppression for essentials license', () => { + login(); + visit(CREATE_RULE_URL); + selectIndicatorMatchType(); + fillDefineIndicatorMatchRule(rule); + + // selecting only suppression fields, the rest options would be default + fillAlertSuppressionFields(SUPPRESS_BY_FIELDS); + continueFromDefineStep(); + + fillAboutRuleMinimumAndContinue(rule); + skipScheduleRuleAction(); + createRuleWithoutEnabling(); + + cy.get(DEFINITION_DETAILS).within(() => { + getDetails(SUPPRESS_BY_DETAILS).should('have.text', SUPPRESS_BY_FIELDS.join('')); + }); + }); + } +); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_edit/indicator_match_rule.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_edit/indicator_match_rule.cy.ts new file mode 100644 index 0000000000000..60bda5d07df86 --- /dev/null +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_edit/indicator_match_rule.cy.ts @@ -0,0 +1,149 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { getNewThreatIndicatorRule } from '../../../../objects/rule'; + +import { + SUPPRESS_FOR_DETAILS, + DETAILS_TITLE, + SUPPRESS_BY_DETAILS, + SUPPRESS_MISSING_FIELD, + DEFINITION_DETAILS, +} from '../../../../screens/rule_details'; + +import { + ALERT_SUPPRESSION_DURATION_INPUT, + ALERT_SUPPRESSION_FIELDS, + ALERT_SUPPRESSION_MISSING_FIELDS_DO_NOT_SUPPRESS, +} from '../../../../screens/create_new_rule'; + +import { createRule } from '../../../../tasks/api_calls/rules'; + +import { RULES_MANAGEMENT_URL } from '../../../../urls/rules_management'; +import { getDetails } from '../../../../tasks/rule_details'; +import { deleteAlertsAndRules } from '../../../../tasks/api_calls/common'; +import { login } from '../../../../tasks/login'; + +import { editFirstRule, goToRuleDetailsOf } from '../../../../tasks/alerts_detection_rules'; + +import { saveEditedRule } from '../../../../tasks/edit_rule'; +import { goToRuleEditSettings } from '../../../../tasks/rule_details'; +import { + fillAlertSuppressionFields, + selectAlertSuppressionPerRuleExecution, + selectAlertSuppressionPerInterval, + setAlertSuppressionDuration, +} from '../../../../tasks/create_new_rule'; +import { visit } from '../../../../tasks/navigation'; + +const SUPPRESS_BY_FIELDS = ['myhash.mysha256', 'source.ip.keyword']; + +const rule = getNewThreatIndicatorRule(); + +describe( + 'Detection rules, Indicator Match, Edit', + { + tags: ['@ess', '@serverless'], + env: { + ftrConfig: { + kbnServerArgs: [ + `--xpack.securitySolution.enableExperimental=${JSON.stringify([ + 'alertSuppressionForIndicatorMatchRuleEnabled', + ])}`, + ], + }, + }, + }, + () => { + beforeEach(() => { + cy.task('esArchiverLoad', { archiveName: 'threat_indicator' }); + cy.task('esArchiverLoad', { archiveName: 'suspicious_source_event' }); + login(); + deleteAlertsAndRules(); + }); + describe('without suppression', () => { + beforeEach(() => { + createRule(rule); + }); + + it('enables suppression on time interval', () => { + visit(RULES_MANAGEMENT_URL); + editFirstRule(); + + fillAlertSuppressionFields(SUPPRESS_BY_FIELDS); + selectAlertSuppressionPerInterval(); + setAlertSuppressionDuration(2, 'h'); + + saveEditedRule(); + + cy.get(DEFINITION_DETAILS).within(() => { + getDetails(SUPPRESS_BY_DETAILS).should('have.text', SUPPRESS_BY_FIELDS.join('')); + getDetails(SUPPRESS_FOR_DETAILS).should('have.text', '2h'); + getDetails(SUPPRESS_MISSING_FIELD).should( + 'have.text', + 'Suppress and group alerts for events with missing fields' + ); + + // suppression functionality should be under Tech Preview + cy.contains(DETAILS_TITLE, SUPPRESS_FOR_DETAILS).contains('Technical Preview'); + }); + }); + }); + + describe('with suppression configured', () => { + beforeEach(() => { + createRule({ + ...rule, + alert_suppression: { + group_by: SUPPRESS_BY_FIELDS, + duration: { value: 360, unit: 's' }, + missing_fields_strategy: 'doNotSuppress', + }, + }); + }); + + it('displays suppress options correctly on edit form and allows its editing', () => { + visit(RULES_MANAGEMENT_URL); + goToRuleDetailsOf(rule.name); + + cy.get(DEFINITION_DETAILS).within(() => { + getDetails(SUPPRESS_BY_DETAILS).should('have.text', SUPPRESS_BY_FIELDS.join('')); + getDetails(SUPPRESS_FOR_DETAILS).should('have.text', '360s'); + getDetails(SUPPRESS_MISSING_FIELD).should( + 'have.text', + 'Do not suppress alerts for events with missing fields' + ); + }); + + goToRuleEditSettings(); + + // check if suppress settings display correctly + cy.get(ALERT_SUPPRESSION_FIELDS).should('contain', SUPPRESS_BY_FIELDS.join('')); + + cy.get(ALERT_SUPPRESSION_DURATION_INPUT) + .eq(0) + .should('be.enabled') + .should('have.value', 360); + cy.get(ALERT_SUPPRESSION_DURATION_INPUT) + .eq(1) + .should('be.enabled') + .should('have.value', 's'); + + cy.get(ALERT_SUPPRESSION_MISSING_FIELDS_DO_NOT_SUPPRESS).should('be.checked'); + + selectAlertSuppressionPerRuleExecution(); + + saveEditedRule(); + + // check execution duration has changed + cy.get(DEFINITION_DETAILS).within(() => { + getDetails(SUPPRESS_FOR_DETAILS).should('have.text', 'One rule execution'); + }); + }); + }); + } +); diff --git a/x-pack/test/security_solution_cypress/cypress/objects/rule.ts b/x-pack/test/security_solution_cypress/cypress/objects/rule.ts index 681eff67d071e..8dde3456fd7b6 100644 --- a/x-pack/test/security_solution_cypress/cypress/objects/rule.ts +++ b/x-pack/test/security_solution_cypress/cypress/objects/rule.ts @@ -477,6 +477,7 @@ export const getNewThreatIndicatorRule = ( description: 'The threat indicator rule description.', query: '*:*', threat_query: '*:*', + threat_language: 'kuery', index: ['suspicious-*'], severity: 'critical', risk_score: 20, diff --git a/x-pack/test/security_solution_cypress/cypress/screens/create_new_rule.ts b/x-pack/test/security_solution_cypress/cypress/screens/create_new_rule.ts index c7fbe341eb700..0b521f68a787a 100644 --- a/x-pack/test/security_solution_cypress/cypress/screens/create_new_rule.ts +++ b/x-pack/test/security_solution_cypress/cypress/screens/create_new_rule.ts @@ -19,9 +19,13 @@ export const ADD_FALSE_POSITIVE_BTN = export const ADD_REFERENCE_URL_BTN = '[data-test-subj="detectionEngineStepAboutRuleReferenceUrls"] .euiButtonEmpty__text'; -export const ALERT_SUPPRESSION_FIELDS = +export const ALERT_SUPPRESSION_FIELDS = '[data-test-subj="alertSuppressionInput"]'; + +export const ALERT_SUPPRESSION_FIELDS_COMBO_BOX = '[data-test-subj="alertSuppressionInput"] [data-test-subj="comboBoxInput"]'; +export const ALERT_SUPPRESSION_FIELDS_INPUT = `${ALERT_SUPPRESSION_FIELDS_COMBO_BOX} input`; + export const ALERT_SUPPRESSION_DURATION_OPTIONS = '[data-test-subj="alertSuppressionDuration"] [data-test-subj="groupByDurationOptions"]'; @@ -29,6 +33,13 @@ export const ALERT_SUPPRESSION_DURATION_PER_TIME_INTERVAL = `${ALERT_SUPPRESSION export const ALERT_SUPPRESSION_DURATION_PER_RULE_EXECUTION = `${ALERT_SUPPRESSION_DURATION_OPTIONS} #per-rule-execution`; +export const ALERT_SUPPRESSION_MISSING_FIELDS_OPTIONS = + '[data-test-subj="suppressionMissingFieldsOptions"]'; + +export const ALERT_SUPPRESSION_MISSING_FIELDS_SUPPRESS = `${ALERT_SUPPRESSION_MISSING_FIELDS_OPTIONS} #suppress`; + +export const ALERT_SUPPRESSION_MISSING_FIELDS_DO_NOT_SUPPRESS = `${ALERT_SUPPRESSION_MISSING_FIELDS_OPTIONS} #doNotSuppress`; + export const ALERT_SUPPRESSION_DURATION_INPUT = '[data-test-subj="alertSuppressionDuration"] [data-test-subj="alertSuppressionDurationInput"]'; diff --git a/x-pack/test/security_solution_cypress/cypress/screens/rule_details.ts b/x-pack/test/security_solution_cypress/cypress/screens/rule_details.ts index f627ae6546182..b8c524b0084ce 100644 --- a/x-pack/test/security_solution_cypress/cypress/screens/rule_details.ts +++ b/x-pack/test/security_solution_cypress/cypress/screens/rule_details.ts @@ -148,5 +148,8 @@ export const THREAT_TECHNIQUE = '[data-test-subj="threatTechniqueLink"]'; export const THREAT_SUBTECHNIQUE = '[data-test-subj="threatSubtechniqueLink"]'; +export const ALERT_SUPPRESSION_INSUFFICIENT_LICENSING_ICON = + '[data-test-subj="alertSuppressionInsufficientLicensingIcon"]'; + export const HIGHLIGHTED_ROWS_IN_TABLE = '[data-test-subj="euiDataGridBody"] .alertsTableHighlightedRow'; diff --git a/x-pack/test/security_solution_cypress/cypress/tasks/create_new_rule.ts b/x-pack/test/security_solution_cypress/cypress/tasks/create_new_rule.ts index 1089834384374..8875e7b682dd0 100644 --- a/x-pack/test/security_solution_cypress/cypress/tasks/create_new_rule.ts +++ b/x-pack/test/security_solution_cypress/cypress/tasks/create_new_rule.ts @@ -32,6 +32,8 @@ import { convertHistoryStartToSize, getHumanizedDuration } from '../helpers/rule import { ABOUT_CONTINUE_BTN, ALERT_SUPPRESSION_DURATION_INPUT, + ALERT_SUPPRESSION_FIELDS_COMBO_BOX, + ALERT_SUPPRESSION_MISSING_FIELDS_DO_NOT_SUPPRESS, THRESHOLD_ENABLE_SUPPRESSION_CHECKBOX, ALERT_SUPPRESSION_DURATION_PER_RULE_EXECUTION, ALERT_SUPPRESSION_DURATION_PER_TIME_INTERVAL, @@ -686,6 +688,11 @@ export const getCustomQueryInvalidationText = () => cy.contains(CUSTOM_QUERY_REQ * @param rule The rule to use to fill in everything */ export const fillDefineIndicatorMatchRuleAndContinue = (rule: ThreatMatchRuleCreateProps) => { + fillDefineIndicatorMatchRule(rule); + continueFromDefineStep(); +}; + +export const fillDefineIndicatorMatchRule = (rule: ThreatMatchRuleCreateProps) => { if (rule.index) { fillIndexAndIndicatorIndexPattern(rule.index, rule.threat_index); } @@ -694,6 +701,12 @@ export const fillDefineIndicatorMatchRuleAndContinue = (rule: ThreatMatchRuleCre indicatorIndexField: rule.threat_mapping[0].entries[0].value, }); getCustomIndicatorQueryInput().type('{selectall}{enter}*:*'); +}; + +/** + * presses continue on form Define step + */ +export const continueFromDefineStep = () => { getDefineContinueButton().should('exist').click({ force: true }); }; @@ -797,14 +810,38 @@ export const enablesAndPopulatesThresholdSuppression = ( cy.get(THRESHOLD_ENABLE_SUPPRESSION_CHECKBOX).should('not.be.checked'); cy.get(THRESHOLD_ENABLE_SUPPRESSION_CHECKBOX).siblings('label').click(); - cy.get(ALERT_SUPPRESSION_DURATION_INPUT).first().type(`{selectall}${interval}`); - cy.get(ALERT_SUPPRESSION_DURATION_INPUT).eq(1).select(timeUnit); + setAlertSuppressionDuration(interval, timeUnit); // rule execution radio option is disabled, per time interval becomes enabled when suppression enabled cy.get(ALERT_SUPPRESSION_DURATION_PER_RULE_EXECUTION).should('be.disabled'); cy.get(ALERT_SUPPRESSION_DURATION_PER_TIME_INTERVAL).should('be.enabled').should('be.checked'); }; +export const fillAlertSuppressionFields = (fields: string[]) => { + fields.forEach((field) => { + cy.get(ALERT_SUPPRESSION_FIELDS_COMBO_BOX).type(`${field}{enter}`); + }); +}; + +export const selectAlertSuppressionPerInterval = () => { + // checkbox is covered by label, force:true is a workaround + // click on label not working, likely because it has child components + cy.get(ALERT_SUPPRESSION_DURATION_PER_TIME_INTERVAL).click({ force: true }); +}; + +export const selectAlertSuppressionPerRuleExecution = () => { + cy.get(ALERT_SUPPRESSION_DURATION_PER_RULE_EXECUTION).siblings('label').click(); +}; + +export const selectDoNotSuppressForMissingFields = () => { + cy.get(ALERT_SUPPRESSION_MISSING_FIELDS_DO_NOT_SUPPRESS).siblings('label').click(); +}; + +export const setAlertSuppressionDuration = (interval: number, timeUnit: 's' | 'm' | 'h') => { + cy.get(ALERT_SUPPRESSION_DURATION_INPUT).first().type(`{selectall}${interval}`); + cy.get(ALERT_SUPPRESSION_DURATION_INPUT).eq(1).select(timeUnit); +}; + export const checkLoadQueryDynamically = () => { cy.get(LOAD_QUERY_DYNAMICALLY_CHECKBOX).click({ force: true }); cy.get(LOAD_QUERY_DYNAMICALLY_CHECKBOX).should('be.checked'); diff --git a/x-pack/test/security_solution_cypress/es_archives/threat_indicator/mappings.json b/x-pack/test/security_solution_cypress/es_archives/threat_indicator/mappings.json index bc5f6e3db9169..6d4274a7a0977 100644 --- a/x-pack/test/security_solution_cypress/es_archives/threat_indicator/mappings.json +++ b/x-pack/test/security_solution_cypress/es_archives/threat_indicator/mappings.json @@ -811,18 +811,12 @@ }, "settings": { "index": { - "lifecycle": { - "name": "filebeat", - "rollover_alias": "filebeat-7.12.0" - }, "mapping": { "total_fields": { "limit": "10000" } }, "max_docvalue_fields_search": "200", - "number_of_replicas": "1", - "number_of_shards": "1", "refresh_interval": "5s" } } diff --git a/x-pack/test/security_solution_cypress/serverless_config.ts b/x-pack/test/security_solution_cypress/serverless_config.ts index d0ee1613f6e4c..f147c973d535b 100644 --- a/x-pack/test/security_solution_cypress/serverless_config.ts +++ b/x-pack/test/security_solution_cypress/serverless_config.ts @@ -34,6 +34,9 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) { { product_line: 'endpoint', product_tier: 'complete' }, { product_line: 'cloud', product_tier: 'complete' }, ])}`, + `--xpack.securitySolution.enableExperimental=${JSON.stringify([ + 'alertSuppressionForIndicatorMatchRuleEnabled', + ])}`, ], }, testRunner: SecuritySolutionConfigurableCypressTestRunner, From 8861a8b8f5a4cfc472d4257182c55fcd3e57c8c6 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com> Date: Wed, 10 Jan 2024 17:44:04 +0000 Subject: [PATCH 07/28] Update mappings.json --- .../es_archives/threat_indicator/mappings.json | 2 ++ 1 file changed, 2 insertions(+) diff --git a/x-pack/test/security_solution_cypress/es_archives/threat_indicator/mappings.json b/x-pack/test/security_solution_cypress/es_archives/threat_indicator/mappings.json index 6d4274a7a0977..e22f719255fe5 100644 --- a/x-pack/test/security_solution_cypress/es_archives/threat_indicator/mappings.json +++ b/x-pack/test/security_solution_cypress/es_archives/threat_indicator/mappings.json @@ -817,6 +817,8 @@ } }, "max_docvalue_fields_search": "200", + "number_of_replicas": "1", + "number_of_shards": "1", "refresh_interval": "5s" } } From 8a14d4f1b6b625a28264aa53f7555136dc55d80c Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com> Date: Fri, 12 Jan 2024 10:39:51 +0000 Subject: [PATCH 08/28] add use-callback (#174702) ## Summary Summarize your PR. If it involves visual changes include a screenshot or gif. ### Checklist Delete any items that are not applicable to this PR. - [ ] Any text added follows [EUI's writing guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses sentence case text and includes [i18n support](https://github.com/elastic/kibana/blob/main/packages/kbn-i18n/README.md) - [ ] [Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html) was added for features that require explanation or tutorials - [ ] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios - [ ] [Flaky Test Runner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1) was used on any tests changed - [ ] Any UI touched in this PR is usable by keyboard only (learn more about [keyboard accessibility](https://webaim.org/techniques/keyboard/)) - [ ] Any UI touched in this PR does not create any new axe failures (run axe in browser: [FF](https://addons.mozilla.org/en-US/firefox/addon/axe-devtools/), [Chrome](https://chrome.google.com/webstore/detail/axe-web-accessibility-tes/lhdoppojpmngadmnindnejefpokejbdd?hl=en-US)) - [ ] If a plugin configuration key changed, check if it needs to be allowlisted in the cloud and added to the [docker list](https://github.com/elastic/kibana/blob/main/src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker) - [ ] This renders correctly on smaller devices using a responsive layout. (You can test this [in your browser](https://www.browserstack.com/guide/responsive-testing-on-local-server)) - [ ] This was checked for [cross-browser compatibility](https://www.elastic.co/support/matrix#matrix_browsers) ### Risk Matrix Delete this section if it is not applicable to this PR. Before closing this PR, invite QA, stakeholders, and other developers to identify risks that should be tested prior to the change/feature release. When forming the risk matrix, consider some of the following examples and how they may potentially impact the change: | Risk | Probability | Severity | Mitigation/Notes | |---------------------------|-------------|----------|-------------------------| | Multiple Spaces—unexpected behavior in non-default Kibana Space. | Low | High | Integration tests will verify that all features are still supported in non-default Kibana Space and when user switches between spaces. | | Multiple nodes—Elasticsearch polling might have race conditions when multiple Kibana nodes are polling for the same tasks. | High | Low | Tasks are idempotent, so executing them multiple times will not result in logical error, but will degrade performance. To test for this case we add plenty of unit tests around this logic and document manual testing procedure. | | Code should gracefully handle cases when feature X or plugin Y are disabled. | Medium | High | Unit tests will verify that any feature flag or plugin combination still results in our service operational. | | [See more potential risk examples](https://github.com/elastic/kibana/blob/main/RISK_MATRIX.mdx) | ### For maintainers - [ ] This was checked for breaking API changes and was [labeled appropriately](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process) --- .../rule_schema/rule_request_schema.mock.ts | 12 +++ .../rule_schema/rule_request_schema.test.ts | 76 +++++++++++++++++++ .../components/step_define_rule/index.tsx | 4 +- ...e_experimental_feature_fields_transform.ts | 51 +++++++++++++ .../pages/rule_creation/index.tsx | 8 +- .../normalization/rule_converters.test.ts | 40 ++++++++++ .../normalization/rule_converters.ts | 4 + 7 files changed, 192 insertions(+), 3 deletions(-) create mode 100644 x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/use_experimental_feature_fields_transform.ts diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_request_schema.mock.ts b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_request_schema.mock.ts index 968199a9459a6..e42f2aff39060 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_request_schema.mock.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_request_schema.mock.ts @@ -12,6 +12,7 @@ import type { QueryRuleCreateProps, QueryRuleUpdateProps, EsqlRuleCreateProps, + EqlRuleCreateProps, SavedQueryRuleCreateProps, ThreatMatchRuleCreateProps, ThresholdRuleCreateProps, @@ -133,6 +134,17 @@ export const getCreateThresholdRulesSchemaMock = (ruleId = 'rule-1'): ThresholdR }, }); +export const getCreateEqlRuleSchemaMock = (ruleId = 'rule-1'): EqlRuleCreateProps => ({ + description: 'Event correlation index pattern rule', + name: 'Event correlation index pattern rule', + severity: 'high', + risk_score: 55, + rule_id: ruleId, + type: 'eql', + language: 'eql', + query: 'process where process.name == "regsvr32.exe"', +}); + export const getCreateNewTermsRulesSchemaMock = ( ruleId = 'rule-1', enabled = false diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_request_schema.test.ts b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_request_schema.test.ts index 961bc915b6010..2b587d706ad61 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_request_schema.test.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_request_schema.test.ts @@ -15,6 +15,7 @@ import { getCreateSavedQueryRulesSchemaMock, getCreateThreatMatchRulesSchemaMock, getCreateThresholdRulesSchemaMock, + getCreateEqlRuleSchemaMock, } from './rule_request_schema.mock'; import type { SavedQueryRuleCreateProps } from './rule_schemas.gen'; import { RuleCreateProps } from './rule_schemas.gen'; @@ -1260,5 +1261,80 @@ describe('rules schema', () => { `"alert_suppression.duration: Required"` ); }); + // behaviour common for multiple rule types + const cases = [ + { ruleType: 'eql', ruleMock: getCreateEqlRuleSchemaMock() }, + { ruleType: 'threat_match', ruleMock: getCreateThreatMatchRulesSchemaMock() }, + ]; + + cases.forEach(({ ruleType, ruleMock }) => { + test(`should validate suppression fields for "${ruleType}" rule type`, () => { + const payload = { + ...ruleMock, + alert_suppression: { + group_by: ['agent.name'], + duration: { value: 5, unit: 'm' }, + missing_fields_strategy: 'suppress', + }, + }; + + const result = RuleCreateProps.safeParse(payload); + expectParseSuccess(result); + expect(result.data).toEqual(payload); + }); + + test(`should throw error if suppression fields not valid for "${ruleType}" rule`, () => { + const payload = { + ...ruleMock, + alert_suppression: { + group_by: 'not an array', + missing_fields_strategy: 'suppress', + }, + }; + + const result = RuleCreateProps.safeParse(payload); + expectParseError(result); + expect(stringifyZodError(result.error)).toEqual( + 'alert_suppression.group_by: Expected array, received string' + ); + }); + + test(`should throw error if suppression required field is missing for "${ruleType}" rule`, () => { + const payload = { + ...ruleMock, + alert_suppression: { + duration: { value: 5, unit: 'm' }, + missing_fields_strategy: 'suppress', + }, + }; + + const result = RuleCreateProps.safeParse(payload); + expectParseError(result); + expect(stringifyZodError(result.error)).toEqual('alert_suppression.group_by: Required'); + }); + + test(`should drop fields that are not in suppression schema for "${ruleType}" rule`, () => { + const payload = { + ...ruleMock, + alert_suppression: { + group_by: ['agent.name'], + duration: { value: 5, unit: 'm' }, + missing_fields_strategy: 'suppress', + random_field: 1, + }, + }; + + const result = RuleCreateProps.safeParse(payload); + expectParseSuccess(result); + expect(result.data).toEqual({ + ...ruleMock, + alert_suppression: { + group_by: ['agent.name'], + duration: { value: 5, unit: 'm' }, + missing_fields_strategy: 'suppress', + }, + }); + }); + }); }); }); diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.tsx index 47d127d7c6648..8de4657efe9f3 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.tsx @@ -63,6 +63,7 @@ import { Field, Form, getUseField, UseField, UseMultiFields } from '../../../../ import type { FormHook } from '../../../../shared_imports'; import { schema } from './schema'; import { getTermsAggregationFields } from './utils'; +import { useExperimentalFeatureFieldsTransform } from './use_experimental_feature_fields_transform'; import * as i18n from './translations'; import { isEqlRule, @@ -1089,13 +1090,14 @@ const StepDefineRuleReadOnlyComponent: FC = ({ indexPattern, }) => { const dataForDescription: Partial = getStepDataDataSource(data); + const transformFields = useExperimentalFeatureFieldsTransform(); return ( diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/use_experimental_feature_fields_transform.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/use_experimental_feature_fields_transform.ts new file mode 100644 index 0000000000000..399b39de7a71e --- /dev/null +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/use_experimental_feature_fields_transform.ts @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { useCallback } from 'react'; +import type { DefineStepRule } from '../../../../detections/pages/detection_engine/rules/types'; +import { useIsExperimentalFeatureEnabled } from '../../../../common/hooks/use_experimental_features'; +import { isEqlRule, isThreatMatchRule } from '../../../../../common/detection_engine/utils'; + +/** + * transforms DefineStepRule fields according to experimental feature flags + */ +export const useExperimentalFeatureFieldsTransform = >(): (( + fields: T +) => T) => { + const isAlertSuppressionForIndicatorMatchRuleEnabled = useIsExperimentalFeatureEnabled( + 'alertSuppressionForIndicatorMatchRuleEnabled' + ); + const isAlertSuppressionForEqlRuleEnabled = useIsExperimentalFeatureEnabled( + 'alertSuppressionForEqlRuleEnabled' + ); + + const transformer = useCallback( + (fields: T) => { + const isEqlSuppressionDisabled = isEqlRule(fields.ruleType) + ? !isAlertSuppressionForEqlRuleEnabled + : false; + const isIndicatorMatchSuppressionDisabled = isThreatMatchRule(fields.ruleType) + ? !isAlertSuppressionForIndicatorMatchRuleEnabled + : false; + // reset any alert suppression values hidden behind feature flag + if (isEqlSuppressionDisabled || isIndicatorMatchSuppressionDisabled) { + return { + ...fields, + groupByFields: [], + groupByRadioSelection: undefined, + groupByDuration: undefined, + suppressionMissingFields: undefined, + }; + } + + return fields; + }, + [isAlertSuppressionForEqlRuleEnabled, isAlertSuppressionForIndicatorMatchRuleEnabled] + ); + + return transformer; +}; diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_creation/index.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_creation/index.tsx index 3700b09c0067a..2922e26da6b4a 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_creation/index.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_creation/index.tsx @@ -42,6 +42,7 @@ import { SpyRoute } from '../../../../common/utils/route/spy_routes'; import { useUserData } from '../../../../detections/components/user_info'; import { AccordionTitle } from '../../components/accordion_title'; import { StepDefineRule, StepDefineRuleReadOnly } from '../../components/step_define_rule'; +import { useExperimentalFeatureFieldsTransform } from '../../components/step_define_rule/use_experimental_feature_fields_transform'; import { StepAboutRule, StepAboutRuleReadOnly } from '../../components/step_about_rule'; import { StepScheduleRule, StepScheduleRuleReadOnly } from '../../components/step_schedule_rule'; import { @@ -219,6 +220,8 @@ const CreateRulePageComponent: React.FC = () => { [defineStepData.index, esqlIndex, isEsqlRuleValue] ); + const defineFieldsTransform = useExperimentalFeatureFieldsTransform(); + const isPreviewDisabled = getIsRulePreviewDisabled({ ruleType, isQueryBarValid, @@ -355,10 +358,10 @@ const CreateRulePageComponent: React.FC = () => { const valid = await validateStep(step); if (valid) { - const localDefineStepData: DefineStepRule = { + const localDefineStepData: DefineStepRule = defineFieldsTransform({ ...defineStepForm.getFormData(), eqlOptions: eqlOptionsSelected, - }; + }); const localAboutStepData = aboutStepForm.getFormData(); const localScheduleStepData = scheduleStepForm.getFormData(); const localActionsStepData = actionsStepForm.getFormData(); @@ -403,6 +406,7 @@ const CreateRulePageComponent: React.FC = () => { navigateToApp, ruleType, startMlJobs, + defineFieldsTransform, ] ); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/normalization/rule_converters.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/normalization/rule_converters.test.ts index 29a67dc1da16b..fc7852c506a7b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/normalization/rule_converters.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/normalization/rule_converters.test.ts @@ -175,6 +175,46 @@ describe('rule_converters', () => { ); }); + test('should accept eql alerts suppression params', () => { + const patchParams = { + alert_suppression: { + duration: { value: 4, unit: 'h' as const }, + group_by: ['agent.name'], + missing_fields_strategy: 'doNotSuppress' as const, + }, + }; + const rule = getEqlRuleParams(); + const patchedParams = patchTypeSpecificSnakeToCamel(patchParams, rule); + expect(patchedParams).toEqual( + expect.objectContaining({ + alertSuppression: { + duration: { value: 4, unit: 'h' }, + groupBy: ['agent.name'], + missingFieldsStrategy: 'doNotSuppress', + }, + }) + ); + }); + + test('should accept threat_match alerts suppression params', () => { + const patchParams = { + alert_suppression: { + group_by: ['agent.name'], + missing_fields_strategy: 'suppress' as const, + }, + }; + const rule = getThreatRuleParams(); + const patchedParams = patchTypeSpecificSnakeToCamel(patchParams, rule); + expect(patchedParams).toEqual( + expect.objectContaining({ + alertSuppression: { + groupBy: ['agent.name'], + missingFieldsStrategy: 'suppress', + }, + }) + ); + }); + test('should accept machine learning params when existing rule type is machine learning', () => { const patchParams = { anomaly_threshold: 5, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/normalization/rule_converters.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/normalization/rule_converters.ts index 596b81dcbeca2..5e41a055f1acd 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/normalization/rule_converters.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/normalization/rule_converters.ts @@ -223,6 +223,8 @@ const patchEqlParams = ( timestampField: params.timestamp_field ?? existingRule.timestampField, eventCategoryOverride: params.event_category_override ?? existingRule.eventCategoryOverride, tiebreakerField: params.tiebreaker_field ?? existingRule.tiebreakerField, + alertSuppression: + convertAlertSuppressionToCamel(params.alert_suppression) ?? existingRule.alertSuppression, }; }; @@ -257,6 +259,8 @@ const patchThreatMatchParams = ( threatIndicatorPath: params.threat_indicator_path ?? existingRule.threatIndicatorPath, concurrentSearches: params.concurrent_searches ?? existingRule.concurrentSearches, itemsPerSearch: params.items_per_search ?? existingRule.itemsPerSearch, + alertSuppression: + convertAlertSuppressionToCamel(params.alert_suppression) ?? existingRule.alertSuppression, }; }; From 0def0da1441ceff46d0c7fe77f95b267f3fdddae Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com> Date: Tue, 16 Jan 2024 17:19:36 +0000 Subject: [PATCH 09/28] Update indicator_match_rule.cy.ts --- .../detection_engine/rule_edit/indicator_match_rule.cy.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_edit/indicator_match_rule.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_edit/indicator_match_rule.cy.ts index 60bda5d07df86..71e072f5c3af2 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_edit/indicator_match_rule.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_edit/indicator_match_rule.cy.ts @@ -121,9 +121,7 @@ describe( goToRuleEditSettings(); - // check if suppress settings display correctly - cy.get(ALERT_SUPPRESSION_FIELDS).should('contain', SUPPRESS_BY_FIELDS.join('')); - + // check saved suppression settings cy.get(ALERT_SUPPRESSION_DURATION_INPUT) .eq(0) .should('be.enabled') @@ -133,8 +131,11 @@ describe( .should('be.enabled') .should('have.value', 's'); + cy.get(ALERT_SUPPRESSION_FIELDS).should('contain', SUPPRESS_BY_FIELDS.join('')); cy.get(ALERT_SUPPRESSION_MISSING_FIELDS_DO_NOT_SUPPRESS).should('be.checked'); + // set new duration first to overcome some flaky racing conditions during form save + setAlertSuppressionDuration(2, 'h'); selectAlertSuppressionPerRuleExecution(); saveEditedRule(); From e6c2518405be671186a2785ded330ec53a3a7796 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com> Date: Wed, 17 Jan 2024 14:28:48 +0000 Subject: [PATCH 10/28] remove eql from suppression PR (#175001) ## Summary Summarize your PR. If it involves visual changes include a screenshot or gif. ### Checklist Delete any items that are not applicable to this PR. - [ ] Any text added follows [EUI's writing guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses sentence case text and includes [i18n support](https://github.com/elastic/kibana/blob/main/packages/kbn-i18n/README.md) - [ ] [Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html) was added for features that require explanation or tutorials - [ ] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios - [ ] [Flaky Test Runner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1) was used on any tests changed - [ ] Any UI touched in this PR is usable by keyboard only (learn more about [keyboard accessibility](https://webaim.org/techniques/keyboard/)) - [ ] Any UI touched in this PR does not create any new axe failures (run axe in browser: [FF](https://addons.mozilla.org/en-US/firefox/addon/axe-devtools/), [Chrome](https://chrome.google.com/webstore/detail/axe-web-accessibility-tes/lhdoppojpmngadmnindnejefpokejbdd?hl=en-US)) - [ ] If a plugin configuration key changed, check if it needs to be allowlisted in the cloud and added to the [docker list](https://github.com/elastic/kibana/blob/main/src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker) - [ ] This renders correctly on smaller devices using a responsive layout. (You can test this [in your browser](https://www.browserstack.com/guide/responsive-testing-on-local-server)) - [ ] This was checked for [cross-browser compatibility](https://www.elastic.co/support/matrix#matrix_browsers) ### Risk Matrix Delete this section if it is not applicable to this PR. Before closing this PR, invite QA, stakeholders, and other developers to identify risks that should be tested prior to the change/feature release. When forming the risk matrix, consider some of the following examples and how they may potentially impact the change: | Risk | Probability | Severity | Mitigation/Notes | |---------------------------|-------------|----------|-------------------------| | Multiple Spaces—unexpected behavior in non-default Kibana Space. | Low | High | Integration tests will verify that all features are still supported in non-default Kibana Space and when user switches between spaces. | | Multiple nodes—Elasticsearch polling might have race conditions when multiple Kibana nodes are polling for the same tasks. | High | Low | Tasks are idempotent, so executing them multiple times will not result in logical error, but will degrade performance. To test for this case we add plenty of unit tests around this logic and document manual testing procedure. | | Code should gracefully handle cases when feature X or plugin Y are disabled. | Medium | High | Unit tests will verify that any feature flag or plugin combination still results in our service operational. | | [See more potential risk examples](https://github.com/elastic/kibana/blob/main/RISK_MATRIX.mdx) | ### For maintainers - [ ] This was checked for breaking API changes and was [labeled appropriately](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process) --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- .../rule_schema/rule_request_schema.mock.ts | 12 ----------- .../rule_schema/rule_request_schema.test.ts | 6 +----- .../model/rule_schema/rule_schemas.gen.ts | 3 +-- .../rule_schema/rule_schemas.schema.yaml | 2 -- .../common/experimental_features.ts | 5 ----- .../description_step/index.test.tsx | 2 +- .../components/description_step/index.tsx | 10 +++------ .../components/step_define_rule/index.tsx | 6 +----- ...e_experimental_feature_fields_transform.ts | 12 +++-------- .../pages/rule_creation/helpers.ts | 1 - .../rule_details/rule_definition_section.tsx | 11 ++-------- .../normalization/rule_converters.test.ts | 21 ------------------- .../normalization/rule_converters.ts | 4 ---- .../rule_schema/model/rule_schemas.ts | 1 - .../prebuilt_rules_preview.cy.ts | 1 - 15 files changed, 12 insertions(+), 85 deletions(-) diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_request_schema.mock.ts b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_request_schema.mock.ts index e42f2aff39060..968199a9459a6 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_request_schema.mock.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_request_schema.mock.ts @@ -12,7 +12,6 @@ import type { QueryRuleCreateProps, QueryRuleUpdateProps, EsqlRuleCreateProps, - EqlRuleCreateProps, SavedQueryRuleCreateProps, ThreatMatchRuleCreateProps, ThresholdRuleCreateProps, @@ -134,17 +133,6 @@ export const getCreateThresholdRulesSchemaMock = (ruleId = 'rule-1'): ThresholdR }, }); -export const getCreateEqlRuleSchemaMock = (ruleId = 'rule-1'): EqlRuleCreateProps => ({ - description: 'Event correlation index pattern rule', - name: 'Event correlation index pattern rule', - severity: 'high', - risk_score: 55, - rule_id: ruleId, - type: 'eql', - language: 'eql', - query: 'process where process.name == "regsvr32.exe"', -}); - export const getCreateNewTermsRulesSchemaMock = ( ruleId = 'rule-1', enabled = false diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_request_schema.test.ts b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_request_schema.test.ts index 2b587d706ad61..d99beea8acc90 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_request_schema.test.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_request_schema.test.ts @@ -15,7 +15,6 @@ import { getCreateSavedQueryRulesSchemaMock, getCreateThreatMatchRulesSchemaMock, getCreateThresholdRulesSchemaMock, - getCreateEqlRuleSchemaMock, } from './rule_request_schema.mock'; import type { SavedQueryRuleCreateProps } from './rule_schemas.gen'; import { RuleCreateProps } from './rule_schemas.gen'; @@ -1262,10 +1261,7 @@ describe('rules schema', () => { ); }); // behaviour common for multiple rule types - const cases = [ - { ruleType: 'eql', ruleMock: getCreateEqlRuleSchemaMock() }, - { ruleType: 'threat_match', ruleMock: getCreateThreatMatchRulesSchemaMock() }, - ]; + const cases = [{ ruleType: 'threat_match', ruleMock: getCreateThreatMatchRulesSchemaMock() }]; cases.forEach(({ ruleType, ruleMock }) => { test(`should validate suppression fields for "${ruleType}" rule type`, () => { diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.gen.ts b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.gen.ts index d5f614393e0f1..e10ec2c5ef354 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.gen.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.gen.ts @@ -58,8 +58,8 @@ import { IndexPatternArray, DataViewId, RuleFilterArray, - AlertSuppression, SavedQueryId, + AlertSuppression, KqlQueryLanguage, } from './common_attributes.gen'; import { RuleExecutionSummary } from '../../rule_monitoring/model/execution_summary.gen'; @@ -215,7 +215,6 @@ export const EqlOptionalFields = z.object({ event_category_override: EventCategoryOverride.optional(), tiebreaker_field: TiebreakerField.optional(), timestamp_field: TimestampField.optional(), - alert_suppression: AlertSuppression.optional(), }); export type EqlRuleCreateFields = z.infer; diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.schema.yaml b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.schema.yaml index 172b04c92e884..22308557c3aaa 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.schema.yaml +++ b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.schema.yaml @@ -279,8 +279,6 @@ components: $ref: './specific_attributes/eql_attributes.schema.yaml#/components/schemas/TiebreakerField' timestamp_field: $ref: './specific_attributes/eql_attributes.schema.yaml#/components/schemas/TimestampField' - alert_suppression: - $ref: './common_attributes.schema.yaml#/components/schemas/AlertSuppression' EqlRuleCreateFields: allOf: diff --git a/x-pack/plugins/security_solution/common/experimental_features.ts b/x-pack/plugins/security_solution/common/experimental_features.ts index 79ed0cb6fca73..6f94f428bf1ab 100644 --- a/x-pack/plugins/security_solution/common/experimental_features.ts +++ b/x-pack/plugins/security_solution/common/experimental_features.ts @@ -148,11 +148,6 @@ export const allowedExperimentalValues = Object.freeze({ */ alertSuppressionForIndicatorMatchRuleEnabled: false, - /** - * Enables alerts suppression for EQL rules - */ - alertSuppressionForEqlRuleEnabled: false, - /* * Enables experimental Entity Analytics Asset Criticality feature */ diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/description_step/index.test.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/description_step/index.test.tsx index da49f8d6b8707..ccc9767d742f7 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/description_step/index.test.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/description_step/index.test.tsx @@ -537,7 +537,7 @@ describe('description_step', () => { }); describe('alert suppression', () => { - const ruleTypesWithoutSuppression: Type[] = ['esql', 'machine_learning', 'new_terms']; + const ruleTypesWithoutSuppression: Type[] = ['eql', 'esql', 'machine_learning', 'new_terms']; const suppressionFields = { groupByDuration: { unit: 'm', diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/description_step/index.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/description_step/index.tsx index 9eae5297e4fc0..a599f86ad8d78 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/description_step/index.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/description_step/index.tsx @@ -60,7 +60,6 @@ import { isThresholdRule, isQueryRule, isThreatMatchRule, - isEqlRule, } from '../../../../../common/detection_engine/utils'; const DescriptionListContainer = styled(EuiDescriptionList)` @@ -211,8 +210,7 @@ export const getDescriptionItem = ( return []; } else if (field === 'groupByFields') { const ruleType: Type = get('ruleType', data); - const ruleCanHaveGroupByFields = - isQueryRule(ruleType) || isThreatMatchRule(ruleType) || isEqlRule(ruleType); + const ruleCanHaveGroupByFields = isQueryRule(ruleType) || isThreatMatchRule(ruleType); if (!ruleCanHaveGroupByFields) { return []; } @@ -225,8 +223,7 @@ export const getDescriptionItem = ( const ruleCanHaveDuration = isQueryRule(ruleType) || isThresholdRule(ruleType) || - isThreatMatchRule(ruleType) || - isEqlRule(ruleType); + isThreatMatchRule(ruleType); if (!ruleCanHaveDuration) { return []; } @@ -249,8 +246,7 @@ export const getDescriptionItem = ( } } else if (field === 'suppressionMissingFields') { const ruleType: Type = get('ruleType', data); - const ruleCanHaveSuppressionMissingFields = - isQueryRule(ruleType) || isThreatMatchRule(ruleType) || isEqlRule(ruleType); + const ruleCanHaveSuppressionMissingFields = isQueryRule(ruleType) || isThreatMatchRule(ruleType); if (!ruleCanHaveSuppressionMissingFields) { return []; } diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.tsx index 8de4657efe9f3..6d4e4c6cb381b 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.tsx @@ -189,9 +189,6 @@ const StepDefineRuleComponent: FC = ({ const isAlertSuppressionForIndicatorMatchRuleEnabled = useIsExperimentalFeatureEnabled( 'alertSuppressionForIndicatorMatchRuleEnabled' ); - const isAlertSuppressionForEqlRuleEnabled = useIsExperimentalFeatureEnabled( - 'alertSuppressionForEqlRuleEnabled' - ); const isAlertSuppressionLicenseValid = license.isAtLeast(MINIMUM_LICENSE_FOR_SUPPRESSION); @@ -819,8 +816,7 @@ const StepDefineRuleComponent: FC = ({ const isAlertSuppressionEnabled = isQueryRule(ruleType) || isThresholdRule || - (isAlertSuppressionForIndicatorMatchRuleEnabled && isThreatMatchRule(ruleType)) || - (isAlertSuppressionForEqlRuleEnabled && isEqlRule(ruleType)); + (isAlertSuppressionForIndicatorMatchRuleEnabled && isThreatMatchRule(ruleType)); return ( <> diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/use_experimental_feature_fields_transform.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/use_experimental_feature_fields_transform.ts index 399b39de7a71e..93b2c74ed8e31 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/use_experimental_feature_fields_transform.ts +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/use_experimental_feature_fields_transform.ts @@ -8,7 +8,7 @@ import { useCallback } from 'react'; import type { DefineStepRule } from '../../../../detections/pages/detection_engine/rules/types'; import { useIsExperimentalFeatureEnabled } from '../../../../common/hooks/use_experimental_features'; -import { isEqlRule, isThreatMatchRule } from '../../../../../common/detection_engine/utils'; +import { isThreatMatchRule } from '../../../../../common/detection_engine/utils'; /** * transforms DefineStepRule fields according to experimental feature flags @@ -19,20 +19,14 @@ export const useExperimentalFeatureFieldsTransform = { - const isEqlSuppressionDisabled = isEqlRule(fields.ruleType) - ? !isAlertSuppressionForEqlRuleEnabled - : false; const isIndicatorMatchSuppressionDisabled = isThreatMatchRule(fields.ruleType) ? !isAlertSuppressionForIndicatorMatchRuleEnabled : false; // reset any alert suppression values hidden behind feature flag - if (isEqlSuppressionDisabled || isIndicatorMatchSuppressionDisabled) { + if (isIndicatorMatchSuppressionDisabled) { return { ...fields, groupByFields: [], @@ -44,7 +38,7 @@ export const useExperimentalFeatureFieldsTransform = , isInteractive: boolean, savedQuery: SavedQuery | undefined, - { - alertSuppressionForEqlRuleEnabled, - alertSuppressionForIndicatorMatchRuleEnabled, - }: Partial = {} + { alertSuppressionForIndicatorMatchRuleEnabled }: Partial = {} ): EuiDescriptionListProps['listItems'] => { const definitionSectionListItems: EuiDescriptionListProps['listItems'] = []; @@ -661,7 +658,6 @@ const prepareDefinitionSectionListItems = ( } const isSuppressionEnabled = - (rule.type === 'eql' && alertSuppressionForEqlRuleEnabled) || (rule.type === 'threat_match' && alertSuppressionForIndicatorMatchRuleEnabled) || (rule.type && (['query', 'saved_query', 'threshold'] as Type[]).includes(rule.type)); @@ -750,15 +746,12 @@ export const RuleDefinitionSection = ({ const alertSuppressionForIndicatorMatchRuleEnabled = useIsExperimentalFeatureEnabled( 'alertSuppressionForIndicatorMatchRuleEnabled' ); - const alertSuppressionForEqlRuleEnabled = useIsExperimentalFeatureEnabled( - 'alertSuppressionForEqlRuleEnabled' - ); const definitionSectionListItems = prepareDefinitionSectionListItems( rule, isInteractive, savedQuery, - { alertSuppressionForIndicatorMatchRuleEnabled, alertSuppressionForEqlRuleEnabled } + { alertSuppressionForIndicatorMatchRuleEnabled } ); return ( diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/normalization/rule_converters.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/normalization/rule_converters.test.ts index fc7852c506a7b..c2cade2c0146b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/normalization/rule_converters.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/normalization/rule_converters.test.ts @@ -175,27 +175,6 @@ describe('rule_converters', () => { ); }); - test('should accept eql alerts suppression params', () => { - const patchParams = { - alert_suppression: { - duration: { value: 4, unit: 'h' as const }, - group_by: ['agent.name'], - missing_fields_strategy: 'doNotSuppress' as const, - }, - }; - const rule = getEqlRuleParams(); - const patchedParams = patchTypeSpecificSnakeToCamel(patchParams, rule); - expect(patchedParams).toEqual( - expect.objectContaining({ - alertSuppression: { - duration: { value: 4, unit: 'h' }, - groupBy: ['agent.name'], - missingFieldsStrategy: 'doNotSuppress', - }, - }) - ); - }); - test('should accept threat_match alerts suppression params', () => { const patchParams = { alert_suppression: { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/normalization/rule_converters.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/normalization/rule_converters.ts index 5e41a055f1acd..f34b726e6323e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/normalization/rule_converters.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/normalization/rule_converters.ts @@ -113,7 +113,6 @@ export const typeSpecificSnakeToCamel = ( timestampField: params.timestamp_field, eventCategoryOverride: params.event_category_override, tiebreakerField: params.tiebreaker_field, - alertSuppression: convertAlertSuppressionToCamel(params.alert_suppression), }; } case 'esql': { @@ -223,8 +222,6 @@ const patchEqlParams = ( timestampField: params.timestamp_field ?? existingRule.timestampField, eventCategoryOverride: params.event_category_override ?? existingRule.eventCategoryOverride, tiebreakerField: params.tiebreaker_field ?? existingRule.tiebreakerField, - alertSuppression: - convertAlertSuppressionToCamel(params.alert_suppression) ?? existingRule.alertSuppression, }; }; @@ -562,7 +559,6 @@ export const typeSpecificCamelToSnake = ( timestamp_field: params.timestampField, event_category_override: params.eventCategoryOverride, tiebreaker_field: params.tiebreakerField, - alert_suppression: convertAlertSuppressionToSnake(params.alertSuppression), }; } case 'esql': { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_schema/model/rule_schemas.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_schema/model/rule_schemas.ts index 2151260292841..deb853335bf24 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_schema/model/rule_schemas.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_schema/model/rule_schemas.ts @@ -153,7 +153,6 @@ export const EqlSpecificRuleParams = z.object({ eventCategoryOverride: EventCategoryOverride.optional(), timestampField: TimestampField.optional(), tiebreakerField: TiebreakerField.optional(), - alertSuppression: AlertSuppressionCamel.optional(), }); export type EqlRuleParams = BaseRuleParams & EqlSpecificRuleParams; diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/prebuilt_rules_preview.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/prebuilt_rules_preview.cy.ts index 8f0d972f6cabf..855e2f554c5b6 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/prebuilt_rules_preview.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/prebuilt_rules_preview.cy.ts @@ -240,7 +240,6 @@ describe('Detection rules, Prebuilt Rules Installation and Update workflow', () query: 'process where process.name == "regsvr32.exe"', index: ['winlogbeat-*', 'logs-endpoint.events.*'], filters, - alert_suppression: undefined, }); const THREAT_MATCH_INDEX_PATTERN_RULE = createRuleAssetSavedObject({ From 48334dd2c6cd43548d1136c6e0d331b6450c9bd6 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com> Date: Wed, 17 Jan 2024 14:30:04 +0000 Subject: [PATCH 11/28] Update schema.tsx --- .../rule_creation_ui/components/step_define_rule/schema.tsx | 2 -- 1 file changed, 2 deletions(-) diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/schema.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/schema.tsx index b5ba36fc56bbd..c5002f7adec4b 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/schema.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/schema.tsx @@ -17,7 +17,6 @@ import { customValidators, } from '../../../../common/components/threat_match/helpers'; import { - isEqlRule, isEsqlRule, isNewTermsRule, isQueryRule, @@ -606,7 +605,6 @@ export const schema: FormSchema = { const [{ formData }] = args; const needsValidation = isQueryRule(formData.ruleType) || - isEqlRule(formData.ruleType) || isThreatMatchRule(formData.ruleType); if (!needsValidation) { return; From 951da77b86c0c6f7fc69351de761f20c1cb3b5c2 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com> Date: Wed, 17 Jan 2024 14:49:45 +0000 Subject: [PATCH 12/28] [Security Solution][Detection Engine] adds FTR tests for IM suppression according to test plan (#174586) ## Summary FTR tests according to [test plan]( https://docs.google.com/document/d/1J4jEzsywCvg2NRhmGyDDWqqSNB-qjftQ8SqrQPlL__M/edit), some of test scenario omitted - only most important added ### Checklist Delete any items that are not applicable to this PR. - [ ] Any text added follows [EUI's writing guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses sentence case text and includes [i18n support](https://github.com/elastic/kibana/blob/main/packages/kbn-i18n/README.md) - [ ] [Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html) was added for features that require explanation or tutorials - [ ] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios - [ ] [Flaky Test Runner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1) was used on any tests changed - [ ] Any UI touched in this PR is usable by keyboard only (learn more about [keyboard accessibility](https://webaim.org/techniques/keyboard/)) - [ ] Any UI touched in this PR does not create any new axe failures (run axe in browser: [FF](https://addons.mozilla.org/en-US/firefox/addon/axe-devtools/), [Chrome](https://chrome.google.com/webstore/detail/axe-web-accessibility-tes/lhdoppojpmngadmnindnejefpokejbdd?hl=en-US)) - [ ] If a plugin configuration key changed, check if it needs to be allowlisted in the cloud and added to the [docker list](https://github.com/elastic/kibana/blob/main/src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker) - [ ] This renders correctly on smaller devices using a responsive layout. (You can test this [in your browser](https://www.browserstack.com/guide/responsive-testing-on-local-server)) - [ ] This was checked for [cross-browser compatibility](https://www.elastic.co/support/matrix#matrix_browsers) ### Risk Matrix Delete this section if it is not applicable to this PR. Before closing this PR, invite QA, stakeholders, and other developers to identify risks that should be tested prior to the change/feature release. When forming the risk matrix, consider some of the following examples and how they may potentially impact the change: | Risk | Probability | Severity | Mitigation/Notes | |---------------------------|-------------|----------|-------------------------| | Multiple Spaces—unexpected behavior in non-default Kibana Space. | Low | High | Integration tests will verify that all features are still supported in non-default Kibana Space and when user switches between spaces. | | Multiple nodes—Elasticsearch polling might have race conditions when multiple Kibana nodes are polling for the same tasks. | High | Low | Tasks are idempotent, so executing them multiple times will not result in logical error, but will degrade performance. To test for this case we add plenty of unit tests around this logic and document manual testing procedure. | | Code should gracefully handle cases when feature X or plugin Y are disabled. | Medium | High | Unit tests will verify that any feature flag or plugin combination still results in our service operational. | | [See more potential risk examples](https://github.com/elastic/kibana/blob/main/RISK_MATRIX.mdx) | ### For maintainers - [ ] This was checked for breaking API changes and was [labeled appropriately](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process) --- .../execution_logic/index.ts | 3 +- .../{threat_match.ts => indicator_match.ts} | 0 .../indicator_match_alert_suppression.ts | 1365 +++++++++++++++++ ...get_threat_match_rule_for_alert_testing.ts | 1 + 4 files changed, 1368 insertions(+), 1 deletion(-) rename x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_execution_logic/execution_logic/{threat_match.ts => indicator_match.ts} (100%) create mode 100644 x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_execution_logic/execution_logic/indicator_match_alert_suppression.ts diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_execution_logic/execution_logic/index.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_execution_logic/execution_logic/index.ts index f164857d9bd8f..20bb538daed96 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_execution_logic/execution_logic/index.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_execution_logic/execution_logic/index.ts @@ -14,7 +14,8 @@ export default ({ loadTestFile }: FtrProviderContext): void => { loadTestFile(require.resolve('./machine_learning')); loadTestFile(require.resolve('./new_terms')); loadTestFile(require.resolve('./saved_query')); - loadTestFile(require.resolve('./threat_match')); + loadTestFile(require.resolve('./indicator_match')); + loadTestFile(require.resolve('./indicator_match_alert_suppression')); loadTestFile(require.resolve('./threshold')); loadTestFile(require.resolve('./threshold_alert_suppression')); loadTestFile(require.resolve('./non_ecs_fields')); diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_execution_logic/execution_logic/threat_match.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_execution_logic/execution_logic/indicator_match.ts similarity index 100% rename from x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_execution_logic/execution_logic/threat_match.ts rename to x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_execution_logic/execution_logic/indicator_match.ts diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_execution_logic/execution_logic/indicator_match_alert_suppression.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_execution_logic/execution_logic/indicator_match_alert_suppression.ts new file mode 100644 index 0000000000000..3d5712e1bc066 --- /dev/null +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_execution_logic/execution_logic/indicator_match_alert_suppression.ts @@ -0,0 +1,1365 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { v4 as uuidv4 } from 'uuid'; +import expect from 'expect'; + +import { + ALERT_SUPPRESSION_START, + ALERT_SUPPRESSION_END, + ALERT_SUPPRESSION_DOCS_COUNT, + ALERT_SUPPRESSION_TERMS, + ALERT_LAST_DETECTED, + TIMESTAMP, +} from '@kbn/rule-data-utils'; + +import { DETECTION_ENGINE_SIGNALS_STATUS_URL as DETECTION_ENGINE_ALERTS_STATUS_URL } from '@kbn/security-solution-plugin/common/constants'; + +import { ThreatMatchRuleCreateProps } from '@kbn/security-solution-plugin/common/api/detection_engine'; +import { RuleExecutionStatusEnum } from '@kbn/security-solution-plugin/common/api/detection_engine/rule_monitoring'; + +import { ALERT_ORIGINAL_TIME } from '@kbn/security-solution-plugin/common/field_maps/field_names'; +import { + createRule, + getOpenAlerts, + getPreviewAlerts, + getThreatMatchRuleForAlertTesting, + previewRule, + patchRule, + setAlertStatus, + dataGeneratorFactory, +} from '../../../utils'; +import { FtrProviderContext } from '../../../../../ftr_provider_context'; + +export default ({ getService }: FtrProviderContext) => { + const supertest = getService('supertest'); + const esArchiver = getService('esArchiver'); + const es = getService('es'); + const log = getService('log'); + + const { + indexListOfDocuments: indexListOfSourceDocuments, + indexGeneratedDocuments: indexGeneratedSourceDocuments, + } = dataGeneratorFactory({ + es, + index: 'ecs_compliant', + log, + }); + + // ensures minimal number of events indexed that will be queried within rule runs + // it is needed to ensure we cover IM rule code branch execution in which number of events is greater than number of threats + // takes array of timestamps, so events can be indexed for multiple rule executions + const eventsFiller = async ({ + id, + timestamp, + count, + }: { + id: string; + count: number; + timestamp: string[]; + }) => { + if (!count) { + return; + } + + await Promise.all( + timestamp.map((t) => + indexGeneratedSourceDocuments({ + docsCount: count, + seed: (index) => ({ + id, + '@timestamp': t, + host: { + name: `host-event-${index}`, + }, + }), + }) + ) + ); + }; + + // ensures minimal number of threats indexed that will be queried within rule runs + // it is needed to ensure we cover IM rule code branch execution in which number of events is smaller than number of threats + // takes single timestamp, as time window for queried threats is 30 days in past + const threatsFiller = async ({ + id, + timestamp, + count, + }: { + id: string; + count: number; + timestamp: string; + }) => + count && + indexGeneratedSourceDocuments({ + docsCount: count, + seed: (index) => ({ + id, + '@timestamp': timestamp, + 'agent.type': 'threat', + host: { + name: `host-threat-${index}`, + }, + }), + }); + + const addThreatDocuments = ({ + id, + timestamp, + fields, + count, + }: { + id: string; + fields?: Record; + timestamp: string; + count: number; + }) => + indexGeneratedSourceDocuments({ + docsCount: count, + seed: (index) => ({ + id, + '@timestamp': timestamp, + 'agent.type': 'threat', + ...fields, + }), + }); + + // for simplicity IM rule query source events and threats from the same index + // all events with agent.type:threat are categorized as threats + // the rest will be source ones + const indicatorMatchRule = (id: string) => ({ + ...getThreatMatchRuleForAlertTesting(['ecs_compliant']), + query: `id:${id} and NOT agent.type:threat`, + threat_query: `id:${id} and agent.type:threat`, + name: 'ALert suppression IM test rule', + }); + + const cases = [ + { + eventsCount: 10, + threatsCount: 0, + title: `events count is greater than threats count`, + }, + + { + eventsCount: 0, + threatsCount: 10, + title: `events count is smaller than threats count`, + }, + ]; + + describe.only('@ess @serverless Indicator match type rules, alert suppression', () => { + before(async () => { + await esArchiver.load('x-pack/test/functional/es_archives/security_solution/ecs_compliant'); + }); + + after(async () => { + await esArchiver.unload('x-pack/test/functional/es_archives/security_solution/ecs_compliant'); + }); + + cases.forEach(({ eventsCount, threatsCount, title }) => { + describe(`Code execution path: ${title}`, () => { + it('should suppress an alert on real rule executions', async () => { + const id = uuidv4(); + const firstTimestamp = new Date().toISOString(); + + await eventsFiller({ id, count: eventsCount, timestamp: [firstTimestamp] }); + await threatsFiller({ id, count: threatsCount, timestamp: firstTimestamp }); + + const firstDocument = { + id, + '@timestamp': firstTimestamp, + host: { + name: 'host-a', + }, + }; + await indexListOfSourceDocuments([firstDocument]); + + await addThreatDocuments({ + id, + timestamp: firstTimestamp, + fields: { + host: { + name: 'host-a', + }, + }, + count: 1, + }); + + const rule: ThreatMatchRuleCreateProps = { + ...indicatorMatchRule(id), + alert_suppression: { + group_by: ['host.name'], + duration: { + value: 300, + unit: 'm', + }, + missing_fields_strategy: 'suppress', + }, + from: 'now-35m', + interval: '30m', + }; + const createdRule = await createRule(supertest, log, rule); + const alerts = await getOpenAlerts(supertest, log, es, createdRule); + expect(alerts.hits.hits.length).toEqual(1); + + // suppression start equal to alert timestamp + const suppressionStart = alerts.hits.hits[0]._source?.[TIMESTAMP]; + + expect(alerts.hits.hits[0]._source).toEqual( + expect.objectContaining({ + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: 'host-a', + }, + ], + // suppression boundaries equal to alert time, since no alert been suppressed + [ALERT_SUPPRESSION_START]: suppressionStart, + [ALERT_SUPPRESSION_END]: suppressionStart, + [ALERT_ORIGINAL_TIME]: firstTimestamp, + [TIMESTAMP]: suppressionStart, + [ALERT_SUPPRESSION_DOCS_COUNT]: 0, + }) + ); + + const secondTimestamp = new Date().toISOString(); + const secondDocument = { + id, + '@timestamp': secondTimestamp, + host: { + name: 'host-a', + }, + }; + // Add a new document, then disable and re-enable to trigger another rule run. The second doc should + // trigger an update to the existing alert without changing the timestamp + await indexListOfSourceDocuments([secondDocument, secondDocument]); + await eventsFiller({ id, count: eventsCount, timestamp: [secondTimestamp] }); + + await patchRule(supertest, log, { id: createdRule.id, enabled: false }); + await patchRule(supertest, log, { id: createdRule.id, enabled: true }); + const afterTimestamp = new Date(); + const secondAlerts = await getOpenAlerts( + supertest, + log, + es, + createdRule, + RuleExecutionStatusEnum.succeeded, + undefined, + afterTimestamp + ); + expect(secondAlerts.hits.hits.length).toEqual(1); + expect(secondAlerts.hits.hits[0]._source).toEqual( + expect.objectContaining({ + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: 'host-a', + }, + ], + [ALERT_ORIGINAL_TIME]: firstTimestamp, // timestamp is the same + [ALERT_SUPPRESSION_START]: suppressionStart, // suppression start is the same + [ALERT_SUPPRESSION_DOCS_COUNT]: 2, // 2 alerts from second rule run, that's why 2 suppressed + }) + ); + // suppression end value should be greater than second document timestamp, but lesser than current time + const suppressionEnd = new Date( + secondAlerts.hits.hits[0]._source?.[ALERT_SUPPRESSION_END] as string + ).getTime(); + expect(suppressionEnd).toBeLessThan(new Date().getTime()); + expect(suppressionEnd).toBeGreaterThan(new Date(secondTimestamp).getDate()); + }); + + it('should NOT suppress and update an alert if the alert is closed', async () => { + const id = uuidv4(); + const firstTimestamp = new Date().toISOString(); + + await eventsFiller({ id, count: eventsCount, timestamp: [firstTimestamp] }); + await threatsFiller({ id, count: threatsCount, timestamp: firstTimestamp }); + + const firstDocument = { + id, + '@timestamp': firstTimestamp, + host: { + name: 'host-a', + }, + }; + await indexListOfSourceDocuments([firstDocument]); + + await addThreatDocuments({ + id, + timestamp: firstTimestamp, + fields: { + host: { + name: 'host-a', + }, + }, + count: 1, + }); + + const rule: ThreatMatchRuleCreateProps = { + ...indicatorMatchRule(id), + alert_suppression: { + group_by: ['host.name'], + duration: { + value: 300, + unit: 'm', + }, + missing_fields_strategy: 'suppress', + }, + from: 'now-35m', + interval: '30m', + }; + const createdRule = await createRule(supertest, log, rule); + const alerts = await getOpenAlerts(supertest, log, es, createdRule); + + // Close the alert. Subsequent rule executions should ignore this closed alert + // for suppression purposes. + const alertIds = alerts.hits.hits.map((alert) => alert._id); + await supertest + .post(DETECTION_ENGINE_ALERTS_STATUS_URL) + .set('kbn-xsrf', 'true') + .send(setAlertStatus({ alertIds, status: 'closed' })) + .expect(200); + + const secondTimestamp = new Date().toISOString(); + const secondDocument = { + id, + '@timestamp': secondTimestamp, + agent: { + name: 'agent-1', + }, + }; + // Add new documents, then disable and re-enable to trigger another rule run. The second doc should + // trigger a new alert since the first one is now closed. + await indexListOfSourceDocuments([secondDocument]); + await eventsFiller({ id, count: eventsCount, timestamp: [secondTimestamp] }); + + await patchRule(supertest, log, { id: createdRule.id, enabled: false }); + await patchRule(supertest, log, { id: createdRule.id, enabled: true }); + const afterTimestamp = new Date(); + const secondAlerts = await getOpenAlerts( + supertest, + log, + es, + createdRule, + RuleExecutionStatusEnum.succeeded, + undefined, + afterTimestamp + ); + expect(secondAlerts.hits.hits.length).toEqual(1); + expect(alerts.hits.hits[0]._source).toEqual( + expect.objectContaining({ + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: 'host-a', + }, + ], + [ALERT_ORIGINAL_TIME]: firstTimestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 0, + }) + ); + }); + + it('should NOT suppress alerts when suppression period is less than rule interval', async () => { + const id = uuidv4(); + const firstTimestamp = '2020-10-28T05:45:00.000Z'; + const secondTimestamp = '2020-10-28T06:15:00.000Z'; + + await eventsFiller({ + id, + count: eventsCount, + timestamp: [firstTimestamp, secondTimestamp], + }); + await threatsFiller({ id, count: threatsCount, timestamp: firstTimestamp }); + + const firstRunDoc = { + id, + '@timestamp': firstTimestamp, + host: { + name: 'host-a', + }, + }; + + const secondRunDoc = { + ...firstRunDoc, + '@timestamp': secondTimestamp, + }; + + await indexListOfSourceDocuments([firstRunDoc, secondRunDoc]); + + await addThreatDocuments({ + id, + timestamp: firstTimestamp, + fields: { + host: { + name: 'host-a', + }, + }, + count: 1, + }); + + const rule: ThreatMatchRuleCreateProps = { + ...indicatorMatchRule(id), + alert_suppression: { + group_by: ['host.name'], + duration: { + value: 20, + unit: 'm', + }, + missing_fields_strategy: 'suppress', + }, + from: 'now-35m', + interval: '30m', + }; + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T06:30:00.000Z'), + invocationCount: 2, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: [ALERT_ORIGINAL_TIME], + }); + expect(previewAlerts.length).toBe(2); + expect(previewAlerts[0]._source).toEqual( + expect.objectContaining({ + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: 'host-a', + }, + ], + [TIMESTAMP]: '2020-10-28T06:00:00.000Z', + [ALERT_SUPPRESSION_START]: '2020-10-28T06:00:00.000Z', + [ALERT_SUPPRESSION_END]: '2020-10-28T06:00:00.000Z', + [ALERT_SUPPRESSION_DOCS_COUNT]: 0, + }) + ); + + expect(previewAlerts[1]._source).toEqual( + expect.objectContaining({ + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: 'host-a', + }, + ], + [TIMESTAMP]: '2020-10-28T06:30:00.000Z', + [ALERT_SUPPRESSION_START]: '2020-10-28T06:30:00.000Z', + [ALERT_SUPPRESSION_END]: '2020-10-28T06:30:00.000Z', + [ALERT_SUPPRESSION_DOCS_COUNT]: 0, + }) + ); + }); + + it('should suppress alerts in the time window that covers 3 rule executions', async () => { + const id = uuidv4(); + const firstTimestamp = '2020-10-28T05:45:00.000Z'; + const secondTimestamp = '2020-10-28T06:15:00.000Z'; + const thirdTimestamp = '2020-10-28T06:45:00.000Z'; + const firstRunDoc = { + id, + '@timestamp': firstTimestamp, + host: { name: 'host-a' }, + }; + const secondRunDoc = { + ...firstRunDoc, + '@timestamp': secondTimestamp, + }; + const thirdRunDoc = { + ...firstRunDoc, + '@timestamp': thirdTimestamp, + }; + + await eventsFiller({ + id, + count: eventsCount, + timestamp: [firstTimestamp, secondTimestamp, thirdTimestamp], + }); + await threatsFiller({ id, count: threatsCount, timestamp: firstTimestamp }); + + await indexListOfSourceDocuments([ + firstRunDoc, + firstRunDoc, + secondRunDoc, + secondRunDoc, + thirdRunDoc, + ]); + + await addThreatDocuments({ + id, + timestamp: firstTimestamp, + fields: { + host: { + name: 'host-a', + }, + }, + count: 1, + }); + + const rule: ThreatMatchRuleCreateProps = { + ...indicatorMatchRule(id), + alert_suppression: { + group_by: ['host.name'], + duration: { + value: 2, + unit: 'h', + }, + missing_fields_strategy: 'suppress', + }, + from: 'now-35m', + interval: '30m', + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T07:00:00.000Z'), + invocationCount: 3, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: [ALERT_ORIGINAL_TIME], + }); + expect(previewAlerts.length).toEqual(1); + expect(previewAlerts[0]._source).toEqual({ + ...previewAlerts[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: 'host-a', + }, + ], + [TIMESTAMP]: '2020-10-28T06:00:00.000Z', + [ALERT_LAST_DETECTED]: '2020-10-28T07:00:00.000Z', // Note: ALERT_LAST_DETECTED gets updated, timestamp does not + [ALERT_ORIGINAL_TIME]: firstTimestamp, + [ALERT_SUPPRESSION_START]: '2020-10-28T06:00:00.000Z', + [ALERT_SUPPRESSION_END]: '2020-10-28T07:00:00.000Z', + [ALERT_SUPPRESSION_DOCS_COUNT]: 4, // in total 4 alert got suppressed: 1 from the first run, 2 from the second, 1 from the third + }); + }); + + it('should suppress the correct alerts based on multi values group_by', async () => { + const id = uuidv4(); + const firstTimestamp = '2020-10-28T05:45:00.000Z'; + const secondTimestamp = '2020-10-28T06:15:00.000Z'; + const firstRunDocA = { + id, + '@timestamp': firstTimestamp, + 'host.name': 'host-a', + 'agent.version': 1, + }; + const firstRunDoc2 = { + ...firstRunDocA, + 'agent.version': 2, + }; + const firstRunDocB = { + ...firstRunDocA, + 'host.name': 'host-b', + 'agent.version': 1, + }; + + const secondRunDocA = { + ...firstRunDocA, + '@timestamp': secondTimestamp, + }; + + await eventsFiller({ + id, + count: eventsCount, + timestamp: [firstTimestamp, secondTimestamp], + }); + await threatsFiller({ id, count: threatsCount, timestamp: firstTimestamp }); + + await indexListOfSourceDocuments([ + firstRunDocA, + firstRunDoc2, + firstRunDocB, + secondRunDocA, + secondRunDocA, + secondRunDocA, + ]); + await addThreatDocuments({ + id, + timestamp: firstTimestamp, + fields: { + host: { + name: 'host-a', + }, + }, + count: 1, + }); + + const rule: ThreatMatchRuleCreateProps = { + ...indicatorMatchRule(id), + alert_suppression: { + group_by: ['host.name', 'agent.version'], + duration: { + value: 2, + unit: 'h', + }, + missing_fields_strategy: 'suppress', + }, + from: 'now-35m', + interval: '30m', + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T06:30:00.000Z'), + invocationCount: 2, + }); + + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: ['host.name', 'agent.version', ALERT_ORIGINAL_TIME], + }); + // 3 alerts should be generated: + // 1. for pair 'host-a', 1 - suppressed + // 2. for pair 'host-a', 2 - not suppressed + expect(previewAlerts.length).toEqual(2); + expect(previewAlerts[0]._source).toEqual({ + ...previewAlerts[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: 'host-a', + }, + { + field: 'agent.version', + value: '1', + }, + ], + [TIMESTAMP]: '2020-10-28T06:00:00.000Z', + [ALERT_LAST_DETECTED]: '2020-10-28T06:30:00.000Z', // Note: ALERT_LAST_DETECTED gets updated, timestamp does not + [ALERT_ORIGINAL_TIME]: firstTimestamp, + [ALERT_SUPPRESSION_START]: '2020-10-28T06:00:00.000Z', + [ALERT_SUPPRESSION_END]: '2020-10-28T06:30:00.000Z', + [ALERT_SUPPRESSION_DOCS_COUNT]: 3, // 3 alerts suppressed from the second run + }); + expect(previewAlerts[1]._source).toEqual({ + ...previewAlerts[1]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: 'host-a', + }, + { + field: 'agent.version', + value: '2', + }, + ], + [TIMESTAMP]: '2020-10-28T06:00:00.000Z', + [ALERT_LAST_DETECTED]: '2020-10-28T06:00:00.000Z', + [ALERT_ORIGINAL_TIME]: firstTimestamp, + [ALERT_SUPPRESSION_START]: '2020-10-28T06:00:00.000Z', + [ALERT_SUPPRESSION_END]: '2020-10-28T06:00:00.000Z', + [ALERT_SUPPRESSION_DOCS_COUNT]: 0, // no suppressed alerts + }); + }); + + it('should correctly suppress when using a timestamp override', async () => { + const id = uuidv4(); + const firstTimestamp = '2020-10-28T05:45:00.000Z'; + const secondTimestamp = '2020-10-28T06:10:00.000Z'; + const docWithoutOverride = { + id, + '@timestamp': firstTimestamp, + host: { name: 'host-a' }, + event: { + ingested: firstTimestamp, + }, + }; + const docWithOverride = { + ...docWithoutOverride, + // This doc simulates a very late arriving doc + '@timestamp': '2020-10-28T03:00:00.000Z', + event: { + ingested: secondTimestamp, + }, + }; + + await eventsFiller({ + id, + count: eventsCount, + timestamp: [firstTimestamp, secondTimestamp], + }); + await threatsFiller({ id, count: threatsCount, timestamp: firstTimestamp }); + + await indexListOfSourceDocuments([docWithoutOverride, docWithOverride]); + + await addThreatDocuments({ + id, + timestamp: firstTimestamp, + fields: { + host: { + name: 'host-a', + }, + }, + count: 1, + }); + + const rule: ThreatMatchRuleCreateProps = { + ...indicatorMatchRule(id), + alert_suppression: { + group_by: ['host.name'], + duration: { + value: 300, + unit: 'm', + }, + missing_fields_strategy: 'suppress', + }, + from: 'now-35m', + interval: '30m', + timestamp_override: 'event.ingested', + }; + // 1 alert should be suppressed, based on event.ingested value of a document + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T06:30:00.000Z'), + invocationCount: 2, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: ['host.name', ALERT_ORIGINAL_TIME], + }); + + expect(previewAlerts.length).toEqual(1); + expect(previewAlerts[0]._source).toEqual({ + ...previewAlerts[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: 'host-a', + }, + ], + [ALERT_ORIGINAL_TIME]: firstTimestamp, + [ALERT_SUPPRESSION_START]: '2020-10-28T06:00:00.000Z', + [ALERT_SUPPRESSION_END]: '2020-10-28T06:30:00.000Z', + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }); + }); + + it('should generate and update up to max_signals alerts', async () => { + const expectedMaxSignals = 40; + const id = uuidv4(); + const firstTimestamp = '2020-10-28T05:45:00.000Z'; + const secondTimestamp = '2020-10-28T06:10:00.000Z'; + + await eventsFiller({ + id, + count: eventsCount * 20, + timestamp: [firstTimestamp, secondTimestamp], + }); + await threatsFiller({ id, count: threatsCount * 20, timestamp: firstTimestamp }); + + await Promise.all( + [firstTimestamp, secondTimestamp].map((t) => + indexGeneratedSourceDocuments({ + docsCount: expectedMaxSignals, + seed: (index) => ({ + id, + '@timestamp': t, + host: { + name: `host-a`, + }, + 'agent.name': `agent-${index}`, + }), + }) + ) + ); + await addThreatDocuments({ + id, + timestamp: firstTimestamp, + fields: { + host: { + name: 'host-a', + }, + }, + count: 1, + }); + + const rule: ThreatMatchRuleCreateProps = { + ...indicatorMatchRule(id), + alert_suppression: { + group_by: ['agent.name'], + duration: { + value: 300, + unit: 'm', + }, + missing_fields_strategy: 'suppress', + }, + from: 'now-35m', + interval: '30m', + max_signals: expectedMaxSignals, + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T06:30:00.000Z'), + invocationCount: 2, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + size: 1000, + sort: ['agent.name', ALERT_ORIGINAL_TIME], + }); + expect(previewAlerts.length).toEqual(expectedMaxSignals); + expect(previewAlerts[0]._source).toEqual( + expect.objectContaining({ + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'agent.name', + value: 'agent-0', + }, + ], + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }) + ); + }); + + it('should deduplicate alerts while suppressing new ones', async () => { + const id = uuidv4(); + const firstTimestamp = '2020-10-28T05:45:00.000Z'; + const secondTimestamp = '2020-10-28T06:10:00.000Z'; + const doc1 = { + id, + '@timestamp': firstTimestamp, + host: { name: 'host-a' }, + }; + + const doc2 = { + ...doc1, + '@timestamp': secondTimestamp, + }; + + await eventsFiller({ + id, + count: eventsCount, + timestamp: [firstTimestamp, secondTimestamp], + }); + await threatsFiller({ id, count: threatsCount, timestamp: firstTimestamp }); + + // 4 alert should be suppressed + await indexListOfSourceDocuments([doc1, doc1, doc2, doc2, doc2]); + + await addThreatDocuments({ + id, + timestamp: firstTimestamp, + fields: { + host: { + name: 'host-a', + }, + }, + count: 1, + }); + + const rule: ThreatMatchRuleCreateProps = { + ...indicatorMatchRule(id), + alert_suppression: { + group_by: ['host.name'], + duration: { + value: 300, + unit: 'm', + }, + missing_fields_strategy: 'suppress', + }, + // large look-back time covers all docs + from: 'now-1h', + interval: '30m', + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T06:30:00.000Z'), + invocationCount: 2, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: ['host.name', ALERT_ORIGINAL_TIME], + }); + expect(previewAlerts.length).toEqual(1); + expect(previewAlerts[0]._source).toEqual({ + ...previewAlerts[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: 'host-a', + }, + ], + [ALERT_ORIGINAL_TIME]: firstTimestamp, + [ALERT_SUPPRESSION_START]: '2020-10-28T06:00:00.000Z', + [ALERT_SUPPRESSION_END]: '2020-10-28T06:30:00.000Z', + [ALERT_SUPPRESSION_DOCS_COUNT]: 4, + }); + }); + + it('should suppress alerts with missing fields', async () => { + const id = uuidv4(); + const firstTimestamp = '2020-10-28T05:45:00.000Z'; + const secondTimestamp = '2020-10-28T06:10:00.000Z'; + const docWithMissingField1 = { + id, + '@timestamp': firstTimestamp, + host: { name: 'host-a' }, + }; + + const doc1 = { + ...docWithMissingField1, + agent: { name: 'agent-1' }, + }; + + const docWithMissingField2 = { + ...docWithMissingField1, + '@timestamp': secondTimestamp, + }; + + const doc2 = { + ...doc1, + '@timestamp': secondTimestamp, + }; + + await eventsFiller({ + id, + count: eventsCount, + timestamp: [firstTimestamp, secondTimestamp], + }); + await threatsFiller({ id, count: threatsCount, timestamp: firstTimestamp }); + + // 4 alert should be suppressed: 2 for agent.name: 'agent-1' and 2 for missing agent.name + await indexListOfSourceDocuments([ + doc1, + doc1, + docWithMissingField1, + docWithMissingField1, + docWithMissingField2, + doc2, + ]); + + await addThreatDocuments({ + id, + timestamp: firstTimestamp, + fields: { + host: { + name: 'host-a', + }, + }, + count: 1, + }); + + const rule: ThreatMatchRuleCreateProps = { + ...indicatorMatchRule(id), + alert_suppression: { + group_by: ['agent.name'], + duration: { + value: 300, + unit: 'm', + }, + missing_fields_strategy: 'suppress', + }, + from: 'now-35m', + interval: '30m', + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T06:30:00.000Z'), + invocationCount: 2, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: ['agent.name', ALERT_ORIGINAL_TIME], + }); + expect(previewAlerts.length).toEqual(2); + expect(previewAlerts[0]._source).toEqual({ + ...previewAlerts[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'agent.name', + value: 'agent-1', + }, + ], + [ALERT_ORIGINAL_TIME]: firstTimestamp, + [ALERT_SUPPRESSION_START]: '2020-10-28T06:00:00.000Z', + [ALERT_SUPPRESSION_END]: '2020-10-28T06:30:00.000Z', + [ALERT_SUPPRESSION_DOCS_COUNT]: 2, + }); + + expect(previewAlerts[1]._source).toEqual({ + ...previewAlerts[1]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'agent.name', + value: null, + }, + ], + [ALERT_ORIGINAL_TIME]: firstTimestamp, + [ALERT_SUPPRESSION_START]: '2020-10-28T06:00:00.000Z', + [ALERT_SUPPRESSION_END]: '2020-10-28T06:30:00.000Z', + [ALERT_SUPPRESSION_DOCS_COUNT]: 2, + }); + }); + + it('should not suppress alerts with missing fields if configured so', async () => { + const id = uuidv4(); + const firstTimestamp = '2020-10-28T05:45:00.000Z'; + const secondTimestamp = '2020-10-28T06:10:00.000Z'; + const docWithMissingField1 = { + id, + '@timestamp': firstTimestamp, + host: { name: 'host-a' }, + }; + + const doc1 = { + ...docWithMissingField1, + agent: { name: 'agent-1' }, + }; + + const docWithMissingField2 = { + ...docWithMissingField1, + '@timestamp': secondTimestamp, + }; + + const doc2 = { + ...doc1, + '@timestamp': secondTimestamp, + }; + + await eventsFiller({ + id, + count: eventsCount, + timestamp: [firstTimestamp, secondTimestamp], + }); + await threatsFiller({ id, count: threatsCount, timestamp: firstTimestamp }); + + // 2 alert should be suppressed: 2 for agent.name: 'agent-1' and not any for missing agent.name + await indexListOfSourceDocuments([ + doc1, + doc1, + docWithMissingField1, + docWithMissingField1, + docWithMissingField2, + doc2, + ]); + + await addThreatDocuments({ + id, + timestamp: firstTimestamp, + fields: { + host: { + name: 'host-a', + }, + }, + count: 1, + }); + + const rule: ThreatMatchRuleCreateProps = { + ...indicatorMatchRule(id), + alert_suppression: { + group_by: ['agent.name'], + duration: { + value: 300, + unit: 'm', + }, + missing_fields_strategy: 'doNotSuppress', + }, + from: 'now-35m', + interval: '30m', + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T06:30:00.000Z'), + invocationCount: 2, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: ['agent.name', ALERT_ORIGINAL_TIME], + }); + expect(previewAlerts.length).toEqual(4); + expect(previewAlerts[0]._source).toEqual({ + ...previewAlerts[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'agent.name', + value: 'agent-1', + }, + ], + [ALERT_ORIGINAL_TIME]: firstTimestamp, + [ALERT_SUPPRESSION_START]: '2020-10-28T06:00:00.000Z', + [ALERT_SUPPRESSION_END]: '2020-10-28T06:30:00.000Z', + [ALERT_SUPPRESSION_DOCS_COUNT]: 2, + }); + + // rest of alerts are not suppressed and do not have suppress properties + previewAlerts.slice(1).forEach((previewAlert) => { + const source = previewAlert._source; + expect(source).toHaveProperty('id', id); + expect(source).not.toHaveProperty(ALERT_SUPPRESSION_DOCS_COUNT); + expect(source).not.toHaveProperty(ALERT_SUPPRESSION_END); + expect(source).not.toHaveProperty(ALERT_SUPPRESSION_TERMS); + expect(source).not.toHaveProperty(ALERT_SUPPRESSION_DOCS_COUNT); + }); + }); + + describe('rule execution only', () => { + it('should suppress alerts during rule execution only', async () => { + const id = uuidv4(); + const timestamp = '2020-10-28T06:45:00.000Z'; + const doc1 = { + id, + '@timestamp': timestamp, + host: { name: 'host-a' }, + }; + // doc2 does not generate alert + const doc2 = { + ...doc1, + host: { name: 'host-b' }, + }; + + await eventsFiller({ id, count: eventsCount, timestamp: [timestamp] }); + await threatsFiller({ id, count: threatsCount, timestamp }); + + await indexListOfSourceDocuments([doc1, doc1, doc1, doc2]); + + await addThreatDocuments({ + id, + timestamp, + fields: { + host: { + name: 'host-a', + }, + }, + count: 1, + }); + + const rule: ThreatMatchRuleCreateProps = { + ...indicatorMatchRule(id), + alert_suppression: { + group_by: ['host.name'], + missing_fields_strategy: 'suppress', + }, + from: 'now-35m', + interval: '30m', + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T07:00:00.000Z'), + invocationCount: 1, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: [ALERT_ORIGINAL_TIME], + }); + expect(previewAlerts.length).toEqual(1); + expect(previewAlerts[0]._source).toEqual({ + ...previewAlerts[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: 'host-a', + }, + ], + [TIMESTAMP]: '2020-10-28T07:00:00.000Z', + [ALERT_LAST_DETECTED]: '2020-10-28T07:00:00.000Z', + [ALERT_ORIGINAL_TIME]: timestamp, + [ALERT_SUPPRESSION_START]: '2020-10-28T07:00:00.000Z', + [ALERT_SUPPRESSION_END]: '2020-10-28T07:00:00.000Z', + [ALERT_SUPPRESSION_DOCS_COUNT]: 2, + }); + }); + + it('should suppress alerts with missing fields during rule execution only', async () => { + const id = uuidv4(); + const timestamp = '2020-10-28T06:45:00.000Z'; + const doc1 = { + id, + '@timestamp': timestamp, + host: { name: 'host-a' }, + }; + + const doc2 = { + ...doc1, + agent: { name: 'agent-1' }, + }; + + await eventsFiller({ id, count: eventsCount, timestamp: [timestamp] }); + await threatsFiller({ id, count: threatsCount, timestamp }); + + // 2 alert should be suppressed: 1 doc and 1 doc2 + await indexListOfSourceDocuments([doc1, doc1, doc2, doc2]); + + await addThreatDocuments({ + id, + timestamp, + fields: { + host: { + name: 'host-a', + }, + }, + count: 1, + }); + + const rule: ThreatMatchRuleCreateProps = { + ...indicatorMatchRule(id), + alert_suppression: { + group_by: ['agent.name'], + missing_fields_strategy: 'suppress', + }, + from: 'now-35m', + interval: '30m', + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T07:00:00.000Z'), + invocationCount: 1, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: ['agent.name', ALERT_ORIGINAL_TIME], + }); + expect(previewAlerts.length).toEqual(2); + expect(previewAlerts[0]._source).toEqual({ + ...previewAlerts[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'agent.name', + value: 'agent-1', + }, + ], + [TIMESTAMP]: '2020-10-28T07:00:00.000Z', + [ALERT_LAST_DETECTED]: '2020-10-28T07:00:00.000Z', + [ALERT_ORIGINAL_TIME]: timestamp, + [ALERT_SUPPRESSION_START]: '2020-10-28T07:00:00.000Z', + [ALERT_SUPPRESSION_END]: '2020-10-28T07:00:00.000Z', + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }); + + expect(previewAlerts[1]._source).toEqual({ + ...previewAlerts[1]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'agent.name', + value: null, + }, + ], + [TIMESTAMP]: '2020-10-28T07:00:00.000Z', + [ALERT_LAST_DETECTED]: '2020-10-28T07:00:00.000Z', + [ALERT_ORIGINAL_TIME]: timestamp, + [ALERT_SUPPRESSION_START]: '2020-10-28T07:00:00.000Z', + [ALERT_SUPPRESSION_END]: '2020-10-28T07:00:00.000Z', + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }); + }); + + it('should not suppress alerts with missing fields during rule execution only if configured so', async () => { + const id = uuidv4(); + const timestamp = '2020-10-28T06:45:00.000Z'; + const doc1 = { + id, + '@timestamp': timestamp, + host: { name: 'host-a' }, + }; + + const doc2 = { + ...doc1, + agent: { name: 'agent-1' }, + }; + + await eventsFiller({ id, count: eventsCount, timestamp: [timestamp] }); + await threatsFiller({ id, count: threatsCount, timestamp }); + + // 1 alert should be suppressed: 1 doc only + await indexListOfSourceDocuments([doc1, doc1, doc2, doc2]); + + await addThreatDocuments({ + id, + timestamp, + fields: { + host: { + name: 'host-a', + }, + }, + count: 1, + }); + + const rule: ThreatMatchRuleCreateProps = { + ...indicatorMatchRule(id), + alert_suppression: { + group_by: ['agent.name'], + missing_fields_strategy: 'doNotSuppress', + }, + from: 'now-35m', + interval: '30m', + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T07:00:00.000Z'), + invocationCount: 1, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: ['agent.name', ALERT_ORIGINAL_TIME], + }); + expect(previewAlerts.length).toEqual(3); + expect(previewAlerts[0]._source).toEqual({ + ...previewAlerts[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'agent.name', + value: 'agent-1', + }, + ], + [TIMESTAMP]: '2020-10-28T07:00:00.000Z', + [ALERT_LAST_DETECTED]: '2020-10-28T07:00:00.000Z', + [ALERT_ORIGINAL_TIME]: timestamp, + [ALERT_SUPPRESSION_START]: '2020-10-28T07:00:00.000Z', + [ALERT_SUPPRESSION_END]: '2020-10-28T07:00:00.000Z', + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }); + + // rest of alerts are not suppressed and do not have suppress properties + previewAlerts.slice(1).forEach((previewAlert) => { + const source = previewAlert._source; + expect(source).toHaveProperty('id', id); + expect(source).not.toHaveProperty(ALERT_SUPPRESSION_DOCS_COUNT); + expect(source).not.toHaveProperty(ALERT_SUPPRESSION_END); + expect(source).not.toHaveProperty(ALERT_SUPPRESSION_TERMS); + expect(source).not.toHaveProperty(ALERT_SUPPRESSION_DOCS_COUNT); + }); + }); + }); + }); + }); + }); +}; diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/get_threat_match_rule_for_alert_testing.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/get_threat_match_rule_for_alert_testing.ts index b0435ccaaaba0..5537929033dfd 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/get_threat_match_rule_for_alert_testing.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/get_threat_match_rule_for_alert_testing.ts @@ -25,6 +25,7 @@ export const getThreatMatchRuleForAlertTesting = ( language: 'kuery', query: '*:*', threat_query: '*:*', + threat_language: 'kuery', threat_mapping: [ // We match host.name against host.name { From d0e65b3f2b9e455f2cc501c5605847cda25e8252 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Wed, 17 Jan 2024 15:19:50 +0000 Subject: [PATCH 13/28] [CI] Auto-commit changed files from 'node scripts/eslint --no-cache --fix' --- .../rule_creation_ui/components/description_step/index.tsx | 7 +++---- .../components/step_define_rule/schema.tsx | 3 +-- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/description_step/index.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/description_step/index.tsx index a599f86ad8d78..e662d4aaabf10 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/description_step/index.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/description_step/index.tsx @@ -221,9 +221,7 @@ export const getDescriptionItem = ( } else if (field === 'groupByDuration') { const ruleType: Type = get('ruleType', data); const ruleCanHaveDuration = - isQueryRule(ruleType) || - isThresholdRule(ruleType) || - isThreatMatchRule(ruleType); + isQueryRule(ruleType) || isThresholdRule(ruleType) || isThreatMatchRule(ruleType); if (!ruleCanHaveDuration) { return []; } @@ -246,7 +244,8 @@ export const getDescriptionItem = ( } } else if (field === 'suppressionMissingFields') { const ruleType: Type = get('ruleType', data); - const ruleCanHaveSuppressionMissingFields = isQueryRule(ruleType) || isThreatMatchRule(ruleType); + const ruleCanHaveSuppressionMissingFields = + isQueryRule(ruleType) || isThreatMatchRule(ruleType); if (!ruleCanHaveSuppressionMissingFields) { return []; } diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/schema.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/schema.tsx index c5002f7adec4b..8a43698ab5ad4 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/schema.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/schema.tsx @@ -604,8 +604,7 @@ export const schema: FormSchema = { ): ReturnType> | undefined => { const [{ formData }] = args; const needsValidation = - isQueryRule(formData.ruleType) || - isThreatMatchRule(formData.ruleType); + isQueryRule(formData.ruleType) || isThreatMatchRule(formData.ruleType); if (!needsValidation) { return; } From 86320f0840dd8ed672465b05c1675f713b6a7abc Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com> Date: Wed, 24 Jan 2024 10:40:53 +0000 Subject: [PATCH 14/28] Update indicator_match_alert_suppression.ts --- .../execution_logic/indicator_match_alert_suppression.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_execution_logic/execution_logic/indicator_match_alert_suppression.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_execution_logic/execution_logic/indicator_match_alert_suppression.ts index 3d5712e1bc066..d3006c2ffa003 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_execution_logic/execution_logic/indicator_match_alert_suppression.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_execution_logic/execution_logic/indicator_match_alert_suppression.ts @@ -152,7 +152,7 @@ export default ({ getService }: FtrProviderContext) => { }, ]; - describe.only('@ess @serverless Indicator match type rules, alert suppression', () => { + describe('@ess @serverless Indicator match type rules, alert suppression', () => { before(async () => { await esArchiver.load('x-pack/test/functional/es_archives/security_solution/ecs_compliant'); }); From a7a42812d9a0eba6657532bab8ae7222936ec4fe Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com> Date: Fri, 26 Jan 2024 18:32:44 +0000 Subject: [PATCH 15/28] [Security Solution][Detection Engine] adds backend implementation for IM alert suppression (#175032) ## Summary Summarize your PR. If it involves visual changes include a screenshot or gif. ### Checklist Delete any items that are not applicable to this PR. - [ ] Any text added follows [EUI's writing guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses sentence case text and includes [i18n support](https://github.com/elastic/kibana/blob/main/packages/kbn-i18n/README.md) - [ ] [Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html) was added for features that require explanation or tutorials - [ ] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios - [ ] [Flaky Test Runner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1) was used on any tests changed - [ ] Any UI touched in this PR is usable by keyboard only (learn more about [keyboard accessibility](https://webaim.org/techniques/keyboard/)) - [ ] Any UI touched in this PR does not create any new axe failures (run axe in browser: [FF](https://addons.mozilla.org/en-US/firefox/addon/axe-devtools/), [Chrome](https://chrome.google.com/webstore/detail/axe-web-accessibility-tes/lhdoppojpmngadmnindnejefpokejbdd?hl=en-US)) - [ ] If a plugin configuration key changed, check if it needs to be allowlisted in the cloud and added to the [docker list](https://github.com/elastic/kibana/blob/main/src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker) - [ ] This renders correctly on smaller devices using a responsive layout. (You can test this [in your browser](https://www.browserstack.com/guide/responsive-testing-on-local-server)) - [ ] This was checked for [cross-browser compatibility](https://www.elastic.co/support/matrix#matrix_browsers) ### Risk Matrix Delete this section if it is not applicable to this PR. Before closing this PR, invite QA, stakeholders, and other developers to identify risks that should be tested prior to the change/feature release. When forming the risk matrix, consider some of the following examples and how they may potentially impact the change: | Risk | Probability | Severity | Mitigation/Notes | |---------------------------|-------------|----------|-------------------------| | Multiple Spaces—unexpected behavior in non-default Kibana Space. | Low | High | Integration tests will verify that all features are still supported in non-default Kibana Space and when user switches between spaces. | | Multiple nodes—Elasticsearch polling might have race conditions when multiple Kibana nodes are polling for the same tasks. | High | Low | Tasks are idempotent, so executing them multiple times will not result in logical error, but will degrade performance. To test for this case we add plenty of unit tests around this logic and document manual testing procedure. | | Code should gracefully handle cases when feature X or plugin Y are disabled. | Medium | High | Unit tests will verify that any feature flag or plugin combination still results in our service operational. | | [See more potential risk examples](https://github.com/elastic/kibana/blob/main/RISK_MATRIX.mdx) | ### For maintainers - [ ] This was checked for breaking API changes and was [labeled appropriately](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process) --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- .../common/schemas/8.13.0/index.ts | 31 + .../rule_registry/common/schemas/index.ts | 12 +- ...eate_persistence_rule_type_wrapper.test.ts | 356 ++++++ .../create_persistence_rule_type_wrapper.ts | 295 ++++- .../server/utils/persistence_types.ts | 10 +- .../factories/bulk_create_factory.ts | 1 + .../create_indicator_match_alert_type.ts | 29 +- .../indicator_match/indicator_match.ts | 12 +- .../threat_mapping/create_event_signal.ts | 90 +- .../threat_mapping/create_threat_signal.ts | 86 +- .../threat_mapping/create_threat_signals.ts | 23 +- .../indicator_match/threat_mapping/types.ts | 12 + .../threat_mapping/utils.test.ts | 94 ++ .../indicator_match/threat_mapping/utils.ts | 5 + .../rule_types/threshold/threshold.ts | 21 +- .../lib/detection_engine/rule_types/types.ts | 7 + .../utils/bulk_create_with_suppression.ts | 11 +- .../utils/partition_missing_fields_events.ts | 32 + .../utils/search_after_bulk_create.ts | 225 +--- .../utils/search_after_bulk_create_factory.ts | 215 ++++ ...rch_after_bulk_create_suppressed_alerts.ts | 120 ++ .../rule_types/utils/utils.test.ts | 7 + .../rule_types/utils/utils.ts | 14 + .../utils/wrap_suppressed_alerts.ts | 120 ++ .../config/ess/config.base.ts | 1 + .../configs/serverless.config.ts | 1 + .../execution_logic/index.ts | 2 +- ...n.ts => threat_match_alert_suppression.ts} | 1013 ++++++++++++++++- 28 files changed, 2473 insertions(+), 372 deletions(-) create mode 100644 x-pack/plugins/rule_registry/common/schemas/8.13.0/index.ts create mode 100644 x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.test.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/partition_missing_fields_events.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/search_after_bulk_create_factory.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/search_after_bulk_create_suppressed_alerts.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts rename x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/{indicator_match_alert_suppression.ts => threat_match_alert_suppression.ts} (56%) diff --git a/x-pack/plugins/rule_registry/common/schemas/8.13.0/index.ts b/x-pack/plugins/rule_registry/common/schemas/8.13.0/index.ts new file mode 100644 index 0000000000000..a3ed01d861e4e --- /dev/null +++ b/x-pack/plugins/rule_registry/common/schemas/8.13.0/index.ts @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ALERT_SUPPRESSION_TERMS } from '@kbn/rule-data-utils'; +import { AlertWithCommonFields880 } from '../8.8.0'; + +import { SuppressionFields870 } from '../8.7.0'; + +/* DO NOT MODIFY THIS SCHEMA TO ADD NEW FIELDS. These types represent the alerts that shipped in 8.7.0. +Any changes to these types should be bug fixes so the types more accurately represent the alerts from 8.7.0. + +If you are adding new fields for a new release of Kibana, create a new sibling folder to this one +for the version to be released and add the field(s) to the schema in that folder. + +Then, update `../index.ts` to import from the new folder that has the latest schemas, add the +new schemas to the union of all alert schemas, and re-export the new schemas as the `*Latest` schemas. +*/ + +export interface SuppressionFields8130 + extends Omit { + [ALERT_SUPPRESSION_TERMS]: Array<{ + field: string; + value: string | number | null | string[] | number[]; + }>; +} + +export type AlertWithSuppressionFields8130 = AlertWithCommonFields880 & SuppressionFields8130; diff --git a/x-pack/plugins/rule_registry/common/schemas/index.ts b/x-pack/plugins/rule_registry/common/schemas/index.ts index 95b0bf6914aaf..5c168a4b899cc 100644 --- a/x-pack/plugins/rule_registry/common/schemas/index.ts +++ b/x-pack/plugins/rule_registry/common/schemas/index.ts @@ -5,11 +5,7 @@ * 2.0. */ -import type { - AlertWithSuppressionFields870, - SuppressionFields870, - CommonAlertIdFieldName870, -} from './8.7.0'; +import type { CommonAlertIdFieldName870 } from './8.7.0'; import type { AlertWithCommonFields880, @@ -17,9 +13,11 @@ import type { CommonAlertFields880, } from './8.8.0'; +import type { AlertWithSuppressionFields8130, SuppressionFields8130 } from './8.13.0'; + export type { - AlertWithSuppressionFields870 as AlertWithSuppressionFieldsLatest, - SuppressionFields870 as SuppressionFieldsLatest, + AlertWithSuppressionFields8130 as AlertWithSuppressionFieldsLatest, + SuppressionFields8130 as SuppressionFieldsLatest, CommonAlertFieldName880 as CommonAlertFieldNameLatest, CommonAlertIdFieldName870 as CommonAlertIdFieldNameLatest, CommonAlertFields880 as CommonAlertFieldsLatest, diff --git a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.test.ts b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.test.ts new file mode 100644 index 0000000000000..7e1e420859087 --- /dev/null +++ b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.test.ts @@ -0,0 +1,356 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; + +import { + ALERT_INSTANCE_ID, + ALERT_SUPPRESSION_DOCS_COUNT, + ALERT_SUPPRESSION_END, + ALERT_SUPPRESSION_START, + ALERT_RULE_EXECUTION_UUID, +} from '@kbn/rule-data-utils'; + +import { + suppressAlertsInMemory, + isExistingDateGtEqThanAlert, + getUpdatedSuppressionBoundaries, + BackendAlertWithSuppressionFields870, +} from './create_persistence_rule_type_wrapper'; + +describe('suppressAlertsInMemory', () => { + it('should correctly suppress alerts', () => { + const alerts = [ + { + _id: 'alert-a', + _source: { + [ALERT_SUPPRESSION_DOCS_COUNT]: 10, + [ALERT_INSTANCE_ID]: 'instance-id-1', + [ALERT_SUPPRESSION_START]: new Date('2020-10-28T05:45:00.000Z'), + [ALERT_SUPPRESSION_END]: new Date('2020-10-28T05:45:00.000Z'), + }, + }, + { + _id: 'alert-b', + _source: { + [ALERT_SUPPRESSION_DOCS_COUNT]: 2, + [ALERT_INSTANCE_ID]: 'instance-id-1', + [ALERT_SUPPRESSION_START]: new Date('2020-10-28T05:15:00.000Z'), + [ALERT_SUPPRESSION_END]: new Date('2020-10-28T05:15:00.000Z'), + }, + }, + { + _id: 'alert-c', + _source: { + [ALERT_SUPPRESSION_DOCS_COUNT]: 0, + [ALERT_INSTANCE_ID]: 'instance-id-1', + [ALERT_SUPPRESSION_START]: new Date('2020-10-28T05:18:00.000Z'), + [ALERT_SUPPRESSION_END]: new Date('2020-10-28T05:18:00.000Z'), + }, + }, + ]; + + const { alertCandidates, suppressedAlerts } = suppressAlertsInMemory(alerts); + + // 1 alert left, rest suppressed + expect(alertCandidates.length).toBe(1); + expect(suppressedAlerts.length).toBe(2); + + // 1 suppressed alert only + expect(alertCandidates[0]._source[ALERT_SUPPRESSION_DOCS_COUNT]).toBe(14); + + // alert candidate should be alert with oldest suppression date + expect(alertCandidates[0]._id).toBe('alert-b'); + expect(alertCandidates[0]._source[ALERT_SUPPRESSION_START].toISOString()).toBe( + '2020-10-28T05:15:00.000Z' + ); + // suppression end should be latest date + expect(alertCandidates[0]._source[ALERT_SUPPRESSION_END].toISOString()).toBe( + '2020-10-28T05:45:00.000Z' + ); + }); + + it('should suppress by multiple instance ids', () => { + const alerts = [ + { + _id: 'alert-a', + _source: { + [ALERT_SUPPRESSION_DOCS_COUNT]: 10, + [ALERT_INSTANCE_ID]: 'instance-id-1', + [ALERT_SUPPRESSION_START]: new Date('2020-10-28T05:45:00.000Z'), + [ALERT_SUPPRESSION_END]: new Date('2020-10-28T05:45:00.000Z'), + }, + }, + { + _id: 'alert-b', + _source: { + [ALERT_SUPPRESSION_DOCS_COUNT]: 2, + [ALERT_INSTANCE_ID]: 'instance-id-1', + [ALERT_SUPPRESSION_START]: new Date('2020-10-28T05:15:00.000Z'), + [ALERT_SUPPRESSION_END]: new Date('2020-10-28T05:15:00.000Z'), + }, + }, + { + _id: 'alert-c', + _source: { + [ALERT_SUPPRESSION_DOCS_COUNT]: 0, + [ALERT_INSTANCE_ID]: 'instance-id-1', + [ALERT_SUPPRESSION_START]: new Date('2020-10-28T05:18:00.000Z'), + [ALERT_SUPPRESSION_END]: new Date('2020-10-28T05:18:00.000Z'), + }, + }, + { + _id: 'alert-0', + _source: { + [ALERT_SUPPRESSION_DOCS_COUNT]: 0, + [ALERT_INSTANCE_ID]: 'instance-id-2', + [ALERT_SUPPRESSION_START]: new Date('2020-10-28T05:18:00.000Z'), + [ALERT_SUPPRESSION_END]: new Date('2020-10-28T05:18:00.000Z'), + }, + }, + { + _id: 'alert-0', + _source: { + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + [ALERT_INSTANCE_ID]: 'instance-id-2', + [ALERT_SUPPRESSION_START]: new Date('2020-10-28T05:18:00.000Z'), + [ALERT_SUPPRESSION_END]: new Date('2020-10-28T05:18:00.000Z'), + }, + }, + ]; + + const { alertCandidates, suppressedAlerts } = suppressAlertsInMemory(alerts); + + // 1 alert left, rest suppressed + expect(alertCandidates.length).toBe(2); + expect(suppressedAlerts.length).toBe(3); + + // 'instance-id-1' + expect(alertCandidates[0]._source[ALERT_SUPPRESSION_DOCS_COUNT]).toBe(14); + // 'instance-id-2', + expect(alertCandidates[1]._source[ALERT_SUPPRESSION_DOCS_COUNT]).toBe(2); + }); + it('should not suppress alerts if no common instance ids', () => { + const alerts = [ + { + _id: 'alert-a', + _source: { + [ALERT_SUPPRESSION_DOCS_COUNT]: 10, + [ALERT_INSTANCE_ID]: 'instance-id-1', + [ALERT_SUPPRESSION_START]: new Date('2020-10-28T05:45:00.000Z'), + [ALERT_SUPPRESSION_END]: new Date('2020-10-28T05:45:00.000Z'), + }, + }, + { + _id: 'alert-b', + _source: { + [ALERT_SUPPRESSION_DOCS_COUNT]: 2, + [ALERT_INSTANCE_ID]: 'instance-id-2', + [ALERT_SUPPRESSION_START]: new Date('2020-10-28T05:15:00.000Z'), + [ALERT_SUPPRESSION_END]: new Date('2020-10-28T05:15:00.000Z'), + }, + }, + { + _id: 'alert-c', + _source: { + [ALERT_SUPPRESSION_DOCS_COUNT]: 0, + [ALERT_INSTANCE_ID]: 'instance-id-3', + [ALERT_SUPPRESSION_START]: new Date('2020-10-28T05:18:00.000Z'), + [ALERT_SUPPRESSION_END]: new Date('2020-10-28T05:18:00.000Z'), + }, + }, + ]; + + const { alertCandidates, suppressedAlerts } = suppressAlertsInMemory(alerts); + + // no alerts should be suppressed + expect(alertCandidates.length).toBe(3); + expect(suppressedAlerts.length).toBe(0); + }); +}); + +describe('isExistingDateGtEqThanAlert', () => { + it('should return false if existing alert source is undefined', () => { + expect( + isExistingDateGtEqThanAlert( + { _source: undefined, _id: 'a1', _index: 'test-index' }, + { + _id: 'alert-a', + _source: { + [ALERT_SUPPRESSION_START]: new Date('2020-10-28T05:45:00.000Z'), + [ALERT_SUPPRESSION_END]: new Date('2020-10-28T05:45:00.000Z'), + }, + }, + ALERT_SUPPRESSION_START + ) + ).toBe(false); + }); + it('should return false if existing alert date is older', () => { + expect( + isExistingDateGtEqThanAlert( + { + _source: { [ALERT_SUPPRESSION_START]: '2020-10-28T05:42:00.000Z' }, + _id: 'a1', + _index: 'test-index', + } as estypes.SearchHit>, + { + _id: 'alert-a', + _source: { + [ALERT_SUPPRESSION_START]: new Date('2020-10-28T05:45:00.000Z'), + [ALERT_SUPPRESSION_END]: new Date('2020-10-28T05:45:00.000Z'), + }, + }, + ALERT_SUPPRESSION_START + ) + ).toBe(false); + }); + + it('should return true if existing alert date is greater', () => { + expect( + isExistingDateGtEqThanAlert( + { + _source: { [ALERT_SUPPRESSION_START]: '2020-10-28T05:50:00.000Z' }, + _id: 'a1', + _index: 'test-index', + } as estypes.SearchHit>, + { + _id: 'alert-a', + _source: { + [ALERT_SUPPRESSION_START]: new Date('2020-10-28T05:45:00.000Z'), + [ALERT_SUPPRESSION_END]: new Date('2020-10-28T05:45:00.000Z'), + }, + }, + ALERT_SUPPRESSION_START + ) + ).toBe(true); + }); + + it('should return true if existing alert date is the same as alert', () => { + expect( + isExistingDateGtEqThanAlert( + { + _source: { [ALERT_SUPPRESSION_START]: '2020-10-28T05:42:00.000Z' }, + _id: 'a1', + _index: 'test-index', + } as estypes.SearchHit>, + { + _id: 'alert-a', + _source: { + [ALERT_SUPPRESSION_START]: new Date('2020-10-28T05:42:00.000Z'), + [ALERT_SUPPRESSION_END]: new Date('2020-10-28T05:45:00.000Z'), + }, + }, + ALERT_SUPPRESSION_START + ) + ).toBe(true); + }); +}); + +describe('getUpdatedSuppressionBoundaries', () => { + it('should not return suppression end if existing alert has later date', () => { + const boundaries = getUpdatedSuppressionBoundaries( + { + _source: { [ALERT_SUPPRESSION_END]: '2020-10-28T06:00:00.000Z' }, + _id: 'a1', + _index: 'test-index', + } as estypes.SearchHit>, + { + _id: 'alert-a', + _source: { + [ALERT_SUPPRESSION_START]: new Date('2020-10-28T05:42:00.000Z'), + [ALERT_SUPPRESSION_END]: new Date('2020-10-28T05:45:00.000Z'), + }, + }, + 'mock-id' + ); + expect(boundaries[ALERT_SUPPRESSION_END]).toBeUndefined(); + }); + + it('should return updated suppression end if existing alert has older date', () => { + const boundaries = getUpdatedSuppressionBoundaries( + { + _source: { [ALERT_SUPPRESSION_END]: '2020-10-28T05:00:00.000Z' }, + _id: 'a1', + _index: 'test-index', + } as estypes.SearchHit>, + { + _id: 'alert-a', + _source: { + [ALERT_SUPPRESSION_START]: new Date('2020-10-28T05:42:00.000Z'), + [ALERT_SUPPRESSION_END]: new Date('2020-10-28T05:45:00.000Z'), + }, + }, + 'mock-id' + ); + expect(boundaries[ALERT_SUPPRESSION_END]?.toISOString()).toBe('2020-10-28T05:45:00.000Z'); + }); + + it('should not return suppression start if existing alert has older date and matches rule execution id', () => { + const boundaries = getUpdatedSuppressionBoundaries( + { + _source: { + [ALERT_SUPPRESSION_START]: '2020-10-28T05:00:00.000Z', + [ALERT_RULE_EXECUTION_UUID]: 'mock-id', + }, + _id: 'a1', + _index: 'test-index', + } as estypes.SearchHit>, + { + _id: 'alert-a', + _source: { + [ALERT_SUPPRESSION_START]: new Date('2020-10-28T05:42:00.000Z'), + [ALERT_SUPPRESSION_END]: new Date('2020-10-28T05:45:00.000Z'), + }, + }, + 'mock-id' + ); + expect(boundaries[ALERT_SUPPRESSION_START]).toBeUndefined(); + }); + + it('should return updated suppression start if existing alert has later date and matches rule execution id', () => { + const boundaries = getUpdatedSuppressionBoundaries( + { + _source: { + [ALERT_SUPPRESSION_START]: '2020-10-28T06:00:00.000Z', + [ALERT_RULE_EXECUTION_UUID]: 'mock-id', + }, + _id: 'a1', + _index: 'test-index', + } as estypes.SearchHit>, + { + _id: 'alert-a', + _source: { + [ALERT_SUPPRESSION_START]: new Date('2020-10-28T05:42:00.000Z'), + [ALERT_SUPPRESSION_END]: new Date('2020-10-28T05:45:00.000Z'), + }, + }, + 'mock-id' + ); + expect(boundaries[ALERT_SUPPRESSION_START]?.toISOString()).toBe('2020-10-28T05:42:00.000Z'); + }); + + it('should not return suppression start if existing alert has later date but not matches rule execution id', () => { + const boundaries = getUpdatedSuppressionBoundaries( + { + _source: { + [ALERT_SUPPRESSION_START]: '2020-10-28T06:00:00.000Z', + [ALERT_RULE_EXECUTION_UUID]: 'mock-id-prev-rule-execution', + }, + _id: 'a1', + _index: 'test-index', + } as estypes.SearchHit>, + { + _id: 'alert-a', + _source: { + [ALERT_SUPPRESSION_START]: new Date('2020-10-28T05:42:00.000Z'), + [ALERT_SUPPRESSION_END]: new Date('2020-10-28T05:45:00.000Z'), + }, + }, + 'mock-id' + ); + expect(boundaries[ALERT_SUPPRESSION_START]).toBeUndefined(); + }); +}); diff --git a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts index 31304339c1600..4798f29062c6d 100644 --- a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts +++ b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts @@ -5,6 +5,7 @@ * 2.0. */ +import sortBy from 'lodash/sortBy'; import dateMath from '@elastic/datemath'; import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { RuleExecutorOptions } from '@kbn/alerting-plugin/server'; @@ -17,17 +18,31 @@ import { ALERT_START, ALERT_SUPPRESSION_DOCS_COUNT, ALERT_SUPPRESSION_END, + ALERT_SUPPRESSION_START, ALERT_UUID, + ALERT_RULE_EXECUTION_UUID, ALERT_WORKFLOW_STATUS, TIMESTAMP, VERSION, } from '@kbn/rule-data-utils'; import { mapKeys, snakeCase } from 'lodash/fp'; +import type { IRuleDataClient } from '..'; import { getCommonAlertFields } from './get_common_alert_fields'; import { CreatePersistenceRuleTypeWrapper } from './persistence_types'; import { errorAggregator } from './utils'; import { AlertWithSuppressionFields870 } from '../../common/schemas/8.7.0'; +/** + * alerts returned from BE have date type coerce to ISO strings + */ +export type BackendAlertWithSuppressionFields870 = Omit< + AlertWithSuppressionFields870, + typeof ALERT_SUPPRESSION_START | typeof ALERT_SUPPRESSION_END +> & { + [ALERT_SUPPRESSION_START]: string; + [ALERT_SUPPRESSION_END]: string; +}; + export const ALERT_GROUP_INDEX = `${ALERT_NAMESPACE}.group.index` as const; const augmentAlerts = ({ @@ -63,6 +78,158 @@ const mapAlertsToBulkCreate = (alerts: Array<{ _id: string; _source: T }>) => return alerts.flatMap((alert) => [{ create: { _id: alert._id } }, alert._source]); }; +const filterDuplicateAlerts = async ({ + alerts, + spaceId, + ruleDataClient, +}: { + alerts: T[]; + spaceId: string; + ruleDataClient: IRuleDataClient; +}) => { + const CHUNK_SIZE = 10000; + const alertChunks = chunk(alerts, CHUNK_SIZE); + const filteredAlerts: typeof alerts = []; + + for (const alertChunk of alertChunks) { + const request: estypes.SearchRequest = { + body: { + query: { + ids: { + values: alertChunk.map((alert) => alert._id), + }, + }, + aggs: { + uuids: { + terms: { + field: ALERT_UUID, + size: CHUNK_SIZE, + }, + }, + }, + size: 0, + }, + }; + const response = await ruleDataClient.getReader({ namespace: spaceId }).search(request); + const uuidsMap: Record = {}; + const aggs = response.aggregations as + | Record }> + | undefined; + if (aggs != null) { + aggs.uuids.buckets.forEach((bucket) => (uuidsMap[bucket.key] = true)); + const newAlerts = alertChunk.filter((alert) => !uuidsMap[alert._id]); + filteredAlerts.push(...newAlerts); + } else { + filteredAlerts.push(...alertChunk); + } + } + + return filteredAlerts; +}; + +/** + * suppress alerts by ALERT_INSTANCE_ID in memory + */ +export const suppressAlertsInMemory = < + T extends { + _id: string; + _source: { + [ALERT_SUPPRESSION_DOCS_COUNT]: number; + [ALERT_INSTANCE_ID]: string; + [ALERT_SUPPRESSION_START]: Date; + [ALERT_SUPPRESSION_END]: Date; + }; + } +>( + alerts: T[] +): { + alertCandidates: T[]; + suppressedAlerts: T[]; +} => { + const idsMap: Record = {}; + const suppressedAlerts: T[] = []; + + const filteredAlerts = sortBy(alerts, (alert) => alert._source[ALERT_SUPPRESSION_START]).filter( + (alert) => { + const instanceId = alert._source[ALERT_INSTANCE_ID]; + const suppressionDocsCount = alert._source[ALERT_SUPPRESSION_DOCS_COUNT]; + const suppressionEnd = alert._source[ALERT_SUPPRESSION_END]; + + if (instanceId && idsMap[instanceId] != null) { + idsMap[instanceId].count += suppressionDocsCount + 1; + // store the max value of suppression end boundary + if (suppressionEnd > idsMap[instanceId].suppressionEnd) { + idsMap[instanceId].suppressionEnd = suppressionEnd; + } + suppressedAlerts.push(alert); + return false; + } else { + idsMap[instanceId] = { count: suppressionDocsCount, suppressionEnd }; + return true; + } + }, + [] + ); + + const alertCandidates = filteredAlerts.map((alert) => { + const instanceId = alert._source[ALERT_INSTANCE_ID]; + if (instanceId) { + alert._source[ALERT_SUPPRESSION_DOCS_COUNT] = idsMap[instanceId].count; + alert._source[ALERT_SUPPRESSION_END] = idsMap[instanceId].suppressionEnd; + } + return alert; + }); + + return { + alertCandidates, + suppressedAlerts, + }; +}; + +/** + * Compare existing alert suppression date props with alert to suppressed alert values + **/ +export const isExistingDateGtEqThanAlert = < + T extends { [ALERT_SUPPRESSION_END]: Date; [ALERT_SUPPRESSION_START]: Date } +>( + existingAlert: estypes.SearchHit>, + alert: { _id: string; _source: T }, + property: typeof ALERT_SUPPRESSION_END | typeof ALERT_SUPPRESSION_START +) => { + const existingDate = existingAlert?._source?.[property]; + return existingDate ? existingDate >= alert._source[property].toISOString() : false; +}; + +interface SuppressionBoundaries { + [ALERT_SUPPRESSION_END]: Date; + [ALERT_SUPPRESSION_START]: Date; +} + +/** + * returns updated suppression time boundaries + */ +export const getUpdatedSuppressionBoundaries = ( + existingAlert: estypes.SearchHit>, + alert: { _id: string; _source: T }, + executionId: string +): Partial => { + const boundaries: Partial = {}; + + if (!isExistingDateGtEqThanAlert(existingAlert, alert, ALERT_SUPPRESSION_END)) { + boundaries[ALERT_SUPPRESSION_END] = alert._source[ALERT_SUPPRESSION_END]; + } + // start date can only be updated for alert created in the same rule execution + // it can happen when alert was created in first bulk created, but some of the alerts can be suppressed in the next bulk create request + if ( + existingAlert?._source?.[ALERT_RULE_EXECUTION_UUID] === executionId && + isExistingDateGtEqThanAlert(existingAlert, alert, ALERT_SUPPRESSION_START) + ) { + boundaries[ALERT_SUPPRESSION_START] = alert._source[ALERT_SUPPRESSION_START]; + } + + return boundaries; +}; + export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper = ({ logger, ruleDataClient, formatAlert }) => (type) => { @@ -91,44 +258,11 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper ruleDataClient.isWriteEnabled() && options.services.shouldWriteAlerts(); if (writeAlerts && numAlerts) { - const CHUNK_SIZE = 10000; - const alertChunks = chunk(alerts, CHUNK_SIZE); - const filteredAlerts: typeof alerts = []; - - for (const alertChunk of alertChunks) { - const request: estypes.SearchRequest = { - body: { - query: { - ids: { - values: alertChunk.map((alert) => alert._id), - }, - }, - aggs: { - uuids: { - terms: { - field: ALERT_UUID, - size: CHUNK_SIZE, - }, - }, - }, - size: 0, - }, - }; - const response = await ruleDataClient - .getReader({ namespace: options.spaceId }) - .search(request); - const uuidsMap: Record = {}; - const aggs = response.aggregations as - | Record }> - | undefined; - if (aggs != null) { - aggs.uuids.buckets.forEach((bucket) => (uuidsMap[bucket.key] = true)); - const newAlerts = alertChunk.filter((alert) => !uuidsMap[alert._id]); - filteredAlerts.push(...newAlerts); - } else { - filteredAlerts.push(...alertChunk); - } - } + const filteredAlerts: typeof alerts = await filterDuplicateAlerts({ + alerts, + ruleDataClient, + spaceId: options.spaceId, + }); if (filteredAlerts.length === 0) { return { createdAlerts: [], errors: {}, alertsWereTruncated: false }; @@ -219,7 +353,8 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper alerts, suppressionWindow, enrichAlerts, - currentTimeOverride + currentTimeOverride, + isRuleExecutionOnly ) => { const ruleDataClientWriter = await ruleDataClient.getWriter({ namespace: options.spaceId, @@ -238,13 +373,24 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper const suppressionWindowStart = dateMath.parse(suppressionWindow, { forceNow: currentTimeOverride, }); + if (!suppressionWindowStart) { throw new Error('Failed to parse suppression window'); } + const filteredDuplicates = await filterDuplicateAlerts({ + alerts, + ruleDataClient, + spaceId: options.spaceId, + }); + + if (filteredDuplicates.length === 0) { + return { createdAlerts: [], errors: {}, suppressedAlerts: [] }; + } + const suppressionAlertSearchRequest = { body: { - size: alerts.length, + size: filteredDuplicates.length, query: { bool: { filter: [ @@ -257,7 +403,7 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper }, { terms: { - [ALERT_INSTANCE_ID]: alerts.map( + [ALERT_INSTANCE_ID]: filteredDuplicates.map( (alert) => alert._source['kibana.alert.instance.id'] ), }, @@ -287,33 +433,65 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper }, }; - // We use AlertWithSuppressionFields870 explicitly here as the type instead of + // We use BackendAlertWithSuppressionFields870 explicitly here as the type instead of // AlertWithSuppressionFieldsLatest since we're reading alerts rather than writing, // so future versions of Kibana may read 8.7.0 version alerts and need to update them const response = await ruleDataClient .getReader({ namespace: options.spaceId }) - .search>( - suppressionAlertSearchRequest - ); + .search< + typeof suppressionAlertSearchRequest, + BackendAlertWithSuppressionFields870<{}> + >(suppressionAlertSearchRequest); const existingAlertsByInstanceId = response.hits.hits.reduce< - Record>> + Record>> >((acc, hit) => { - acc[hit._source['kibana.alert.instance.id']] = hit; + acc[hit._source[ALERT_INSTANCE_ID]] = hit; return acc; }, {}); - const [duplicateAlerts, newAlerts] = partition( - alerts, - (alert) => - existingAlertsByInstanceId[alert._source['kibana.alert.instance.id']] != null - ); + const nonSuppressedAlerts = filteredDuplicates.filter((alert) => { + const existingAlert = + existingAlertsByInstanceId[alert._source[ALERT_INSTANCE_ID]]; + + // if existing alert was generated earlier during rule execution, it means new ones are not suppressed yet + if ( + !existingAlert || + existingAlert?._source?.[ALERT_RULE_EXECUTION_UUID] === options.executionId + ) { + return true; + } + + return !isExistingDateGtEqThanAlert(existingAlert, alert, ALERT_SUPPRESSION_END); + }); + + if (nonSuppressedAlerts.length === 0) { + return { createdAlerts: [], errors: {}, suppressedAlerts: [] }; + } + + const { alertCandidates, suppressedAlerts: suppressedInMemoryAlerts } = + suppressAlertsInMemory(nonSuppressedAlerts); + + const [duplicateAlerts, newAlerts] = partition(alertCandidates, (alert) => { + const existingAlert = + existingAlertsByInstanceId[alert._source[ALERT_INSTANCE_ID]]; + + // if suppression enabled only on rule execution, we need to account for alerts created earlier + if (isRuleExecutionOnly) { + return ( + existingAlert?._source?.[ALERT_RULE_EXECUTION_UUID] === options.executionId + ); + } else { + return existingAlert != null; + } + }); const duplicateAlertUpdates = duplicateAlerts.flatMap((alert) => { const existingAlert = - existingAlertsByInstanceId[alert._source['kibana.alert.instance.id']]; + existingAlertsByInstanceId[alert._source[ALERT_INSTANCE_ID]]; const existingDocsCount = existingAlert._source?.[ALERT_SUPPRESSION_DOCS_COUNT] ?? 0; + return [ { update: { @@ -324,8 +502,12 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper }, { doc: { + ...getUpdatedSuppressionBoundaries( + existingAlert, + alert, + options.executionId + ), [ALERT_LAST_DETECTED]: currentTimeOverride ?? new Date(), - [ALERT_SUPPRESSION_END]: alert._source[ALERT_SUPPRESSION_END], [ALERT_SUPPRESSION_DOCS_COUNT]: existingDocsCount + alert._source[ALERT_SUPPRESSION_DOCS_COUNT] + 1, }, @@ -358,7 +540,7 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper }); if (bulkResponse == null) { - return { createdAlerts: [], errors: {} }; + return { createdAlerts: [], errors: {}, suppressedAlerts: [] }; } const createdAlerts = augmentedAlerts @@ -402,11 +584,12 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper return { createdAlerts, + suppressedAlerts: [...duplicateAlerts, ...suppressedInMemoryAlerts], errors: errorAggregator(bulkResponse.body, [409]), }; } else { logger.debug('Writing is disabled.'); - return { createdAlerts: [], errors: {} }; + return { createdAlerts: [], errors: {}, suppressedAlerts: [] }; } }, }, diff --git a/x-pack/plugins/rule_registry/server/utils/persistence_types.ts b/x-pack/plugins/rule_registry/server/utils/persistence_types.ts index de34484b5fbfa..1506ad1dd1109 100644 --- a/x-pack/plugins/rule_registry/server/utils/persistence_types.ts +++ b/x-pack/plugins/rule_registry/server/utils/persistence_types.ts @@ -51,8 +51,14 @@ export type SuppressedAlertService = ( alerts: Array<{ _id: string; _source: T }>, params: { spaceId: string } ) => Promise>, - currentTimeOverride?: Date -) => Promise, 'alertsWereTruncated'>>; + currentTimeOverride?: Date, + isRuleExecutionOnly?: boolean +) => Promise>; + +export interface SuppressedAlertServiceResult + extends Omit, 'alertsWereTruncated'> { + suppressedAlerts: Array<{ _id: string; _source: T }>; +} export interface PersistenceAlertServiceResult { createdAlerts: Array & { _id: string; _index: string }>; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/bulk_create_factory.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/bulk_create_factory.ts index 603ff74cf2704..822d0314375d4 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/bulk_create_factory.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/bulk_create_factory.ts @@ -27,6 +27,7 @@ export interface GenericBulkCreateResponse { createdItems: Array & { _id: string; _index: string }>; errors: string[]; alertsWereTruncated: boolean; + suppressedItemsCount?: number; } export const bulkCreateFactory = diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/create_indicator_match_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/create_indicator_match_alert_type.ts index 1c3907d8109a3..0b073d6c6b710 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/create_indicator_match_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/create_indicator_match_alert_type.ts @@ -12,13 +12,15 @@ import { SERVER_APP_ID } from '../../../../../common/constants'; import { ThreatRuleParams } from '../../rule_schema'; import { indicatorMatchExecutor } from './indicator_match'; -import type { CreateRuleOptions, SecurityAlertType } from '../types'; +import type { CreateRuleOptions, SecurityAlertType, SignalSourceHit } from '../types'; import { validateIndexPatterns } from '../utils'; +import { wrapSuppressedAlerts } from '../utils/wrap_suppressed_alerts'; +import type { BuildReasonMessage } from '../utils/reason_formatters'; export const createIndicatorMatchAlertType = ( createOptions: CreateRuleOptions ): SecurityAlertType => { - const { eventsTelemetry, version } = createOptions; + const { eventsTelemetry, version, licensing } = createOptions; return { id: INDICATOR_RULE_TYPE_ID, name: 'Indicator Match Rule', @@ -74,8 +76,28 @@ export const createIndicatorMatchAlertType = ( inputIndexFields, }, services, + spaceId, state, } = execOptions; + const runOpts = execOptions.runOpts; + + const wrapSuppressedHits = ( + events: SignalSourceHit[], + buildReasonMessage: BuildReasonMessage + ) => + wrapSuppressedAlerts({ + events, + spaceId, + completeRule, + mergeStrategy: runOpts.mergeStrategy, + indicesToQuery: runOpts.inputIndex, + buildReasonMessage, + alertTimestampOverride: runOpts.alertTimestampOverride, + ruleExecutionLogger, + publicBaseUrl: runOpts.publicBaseUrl, + primaryTimestamp, + secondaryTimestamp, + }); const result = await indicatorMatchExecutor({ inputIndex, @@ -95,6 +117,9 @@ export const createIndicatorMatchAlertType = ( exceptionFilter, unprocessedExceptions, inputIndexFields, + wrapSuppressedHits, + runOpts, + licensing, }); return { ...result, state }; }, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/indicator_match.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/indicator_match.ts index 94d7d8360dfb2..029aee57d8025 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/indicator_match.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/indicator_match.ts @@ -7,6 +7,7 @@ import type { ExceptionListItemSchema } from '@kbn/securitysolution-io-ts-list-types'; import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import type { LicensingPluginSetup } from '@kbn/licensing-plugin/server'; import type { AlertInstanceContext, @@ -15,7 +16,7 @@ import type { } from '@kbn/alerting-plugin/server'; import type { ListClient } from '@kbn/lists-plugin/server'; import type { Filter, DataViewFieldBase } from '@kbn/es-query'; -import type { RuleRangeTuple, BulkCreate, WrapHits } from '../types'; +import type { RuleRangeTuple, BulkCreate, WrapHits, WrapSuppressedHits, RunOpts } from '../types'; import type { ITelemetryEventsSender } from '../../../telemetry/sender'; import { createThreatSignals } from './threat_mapping/create_threat_signals'; import type { CompleteRule, ThreatRuleParams } from '../../rule_schema'; @@ -41,6 +42,9 @@ export const indicatorMatchExecutor = async ({ exceptionFilter, unprocessedExceptions, inputIndexFields, + wrapSuppressedHits, + runOpts, + licensing, }: { inputIndex: string[]; runtimeMappings: estypes.MappingRuntimeFields | undefined; @@ -59,6 +63,9 @@ export const indicatorMatchExecutor = async ({ exceptionFilter: Filter | undefined; unprocessedExceptions: ExceptionListItemSchema[]; inputIndexFields: DataViewFieldBase[]; + wrapSuppressedHits: WrapSuppressedHits; + runOpts: RunOpts; + licensing: LicensingPluginSetup; }) => { const ruleParams = completeRule.ruleParams; @@ -89,12 +96,15 @@ export const indicatorMatchExecutor = async ({ tuple, type: ruleParams.type, wrapHits, + wrapSuppressedHits, runtimeMappings, primaryTimestamp, secondaryTimestamp, exceptionFilter, unprocessedExceptions, inputIndexFields, + runOpts, + licensing, }); }); }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/create_event_signal.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/create_event_signal.ts index 23f5ea1f746cd..8a487270d5957 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/create_event_signal.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/create_event_signal.ts @@ -5,6 +5,8 @@ * 2.0. */ +import { firstValueFrom } from 'rxjs'; + import { buildThreatMappingFilter } from './build_threat_mapping_filter'; import { getFilter } from '../../utils/get_filter'; import { searchAfterAndBulkCreate } from '../../utils/search_after_bulk_create'; @@ -12,6 +14,7 @@ import { buildReasonMessageForThreatMatchAlert } from '../../utils/reason_format import type { CreateEventSignalOptions } from './types'; import type { SearchAfterAndBulkCreateReturnType } from '../../types'; import { getSignalsQueryMapFromThreatIndex } from './get_signals_map_from_threat_index'; +import { searchAfterAndBulkCreateSuppressedAlerts } from '../../utils/search_after_bulk_create_suppressed_alerts'; import { threatEnrichmentFactory } from './threat_enrichment_factory'; import { getSignalValueMap } from './utils'; @@ -34,6 +37,7 @@ export const createEventSignal = async ({ tuple, type, wrapHits, + wrapSuppressedHits, threatQuery, threatFilters, threatLanguage, @@ -42,6 +46,7 @@ export const createEventSignal = async ({ threatPitId, reassignThreatPitId, runtimeMappings, + runOpts, primaryTimestamp, secondaryTimestamp, exceptionFilter, @@ -50,6 +55,8 @@ export const createEventSignal = async ({ threatMatchedFields, inputIndexFields, threatIndexFields, + completeRule, + licensing, }: CreateEventSignalOptions): Promise => { const threatFiltersFromEvents = buildThreatMappingFilter({ threatMapping, @@ -58,6 +65,9 @@ export const createEventSignal = async ({ allowedFieldsForTermsQuery, }); + const license = await firstValueFrom(licensing.license$); + const hasPlatinumLicense = license.hasAtLeast('platinum'); + if (!threatFiltersFromEvents.query || threatFiltersFromEvents.query?.bool.should.length === 0) { // empty event list and we do not want to return everything as being // a hit so opt to return the existing result. @@ -124,34 +134,70 @@ export const createEventSignal = async ({ threatSearchParams, }); - const result = await searchAfterAndBulkCreate({ - buildReasonMessage: buildReasonMessageForThreatMatchAlert, - bulkCreate, - enrichment, - eventsTelemetry, - exceptionsList: unprocessedExceptions, - filter: esFilter, - inputIndexPattern: inputIndex, - listClient, - pageSize: searchAfterSize, - ruleExecutionLogger, - services, - sortOrder: 'desc', - trackTotalHits: false, - tuple, - wrapHits, - runtimeMappings, - primaryTimestamp, - secondaryTimestamp, - }); + const isAlertSuppressionEnabled = Boolean( + completeRule.ruleParams.alertSuppression?.groupBy?.length + ); + + let createResult: SearchAfterAndBulkCreateReturnType; + if ( + isAlertSuppressionEnabled && + runOpts.experimentalFeatures?.alertSuppressionForIndicatorMatchRuleEnabled && + hasPlatinumLicense + ) { + createResult = await searchAfterAndBulkCreateSuppressedAlerts({ + buildReasonMessage: buildReasonMessageForThreatMatchAlert, + bulkCreate, + enrichment, + eventsTelemetry, + exceptionsList: unprocessedExceptions, + filter: esFilter, + inputIndexPattern: inputIndex, + listClient, + pageSize: searchAfterSize, + ruleExecutionLogger, + services, + sortOrder: 'desc', + trackTotalHits: false, + tuple, + wrapHits, + wrapSuppressedHits, + runtimeMappings, + primaryTimestamp, + secondaryTimestamp, + alertTimestampOverride: runOpts.alertTimestampOverride, + alertWithSuppression: runOpts.alertWithSuppression, + alertSuppression: completeRule.ruleParams.alertSuppression, + }); + } else { + createResult = await searchAfterAndBulkCreate({ + buildReasonMessage: buildReasonMessageForThreatMatchAlert, + bulkCreate, + enrichment, + eventsTelemetry, + exceptionsList: unprocessedExceptions, + filter: esFilter, + inputIndexPattern: inputIndex, + listClient, + pageSize: searchAfterSize, + ruleExecutionLogger, + services, + sortOrder: 'desc', + trackTotalHits: false, + tuple, + wrapHits, + runtimeMappings, + primaryTimestamp, + secondaryTimestamp, + }); + } ruleExecutionLogger.debug( `${ threatFiltersFromEvents.query?.bool.should.length } items have completed match checks and the total times to search were ${ - result.searchAfterTimes.length !== 0 ? result.searchAfterTimes : '(unknown) ' + createResult.searchAfterTimes.length !== 0 ? createResult.searchAfterTimes : '(unknown) ' }ms` ); - return result; + return createResult; } }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/create_threat_signal.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/create_threat_signal.ts index d32c16592aba4..e7f99aa6469d6 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/create_threat_signal.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/create_threat_signal.ts @@ -5,12 +5,15 @@ * 2.0. */ +import { firstValueFrom } from 'rxjs'; + import { buildThreatMappingFilter } from './build_threat_mapping_filter'; import { getFilter } from '../../utils/get_filter'; import { searchAfterAndBulkCreate } from '../../utils/search_after_bulk_create'; import { buildReasonMessageForThreatMatchAlert } from '../../utils/reason_formatters'; import type { CreateThreatSignalOptions } from './types'; import type { SearchAfterAndBulkCreateReturnType } from '../../types'; +import { searchAfterAndBulkCreateSuppressedAlerts } from '../../utils/search_after_bulk_create_suppressed_alerts'; import { buildThreatEnrichment } from './build_threat_enrichment'; export const createThreatSignal = async ({ @@ -34,7 +37,9 @@ export const createThreatSignal = async ({ tuple, type, wrapHits, + wrapSuppressedHits, runtimeMappings, + runOpts, primaryTimestamp, secondaryTimestamp, exceptionFilter, @@ -49,6 +54,7 @@ export const createThreatSignal = async ({ allowedFieldsForTermsQuery, inputIndexFields, threatIndexFields, + licensing, }: CreateThreatSignalOptions): Promise => { const threatFilter = buildThreatMappingFilter({ threatMapping, @@ -57,6 +63,9 @@ export const createThreatSignal = async ({ allowedFieldsForTermsQuery, }); + const license = await firstValueFrom(licensing.license$); + const hasPlatinumLicense = license.hasAtLeast('platinum'); + if (!threatFilter.query || threatFilter.query?.bool.should.length === 0) { // empty threat list and we do not want to return everything as being // a hit so opt to return the existing result. @@ -98,26 +107,63 @@ export const createThreatSignal = async ({ threatIndexFields, }); - const result = await searchAfterAndBulkCreate({ - buildReasonMessage: buildReasonMessageForThreatMatchAlert, - bulkCreate, - enrichment: threatEnrichment, - eventsTelemetry, - exceptionsList: unprocessedExceptions, - filter: esFilter, - inputIndexPattern: inputIndex, - listClient, - pageSize: searchAfterSize, - ruleExecutionLogger, - services, - sortOrder: 'desc', - trackTotalHits: false, - tuple, - wrapHits, - runtimeMappings, - primaryTimestamp, - secondaryTimestamp, - }); + const isAlertSuppressionEnabled = Boolean( + completeRule.ruleParams.alertSuppression?.groupBy?.length + ); + + let result: SearchAfterAndBulkCreateReturnType; + + if ( + isAlertSuppressionEnabled && + runOpts.experimentalFeatures?.alertSuppressionForIndicatorMatchRuleEnabled && + hasPlatinumLicense + ) { + result = await searchAfterAndBulkCreateSuppressedAlerts({ + buildReasonMessage: buildReasonMessageForThreatMatchAlert, + bulkCreate, + enrichment: threatEnrichment, + eventsTelemetry, + exceptionsList: unprocessedExceptions, + filter: esFilter, + inputIndexPattern: inputIndex, + listClient, + pageSize: searchAfterSize, + ruleExecutionLogger, + services, + sortOrder: 'desc', + trackTotalHits: false, + tuple, + wrapHits, + wrapSuppressedHits, + runtimeMappings, + primaryTimestamp, + secondaryTimestamp, + alertTimestampOverride: runOpts.alertTimestampOverride, + alertWithSuppression: runOpts.alertWithSuppression, + alertSuppression: completeRule.ruleParams.alertSuppression, + }); + } else { + result = await searchAfterAndBulkCreate({ + buildReasonMessage: buildReasonMessageForThreatMatchAlert, + bulkCreate, + enrichment: threatEnrichment, + eventsTelemetry, + exceptionsList: unprocessedExceptions, + filter: esFilter, + inputIndexPattern: inputIndex, + listClient, + pageSize: searchAfterSize, + ruleExecutionLogger, + services, + sortOrder: 'desc', + trackTotalHits: false, + tuple, + wrapHits, + runtimeMappings, + primaryTimestamp, + secondaryTimestamp, + }); + } ruleExecutionLogger.debug( `${ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/create_threat_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/create_threat_signals.ts index fc29c3b7c0988..3cd63cd3d9a65 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/create_threat_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/create_threat_signals.ts @@ -56,12 +56,15 @@ export const createThreatSignals = async ({ tuple, type, wrapHits, + wrapSuppressedHits, + runOpts, runtimeMappings, primaryTimestamp, secondaryTimestamp, exceptionFilter, unprocessedExceptions, inputIndexFields, + licensing, }: CreateThreatSignalsOptions): Promise => { const threatMatchedFields = getMatchedFields(threatMapping); const allowedFieldsForTermsQuery = await getAllowedFieldsForTermQuery({ @@ -87,6 +90,7 @@ export const createThreatSignals = async ({ searchAfterTimes: [], lastLookBackDate: null, createdSignalsCount: 0, + suppressedAlertsCount: 0, createdSignals: [], errors: [], warningMessages: [], @@ -177,7 +181,18 @@ export const createThreatSignals = async ({ `bulk create times ${results.bulkCreateTimes}ms,`, `all successes are ${results.success}` ); - if (results.createdSignalsCount >= params.maxSignals) { + // if alerts suppressed it means suppression enabled, so suppression alert limit should be applied (5 * max_signals) + if ( + results.suppressedAlertsCount && + results.suppressedAlertsCount > 0 && + results.suppressedAlertsCount + results.createdSignalsCount >= 5 * params.maxSignals + ) { + // warning should be already set + ruleExecutionLogger.debug( + `Indicator match has reached its max signals count ${params.maxSignals}. Additional documents not checked are ${documentCount}` + ); + break; + } else if (results.createdSignalsCount >= params.maxSignals) { if (results.warningMessages.includes(getMaxSignalsWarning())) { results.warningMessages = uniq(results.warningMessages); } else if (documentCount > 0) { @@ -247,6 +262,7 @@ export const createThreatSignals = async ({ tuple, type, wrapHits, + wrapSuppressedHits, runtimeMappings, primaryTimestamp, secondaryTimestamp, @@ -256,6 +272,8 @@ export const createThreatSignals = async ({ threatMatchedFields, inputIndexFields, threatIndexFields, + runOpts, + licensing, }), }); } else { @@ -302,6 +320,7 @@ export const createThreatSignals = async ({ tuple, type, wrapHits, + wrapSuppressedHits, runtimeMappings, primaryTimestamp, secondaryTimestamp, @@ -317,6 +336,8 @@ export const createThreatSignals = async ({ allowedFieldsForTermsQuery, inputIndexFields, threatIndexFields, + runOpts, + licensing, }), }); } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/types.ts index ff392bc176b6a..ae9548090ea64 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/types.ts @@ -17,6 +17,7 @@ import type { LanguageOrUndefined, Type, } from '@kbn/securitysolution-io-ts-alerting-types'; +import type { LicensingPluginSetup } from '@kbn/licensing-plugin/server'; import type { QueryDslBoolQuery } from '@elastic/elasticsearch/lib/api/types'; import type { ExceptionListItemSchema } from '@kbn/securitysolution-io-ts-list-types'; import type { OpenPointInTimeResponse } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; @@ -34,7 +35,9 @@ import type { RuleRangeTuple, SearchAfterAndBulkCreateReturnType, WrapHits, + WrapSuppressedHits, OverrideBodyQuery, + RunOpts, } from '../../types'; import type { CompleteRule, ThreatRuleParams } from '../../../rule_schema'; import type { IRuleExecutionLogForExecutors } from '../../../rule_monitoring'; @@ -67,12 +70,15 @@ export interface CreateThreatSignalsOptions { tuple: RuleRangeTuple; type: Type; wrapHits: WrapHits; + wrapSuppressedHits: WrapSuppressedHits; runtimeMappings: estypes.MappingRuntimeFields | undefined; primaryTimestamp: string; secondaryTimestamp?: string; exceptionFilter: Filter | undefined; unprocessedExceptions: ExceptionListItemSchema[]; inputIndexFields: DataViewFieldBase[]; + runOpts: RunOpts; + licensing: LicensingPluginSetup; } export interface CreateThreatSignalOptions { @@ -96,6 +102,7 @@ export interface CreateThreatSignalOptions { tuple: RuleRangeTuple; type: Type; wrapHits: WrapHits; + wrapSuppressedHits: WrapSuppressedHits; runtimeMappings: estypes.MappingRuntimeFields | undefined; primaryTimestamp: string; secondaryTimestamp?: string; @@ -112,6 +119,8 @@ export interface CreateThreatSignalOptions { allowedFieldsForTermsQuery: AllowedFieldsForTermsQuery; inputIndexFields: DataViewFieldBase[]; threatIndexFields: DataViewFieldBase[]; + runOpts: RunOpts; + licensing: LicensingPluginSetup; } export interface CreateEventSignalOptions { @@ -134,6 +143,7 @@ export interface CreateEventSignalOptions { tuple: RuleRangeTuple; type: Type; wrapHits: WrapHits; + wrapSuppressedHits: WrapSuppressedHits; threatFilters: unknown[]; threatIndex: ThreatIndex; threatIndicatorPath: ThreatIndicatorPath; @@ -152,6 +162,8 @@ export interface CreateEventSignalOptions { threatMatchedFields: ThreatMatchedFields; inputIndexFields: DataViewFieldBase[]; threatIndexFields: DataViewFieldBase[]; + runOpts: RunOpts; + licensing: LicensingPluginSetup; } type EntryKey = 'field' | 'value'; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/utils.test.ts index 7554222960203..2764316df4ccc 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/utils.test.ts @@ -327,6 +327,7 @@ describe('utils', () => { createdSignals: Array(3).fill(sampleSignalHit()), errors: [], warningMessages: [], + suppressedAlertsCount: 0, }; const combinedResults = combineConcurrentResults(existingResult, []); expect(combinedResults).toEqual(expectedResult); @@ -368,6 +369,96 @@ describe('utils', () => { createdSignals: Array(3).fill(sampleSignalHit()), errors: [], warningMessages: [], + suppressedAlertsCount: 0, + }; + + const combinedResults = combineConcurrentResults(existingResult, [newResult]); + expect(combinedResults).toEqual(expectedResult); + }); + + test('it should combine correctly suppressed alerts count when existing result does not have it', () => { + const existingResult: SearchAfterAndBulkCreateReturnType = { + success: true, + warning: false, + searchAfterTimes: [], + bulkCreateTimes: [], + enrichmentTimes: [], + lastLookBackDate: undefined, + createdSignalsCount: 3, + createdSignals: [], + errors: [], + warningMessages: [], + }; + const newResult: SearchAfterAndBulkCreateReturnType = { + success: true, + warning: false, + searchAfterTimes: [], + bulkCreateTimes: [], + enrichmentTimes: [], + lastLookBackDate: undefined, + createdSignalsCount: 0, + createdSignals: [], + errors: [], + warningMessages: [], + suppressedAlertsCount: 10, + }; + const expectedResult: SearchAfterAndBulkCreateReturnType = { + success: true, + warning: false, + searchAfterTimes: ['0'], + bulkCreateTimes: ['0'], + enrichmentTimes: ['0'], + lastLookBackDate: undefined, + createdSignalsCount: 3, + createdSignals: [], + errors: [], + warningMessages: [], + suppressedAlertsCount: 10, + }; + + const combinedResults = combineConcurrentResults(existingResult, [newResult]); + expect(combinedResults).toEqual(expectedResult); + }); + + test('it should combine correctly suppressed alerts count', () => { + const existingResult: SearchAfterAndBulkCreateReturnType = { + success: true, + warning: false, + searchAfterTimes: [], + bulkCreateTimes: [], + enrichmentTimes: [], + lastLookBackDate: undefined, + createdSignalsCount: 3, + createdSignals: [], + errors: [], + warningMessages: [], + suppressedAlertsCount: 11, + }; + const newResult: SearchAfterAndBulkCreateReturnType = { + success: true, + warning: false, + searchAfterTimes: [], + bulkCreateTimes: [], + enrichmentTimes: [], + lastLookBackDate: undefined, + createdSignalsCount: 0, + createdSignals: [], + errors: [], + warningMessages: [], + suppressedAlertsCount: 10, + }; + const expectedResult: SearchAfterAndBulkCreateReturnType = { + success: true, + warning: false, + searchAfterTimes: ['0'], + bulkCreateTimes: ['0'], + enrichmentTimes: ['0'], + lastLookBackDate: undefined, + createdSignalsCount: 3, + createdSignals: [], + errors: [], + warningMessages: [], + suppressedAlertsCount: 21, }; const combinedResults = combineConcurrentResults(existingResult, [newResult]); @@ -423,6 +514,7 @@ describe('utils', () => { createdSignals: Array(16).fill(sampleSignalHit()), errors: [], warningMessages: [], + suppressedAlertsCount: 0, }; const combinedResults = combineConcurrentResults(existingResult, [newResult1, newResult2]); @@ -478,6 +570,7 @@ describe('utils', () => { createdSignals: Array(16).fill(sampleSignalHit()), errors: [], warningMessages: [], + suppressedAlertsCount: 0, }; const combinedResults = combineConcurrentResults(existingResult, [newResult2, newResult1]); // two array elements are flipped @@ -533,6 +626,7 @@ describe('utils', () => { createdSignals: Array(16).fill(sampleSignalHit()), errors: [], warningMessages: [], + suppressedAlertsCount: 0, }; const combinedResults = combineConcurrentResults(existingResult, [newResult1, newResult2]); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/utils.ts index dd90e52cddd01..cdb2fb808898a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/utils.ts @@ -92,6 +92,8 @@ export const combineResults = ( createdSignals: [...currentResult.createdSignals, ...newResult.createdSignals], warningMessages: [...currentResult.warningMessages, ...newResult.warningMessages], errors: [...new Set([...currentResult.errors, ...newResult.errors])], + suppressedAlertsCount: + (currentResult.suppressedAlertsCount ?? 0) + (newResult.suppressedAlertsCount ?? 0), }); /** @@ -120,6 +122,8 @@ export const combineConcurrentResults = ( createdSignals: [...accum.createdSignals, ...item.createdSignals], warningMessages: [...accum.warningMessages, ...item.warningMessages], errors: [...new Set([...accum.errors, ...item.errors])], + suppressedAlertsCount: + (accum.suppressedAlertsCount ?? 0) + (item.suppressedAlertsCount ?? 0), }; }, { @@ -130,6 +134,7 @@ export const combineConcurrentResults = ( enrichmentTimes: [], lastLookBackDate: undefined, createdSignalsCount: 0, + suppressedAlertsCount: 0, createdSignals: [], errors: [], warningMessages: [], diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/threshold.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/threshold.ts index 72e6311ebdb90..4ade5ff1e2f09 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/threshold.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/threshold.ts @@ -26,8 +26,6 @@ import { findThresholdSignals } from './find_threshold_signals'; import { getThresholdBucketFilters } from './get_threshold_bucket_filters'; import { getThresholdSignalHistory } from './get_threshold_signal_history'; import { bulkCreateSuppressedThresholdAlerts } from './bulk_create_suppressed_threshold_alerts'; -import type { GenericBulkCreateResponse } from '../utils/bulk_create_with_suppression'; -import type { BaseFieldsLatest } from '../../../../../common/api/detection_engine/model/alerts'; import type { BulkCreate, @@ -155,7 +153,6 @@ export const thresholdExecutor = async ({ const alertSuppression = completeRule.ruleParams.alertSuppression; - let createResult: GenericBulkCreateResponse; let newSignalHistory: ThresholdSignalHistory; if (alertSuppression?.duration && hasPlatinumLicense) { @@ -171,13 +168,17 @@ export const thresholdExecutor = async ({ spaceId, runOpts, }); - createResult = suppressedResults.bulkCreateResult; + const createResult = suppressedResults.bulkCreateResult; newSignalHistory = buildThresholdSignalHistory({ alerts: suppressedResults.unsuppressedAlerts, }); + addToSearchAfterReturn({ + current: result, + next: { ...createResult, success: createResult.success && isEmpty(searchErrors) }, + }); } else { - createResult = await bulkCreateThresholdSignals({ + const createResult = await bulkCreateThresholdSignals({ buckets, completeRule, filter: esFilter, @@ -195,12 +196,12 @@ export const thresholdExecutor = async ({ newSignalHistory = buildThresholdSignalHistory({ alerts: transformBulkCreatedItemsToHits(createResult.createdItems), }); - } - addToSearchAfterReturn({ - current: result, - next: { ...createResult, success: createResult.success && isEmpty(searchErrors) }, - }); + addToSearchAfterReturn({ + current: result, + next: { ...createResult, success: createResult.success && isEmpty(searchErrors) }, + }); + } result.errors.push(...previousSearchErrors); result.errors.push(...searchErrors); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts index 4c5aa555ed212..6bd0223652497 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts @@ -10,6 +10,7 @@ import type { Moment } from 'moment'; import type { Logger } from '@kbn/logging'; import type { ExceptionListItemSchema } from '@kbn/securitysolution-io-ts-list-types'; import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import type { SuppressionFieldsLatest } from '@kbn/rule-registry-plugin/common/schemas'; import type { QUERY_RULE_TYPE_ID, SAVED_QUERY_RULE_TYPE_ID } from '@kbn/securitysolution-rules'; @@ -335,6 +336,11 @@ export type WrapHits = ( buildReasonMessage: BuildReasonMessage ) => Array>; +export type WrapSuppressedHits = ( + hits: Array>, + buildReasonMessage: BuildReasonMessage +) => Array>; + export type WrapSequences = ( sequences: Array>, buildReasonMessage: BuildReasonMessage @@ -383,6 +389,7 @@ export interface SearchAfterAndBulkCreateReturnType { createdSignals: unknown[]; errors: string[]; warningMessages: string[]; + suppressedAlertsCount?: number; } // the new fields can be added later if needed diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_with_suppression.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_with_suppression.ts index 7b03211b574b0..5e7d0e9d88454 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_with_suppression.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_with_suppression.ts @@ -27,6 +27,7 @@ export interface GenericBulkCreateResponse { bulkCreateDuration: string; enrichmentDuration: string; createdItemsCount: number; + suppressedItemsCount: number; createdItems: Array & { _id: string; _index: string }>; errors: string[]; } @@ -40,6 +41,7 @@ export const bulkCreateWithSuppression = async < services, suppressionWindow, alertTimestampOverride, + isSuppressionPerRuleExecution, }: { alertWithSuppression: SuppressedAlertService; ruleExecutionLogger: IRuleExecutionLogForExecutors; @@ -47,6 +49,7 @@ export const bulkCreateWithSuppression = async < services: RuleServices; suppressionWindow: string; alertTimestampOverride: Date | undefined; + isSuppressionPerRuleExecution?: boolean; }): Promise> => { if (wrappedDocs.length === 0) { return { @@ -55,6 +58,7 @@ export const bulkCreateWithSuppression = async < enrichmentDuration: '0', bulkCreateDuration: '0', createdItemsCount: 0, + suppressedItemsCount: 0, createdItems: [], }; } @@ -81,7 +85,7 @@ export const bulkCreateWithSuppression = async < } }; - const { createdAlerts, errors } = await alertWithSuppression( + const { createdAlerts, errors, suppressedAlerts } = await alertWithSuppression( wrappedDocs.map((doc) => ({ _id: doc._id, // `fields` should have already been merged into `doc._source` @@ -89,7 +93,8 @@ export const bulkCreateWithSuppression = async < })), suppressionWindow, enrichAlertsWrapper, - alertTimestampOverride + alertTimestampOverride, + isSuppressionPerRuleExecution ); const end = performance.now(); @@ -105,6 +110,7 @@ export const bulkCreateWithSuppression = async < bulkCreateDuration: makeFloatString(end - start), createdItemsCount: createdAlerts.length, createdItems: createdAlerts, + suppressedItemsCount: suppressedAlerts.length, }; } else { return { @@ -114,6 +120,7 @@ export const bulkCreateWithSuppression = async < enrichmentDuration: makeFloatString(enrichmentsTimeFinish - enrichmentsTimeStart), createdItemsCount: createdAlerts.length, createdItems: createdAlerts, + suppressedItemsCount: suppressedAlerts.length, }; } }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/partition_missing_fields_events.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/partition_missing_fields_events.ts new file mode 100644 index 0000000000000..91cbe6ed0ed73 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/partition_missing_fields_events.ts @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import pick from 'lodash/pick'; +import partition from 'lodash/partition'; + +import type { SignalSourceHit } from '../types'; + +/** + * TODO: add description + * @param events + * @param suppressedBy + * @returns + */ +export const partitionMissingFieldsEvents = ( + events: SignalSourceHit[], + suppressedBy: string[] = [] +): SignalSourceHit[][] => { + return partition(events, (event) => { + if (suppressedBy.length === 0) { + return true; + } + const hasMissingFields = + Object.keys(pick(event.fields, suppressedBy)).length < suppressedBy.length; + + return !hasMissingFields; + }); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/search_after_bulk_create.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/search_after_bulk_create.ts index 8e084bd8e35dc..e1888c6f6cadc 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/search_after_bulk_create.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/search_after_bulk_create.ts @@ -5,204 +5,39 @@ * 2.0. */ -import { identity } from 'lodash'; -import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; -import { singleSearchAfter } from './single_search_after'; -import { filterEventsAgainstList } from './large_list_filters/filter_events_against_list'; -import { sendAlertTelemetryEvents } from './send_telemetry_events'; -import { - createSearchAfterReturnType, - createSearchResultReturnType, - createSearchAfterReturnTypeFromResponse, - getTotalHitsValue, - mergeReturns, - mergeSearchResults, - getSafeSortIds, - addToSearchAfterReturn, - getMaxSignalsWarning, -} from './utils'; +import { getMaxSignalsWarning, addToSearchAfterReturn } from './utils'; import type { SearchAfterAndBulkCreateParams, SearchAfterAndBulkCreateReturnType } from '../types'; -import { withSecuritySpan } from '../../../../utils/with_security_span'; import { createEnrichEventsFunction } from './enrichments'; +import type { SearchAfterAndBulkCreateFactoryParams } from './search_after_bulk_create_factory'; +import { searchAfterAndBulkCreateFactory } from './search_after_bulk_create_factory'; // search_after through documents and re-index using bulk endpoint. -export const searchAfterAndBulkCreate = async ({ - buildReasonMessage, - bulkCreate, - enrichment = identity, - eventsTelemetry, - exceptionsList, - filter, - inputIndexPattern, - listClient, - pageSize, - ruleExecutionLogger, - services, - sortOrder, - trackTotalHits, - tuple, - wrapHits, - runtimeMappings, - primaryTimestamp, - secondaryTimestamp, - additionalFilters, -}: SearchAfterAndBulkCreateParams): Promise => { - return withSecuritySpan('searchAfterAndBulkCreate', async () => { - let toReturn = createSearchAfterReturnType(); - let searchingIteration = 0; - - // sortId tells us where to start our next consecutive search_after query - let sortIds: estypes.SortResults | undefined; - let hasSortId = true; // default to true so we execute the search on initial run - - if (tuple == null || tuple.to == null || tuple.from == null) { - ruleExecutionLogger.error( - `missing run options fields: ${!tuple.to ? '"tuple.to"' : ''}, ${ - !tuple.from ? '"tuple.from"' : '' - }` - ); - return createSearchAfterReturnType({ - success: false, - errors: ['malformed date tuple'], - }); - } - - while (toReturn.createdSignalsCount <= tuple.maxSignals) { - const cycleNum = `cycle ${searchingIteration++}`; - try { - let mergedSearchResults = createSearchResultReturnType(); - ruleExecutionLogger.debug( - `[${cycleNum}] Searching events${ - sortIds ? ` after cursor ${JSON.stringify(sortIds)}` : '' - } in index pattern "${inputIndexPattern}"` - ); - - if (hasSortId) { - const { searchResult, searchDuration, searchErrors } = await singleSearchAfter({ - searchAfterSortIds: sortIds, - index: inputIndexPattern, - runtimeMappings, - from: tuple.from.toISOString(), - to: tuple.to.toISOString(), - services, - ruleExecutionLogger, - filter, - pageSize: Math.ceil(Math.min(tuple.maxSignals, pageSize)), - primaryTimestamp, - secondaryTimestamp, - trackTotalHits, - sortOrder, - additionalFilters, - }); - mergedSearchResults = mergeSearchResults([mergedSearchResults, searchResult]); - toReturn = mergeReturns([ - toReturn, - createSearchAfterReturnTypeFromResponse({ - searchResult: mergedSearchResults, - primaryTimestamp, - }), - createSearchAfterReturnType({ - searchAfterTimes: [searchDuration], - errors: searchErrors, - }), - ]); - - // determine if there are any candidate signals to be processed - const totalHits = getTotalHitsValue(mergedSearchResults.hits.total); - const lastSortIds = getSafeSortIds( - searchResult.hits.hits[searchResult.hits.hits.length - 1]?.sort - ); - - if (totalHits === 0 || mergedSearchResults.hits.hits.length === 0) { - ruleExecutionLogger.debug( - `[${cycleNum}] Found 0 events ${ - sortIds ? ` after cursor ${JSON.stringify(sortIds)}` : '' - }` - ); - break; - } else { - ruleExecutionLogger.debug( - `[${cycleNum}] Found ${ - mergedSearchResults.hits.hits.length - } of total ${totalHits} events${ - sortIds ? ` after cursor ${JSON.stringify(sortIds)}` : '' - }, last cursor ${JSON.stringify(lastSortIds)}` - ); - } - - if (lastSortIds != null && lastSortIds.length !== 0) { - sortIds = lastSortIds; - hasSortId = true; - } else { - hasSortId = false; - } - } - - // filter out the search results that match with the values found in the list. - // the resulting set are signals to be indexed, given they are not duplicates - // of signals already present in the signals index. - const [includedEvents, _] = await filterEventsAgainstList({ - listClient, - exceptionsList, - ruleExecutionLogger, - events: mergedSearchResults.hits.hits, - }); - - // only bulk create if there are filteredEvents leftover - // if there isn't anything after going through the value list filter - // skip the call to bulk create and proceed to the next search_after, - // if there is a sort id to continue the search_after with. - if (includedEvents.length !== 0) { - const enrichedEvents = await enrichment(includedEvents); - const wrappedDocs = wrapHits(enrichedEvents, buildReasonMessage); - - const bulkCreateResult = await bulkCreate( - wrappedDocs, - tuple.maxSignals - toReturn.createdSignalsCount, - createEnrichEventsFunction({ - services, - logger: ruleExecutionLogger, - }) - ); - - if (bulkCreateResult.alertsWereTruncated) { - toReturn.warningMessages.push(getMaxSignalsWarning()); - break; - } - - addToSearchAfterReturn({ current: toReturn, next: bulkCreateResult }); - - ruleExecutionLogger.debug( - `[${cycleNum}] Created ${bulkCreateResult.createdItemsCount} alerts from ${enrichedEvents.length} events` - ); - - sendAlertTelemetryEvents( - enrichedEvents, - bulkCreateResult.createdItems, - eventsTelemetry, - ruleExecutionLogger - ); - } - - if (!hasSortId) { - ruleExecutionLogger.debug(`[${cycleNum}] Unable to fetch last event cursor`); - break; - } - } catch (exc: unknown) { - ruleExecutionLogger.error( - 'Unable to extract/process events or create alerts', - JSON.stringify(exc) - ); - return mergeReturns([ - toReturn, - createSearchAfterReturnType({ - success: false, - errors: [`${exc}`], - }), - ]); - } - } - ruleExecutionLogger.debug(`Completed bulk indexing of ${toReturn.createdSignalsCount} alert`); - return toReturn; +export const searchAfterAndBulkCreate = async ( + params: SearchAfterAndBulkCreateParams +): Promise => { + const { wrapHits, bulkCreate, services, buildReasonMessage, ruleExecutionLogger, tuple } = params; + + const bulkCreateExecutor: SearchAfterAndBulkCreateFactoryParams['bulkCreateExecutor'] = async ({ + enrichedEvents, + toReturn, + }) => { + const wrappedDocs = wrapHits(enrichedEvents, buildReasonMessage); + + const bulkCreateResult = await bulkCreate( + wrappedDocs, + tuple.maxSignals - toReturn.createdSignalsCount, + createEnrichEventsFunction({ + services, + logger: ruleExecutionLogger, + }) + ); + addToSearchAfterReturn({ current: toReturn, next: bulkCreateResult }); + return bulkCreateResult; + }; + + return searchAfterAndBulkCreateFactory({ + ...params, + bulkCreateExecutor, + getWarningMessage: getMaxSignalsWarning, }); }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/search_after_bulk_create_factory.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/search_after_bulk_create_factory.ts new file mode 100644 index 0000000000000..2eceb6b6ed072 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/search_after_bulk_create_factory.ts @@ -0,0 +1,215 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { identity } from 'lodash'; +import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import { singleSearchAfter } from './single_search_after'; +import { filterEventsAgainstList } from './large_list_filters/filter_events_against_list'; +import { sendAlertTelemetryEvents } from './send_telemetry_events'; +import { + createSearchAfterReturnType, + createSearchResultReturnType, + createSearchAfterReturnTypeFromResponse, + getTotalHitsValue, + mergeReturns, + mergeSearchResults, + getSafeSortIds, +} from './utils'; +import type { + SearchAfterAndBulkCreateParams, + SearchAfterAndBulkCreateReturnType, + SignalSourceHit, +} from '../types'; +import { withSecuritySpan } from '../../../../utils/with_security_span'; +import type { GenericBulkCreateResponse } from '../factories'; + +import type { BaseFieldsLatest } from '../../../../../common/api/detection_engine/model/alerts'; + +export interface SearchAfterAndBulkCreateFactoryParams extends SearchAfterAndBulkCreateParams { + bulkCreateExecutor: (params: { + enrichedEvents: SignalSourceHit[]; + toReturn: SearchAfterAndBulkCreateReturnType; + }) => Promise>; + // }) => Promise<{ + // bulkCreateResult: GenericBulkCreateResponse; + // alertsWereTruncated: boolean; + // }>; + getWarningMessage: () => string; +} + +export const searchAfterAndBulkCreateFactory = async ({ + enrichment = identity, + eventsTelemetry, + exceptionsList, + filter, + inputIndexPattern, + listClient, + pageSize, + ruleExecutionLogger, + services, + sortOrder, + trackTotalHits, + tuple, + runtimeMappings, + primaryTimestamp, + secondaryTimestamp, + additionalFilters, + bulkCreateExecutor, + getWarningMessage, +}: SearchAfterAndBulkCreateFactoryParams): Promise => { + return withSecuritySpan('searchAfterAndBulkCreate', async () => { + let toReturn = createSearchAfterReturnType(); + let searchingIteration = 0; + + // sortId tells us where to start our next consecutive search_after query + let sortIds: estypes.SortResults | undefined; + let hasSortId = true; // default to true so we execute the search on initial run + + if (tuple == null || tuple.to == null || tuple.from == null) { + ruleExecutionLogger.error( + `missing run options fields: ${!tuple.to ? '"tuple.to"' : ''}, ${ + !tuple.from ? '"tuple.from"' : '' + }` + ); + return createSearchAfterReturnType({ + success: false, + errors: ['malformed date tuple'], + }); + } + + while (toReturn.createdSignalsCount <= tuple.maxSignals) { + const cycleNum = `cycle ${searchingIteration++}`; + try { + let mergedSearchResults = createSearchResultReturnType(); + ruleExecutionLogger.debug( + `[${cycleNum}] Searching events${ + sortIds ? ` after cursor ${JSON.stringify(sortIds)}` : '' + } in index pattern "${inputIndexPattern}"` + ); + + if (hasSortId) { + const { searchResult, searchDuration, searchErrors } = await singleSearchAfter({ + searchAfterSortIds: sortIds, + index: inputIndexPattern, + runtimeMappings, + from: tuple.from.toISOString(), + to: tuple.to.toISOString(), + services, + ruleExecutionLogger, + filter, + pageSize: Math.ceil(Math.min(tuple.maxSignals, pageSize)), + primaryTimestamp, + secondaryTimestamp, + trackTotalHits, + sortOrder, + additionalFilters, + }); + mergedSearchResults = mergeSearchResults([mergedSearchResults, searchResult]); + toReturn = mergeReturns([ + toReturn, + createSearchAfterReturnTypeFromResponse({ + searchResult: mergedSearchResults, + primaryTimestamp, + }), + createSearchAfterReturnType({ + searchAfterTimes: [searchDuration], + errors: searchErrors, + }), + ]); + + // determine if there are any candidate signals to be processed + const totalHits = getTotalHitsValue(mergedSearchResults.hits.total); + const lastSortIds = getSafeSortIds( + searchResult.hits.hits[searchResult.hits.hits.length - 1]?.sort + ); + + if (totalHits === 0 || mergedSearchResults.hits.hits.length === 0) { + ruleExecutionLogger.debug( + `[${cycleNum}] Found 0 events ${ + sortIds ? ` after cursor ${JSON.stringify(sortIds)}` : '' + }` + ); + break; + } else { + ruleExecutionLogger.debug( + `[${cycleNum}] Found ${ + mergedSearchResults.hits.hits.length + } of total ${totalHits} events${ + sortIds ? ` after cursor ${JSON.stringify(sortIds)}` : '' + }, last cursor ${JSON.stringify(lastSortIds)}` + ); + } + + if (lastSortIds != null && lastSortIds.length !== 0) { + sortIds = lastSortIds; + hasSortId = true; + } else { + hasSortId = false; + } + } + + // filter out the search results that match with the values found in the list. + // the resulting set are signals to be indexed, given they are not duplicates + // of signals already present in the signals index. + const [includedEvents, _] = await filterEventsAgainstList({ + listClient, + exceptionsList, + ruleExecutionLogger, + events: mergedSearchResults.hits.hits, + }); + + // only bulk create if there are filteredEvents leftover + // if there isn't anything after going through the value list filter + // skip the call to bulk create and proceed to the next search_after, + // if there is a sort id to continue the search_after with. + if (includedEvents.length !== 0) { + const enrichedEvents = await enrichment(includedEvents); + + const bulkCreateResult = await bulkCreateExecutor({ + enrichedEvents, + toReturn, + }); + + ruleExecutionLogger.debug( + `[${cycleNum}] Created ${bulkCreateResult.createdItemsCount} alerts from ${enrichedEvents.length} events` + ); + + sendAlertTelemetryEvents( + enrichedEvents, + bulkCreateResult.createdItems, + eventsTelemetry, + ruleExecutionLogger + ); + + if (bulkCreateResult.alertsWereTruncated) { + toReturn.warningMessages.push(getWarningMessage()); + break; + } + } + + if (!hasSortId) { + ruleExecutionLogger.debug(`[${cycleNum}] Unable to fetch last event cursor`); + break; + } + } catch (exc: unknown) { + ruleExecutionLogger.error( + 'Unable to extract/process events or create alerts', + JSON.stringify(exc) + ); + return mergeReturns([ + toReturn, + createSearchAfterReturnType({ + success: false, + errors: [`${exc}`], + }), + ]); + } + } + ruleExecutionLogger.debug(`Completed bulk indexing of ${toReturn.createdSignalsCount} alert`); + return toReturn; + }); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/search_after_bulk_create_suppressed_alerts.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/search_after_bulk_create_suppressed_alerts.ts new file mode 100644 index 0000000000000..83a14acf8b625 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/search_after_bulk_create_suppressed_alerts.ts @@ -0,0 +1,120 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SuppressedAlertService } from '@kbn/rule-registry-plugin/server'; + +import { bulkCreateWithSuppression } from './bulk_create_with_suppression'; +import { addToSearchAfterReturn, getSuppressionMaxSignalsWarning } from './utils'; +import type { + SearchAfterAndBulkCreateParams, + SearchAfterAndBulkCreateReturnType, + WrapSuppressedHits, +} from '../types'; + +import { createEnrichEventsFunction } from './enrichments'; +import { AlertSuppressionMissingFieldsStrategyEnum } from '../../../../../common/api/detection_engine/model/rule_schema'; +import type { AlertSuppressionCamel } from '../../../../../common/api/detection_engine/model/rule_schema'; +import { DEFAULT_SUPPRESSION_MISSING_FIELDS_STRATEGY } from '../../../../../common/detection_engine/constants'; +import { partitionMissingFieldsEvents } from './partition_missing_fields_events'; +interface SearchAfterAndBulkCreateSuppressedAlertsParams extends SearchAfterAndBulkCreateParams { + wrapSuppressedHits: WrapSuppressedHits; + alertTimestampOverride: Date | undefined; + alertWithSuppression: SuppressedAlertService; + alertSuppression?: AlertSuppressionCamel; +} + +import type { SearchAfterAndBulkCreateFactoryParams } from './search_after_bulk_create_factory'; +import { searchAfterAndBulkCreateFactory } from './search_after_bulk_create_factory'; + +/** + * search_after through documents and re-index using bulk endpoint + * and suppress alerts + */ +export const searchAfterAndBulkCreateSuppressedAlerts = async ( + params: SearchAfterAndBulkCreateSuppressedAlertsParams +): Promise => { + const { + wrapHits, + bulkCreate, + services, + buildReasonMessage, + ruleExecutionLogger, + tuple, + alertSuppression, + wrapSuppressedHits, + alertWithSuppression, + alertTimestampOverride, + } = params; + + const bulkCreateExecutor: SearchAfterAndBulkCreateFactoryParams['bulkCreateExecutor'] = async ({ + enrichedEvents, + toReturn, + }) => { + // max signals for suppression includes suppressed and created alerts + // this allows to lift max signals limitation to higher value + // and can detects threats beyond default max_signals value + const suppressionMaxSignals = 5 * tuple.maxSignals; + + const suppressionDuration = alertSuppression?.duration; + const suppressionWindow = suppressionDuration + ? `now-${suppressionDuration.value}${suppressionDuration.unit}` + : tuple.from.toISOString(); + + const suppressOnMissingFields = + (alertSuppression?.missingFieldsStrategy ?? DEFAULT_SUPPRESSION_MISSING_FIELDS_STRATEGY) === + AlertSuppressionMissingFieldsStrategyEnum.suppress; + + let suppressibleEvents = enrichedEvents; + if (!suppressOnMissingFields) { + const partitionedEvents = partitionMissingFieldsEvents( + enrichedEvents, + alertSuppression?.groupBy || [] + ); + + const wrappedDocs = wrapHits(partitionedEvents[1], buildReasonMessage); + suppressibleEvents = partitionedEvents[0]; + + const unsuppressedResult = await bulkCreate( + wrappedDocs, + tuple.maxSignals - toReturn.createdSignalsCount, + createEnrichEventsFunction({ + services, + logger: ruleExecutionLogger, + }) + ); + + addToSearchAfterReturn({ current: toReturn, next: unsuppressedResult }); + } + + const wrappedDocs = wrapSuppressedHits(suppressibleEvents, buildReasonMessage); + + const bulkCreateResult = await bulkCreateWithSuppression({ + alertWithSuppression, + ruleExecutionLogger, + wrappedDocs, + services, + suppressionWindow, + alertTimestampOverride, + isSuppressionPerRuleExecution: !suppressionDuration, + }); + + addToSearchAfterReturn({ current: toReturn, next: bulkCreateResult }); + + return { + ...bulkCreateResult, + alertsWereTruncated: + (toReturn.suppressedAlertsCount ?? 0) + toReturn.createdSignalsCount >= + suppressionMaxSignals, + }; + }; + + return searchAfterAndBulkCreateFactory({ + ...params, + bulkCreateExecutor, + getWarningMessage: getSuppressionMaxSignalsWarning, + }); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/utils.test.ts index b2d93868cd976..abf17f9f81b0e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/utils.test.ts @@ -960,6 +960,7 @@ describe('utils', () => { success: true, warning: false, warningMessages: [], + suppressedAlertsCount: 0, }; expect(newSearchResult).toEqual(expected); }); @@ -981,6 +982,7 @@ describe('utils', () => { success: true, warning: false, warningMessages: [], + suppressedAlertsCount: 0, }; expect(newSearchResult).toEqual(expected); }); @@ -1300,6 +1302,7 @@ describe('utils', () => { success: true, warning: false, warningMessages: [], + suppressedAlertsCount: 0, }; expect(searchAfterReturnType).toEqual(expected); }); @@ -1328,6 +1331,7 @@ describe('utils', () => { success: false, warning: true, warningMessages: ['test warning'], + suppressedAlertsCount: 0, }; expect(searchAfterReturnType).toEqual(expected); }); @@ -1349,6 +1353,7 @@ describe('utils', () => { success: true, warning: false, warningMessages: [], + suppressedAlertsCount: 0, }; expect(searchAfterReturnType).toEqual(expected); }); @@ -1368,6 +1373,7 @@ describe('utils', () => { success: true, warning: false, warningMessages: [], + suppressedAlertsCount: 0, }; expect(merged).toEqual(expected); }); @@ -1449,6 +1455,7 @@ describe('utils', () => { success: true, // Defaults to success true is all of it was successful warning: true, warningMessages: ['warning1', 'warning2'], + suppressedAlertsCount: 0, }; expect(merged).toEqual(expected); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/utils.ts index 55728183b8bfd..a3b640f0017dd 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/utils.ts @@ -668,6 +668,7 @@ export const createSearchAfterReturnType = ({ createdSignals, errors, warningMessages, + suppressedAlertsCount, }: { success?: boolean | undefined; warning?: boolean; @@ -679,6 +680,7 @@ export const createSearchAfterReturnType = ({ createdSignals?: unknown[] | undefined; errors?: string[] | undefined; warningMessages?: string[] | undefined; + suppressedAlertsCount?: number | undefined; } = {}): SearchAfterAndBulkCreateReturnType => { return { success: success ?? true, @@ -691,6 +693,7 @@ export const createSearchAfterReturnType = ({ createdSignals: createdSignals ?? [], errors: errors ?? [], warningMessages: warningMessages ?? [], + suppressedAlertsCount: suppressedAlertsCount ?? 0, }; }; @@ -732,6 +735,10 @@ export const addToSearchAfterReturn = ({ current.bulkCreateTimes.push(next.bulkCreateDuration); current.enrichmentTimes.push(next.enrichmentDuration); current.errors = [...new Set([...current.errors, ...next.errors])]; + if (next.suppressedItemsCount != null) { + current.suppressedAlertsCount = + (current.suppressedAlertsCount ?? 0) + next.suppressedItemsCount; + } }; export const mergeReturns = ( @@ -749,6 +756,7 @@ export const mergeReturns = ( createdSignals: existingCreatedSignals, errors: existingErrors, warningMessages: existingWarningMessages, + suppressedAlertsCount: existingSuppressedAlertsCount, }: SearchAfterAndBulkCreateReturnType = prev; const { @@ -762,6 +770,7 @@ export const mergeReturns = ( createdSignals: newCreatedSignals, errors: newErrors, warningMessages: newWarningMessages, + suppressedAlertsCount: newSuppressedAlertsCount, }: SearchAfterAndBulkCreateReturnType = next; return { @@ -775,6 +784,7 @@ export const mergeReturns = ( createdSignals: [...existingCreatedSignals, ...newCreatedSignals], errors: [...new Set([...existingErrors, ...newErrors])], warningMessages: [...existingWarningMessages, ...newWarningMessages], + suppressedAlertsCount: (existingSuppressedAlertsCount ?? 0) + (newSuppressedAlertsCount ?? 0), }; }); }; @@ -973,3 +983,7 @@ export const getUnprocessedExceptionsWarnings = ( export const getMaxSignalsWarning = (): string => { return `This rule reached the maximum alert limit for the rule execution. Some alerts were not created.`; }; + +export const getSuppressionMaxSignalsWarning = (): string => { + return `This rule reached the maximum alert limit for the rule execution. Some alerts were not created or suppressed.`; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts new file mode 100644 index 0000000000000..58b3763136829 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts @@ -0,0 +1,120 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import objectHash from 'object-hash'; +import pick from 'lodash/pick'; +import get from 'lodash/get'; + +import type { SuppressionFieldsLatest } from '@kbn/rule-registry-plugin/common/schemas'; +import { + ALERT_SUPPRESSION_DOCS_COUNT, + ALERT_INSTANCE_ID, + ALERT_SUPPRESSION_TERMS, + ALERT_SUPPRESSION_START, + ALERT_SUPPRESSION_END, + TIMESTAMP, +} from '@kbn/rule-data-utils'; +import type { SignalSourceHit } from '../types'; + +import type { + BaseFieldsLatest, + WrappedFieldsLatest, +} from '../../../../../common/api/detection_engine/model/alerts'; +import type { ConfigType } from '../../../../config'; +import type { CompleteRule, ThreatRuleParams } from '../../rule_schema'; +import type { IRuleExecutionLogForExecutors } from '../../rule_monitoring'; +import { buildBulkBody } from '../factories/utils/build_bulk_body'; + +import type { BuildReasonMessage } from './reason_formatters'; + +/** + * wraps suppressed threshold alerts + * first, transforms aggregation threshold buckets to hits + * creates instanceId hash, which is used to search suppressed on time interval alerts + * populates alert's suppression fields + */ +export const wrapSuppressedAlerts = ({ + events, + spaceId, + completeRule, + mergeStrategy, + indicesToQuery, + buildReasonMessage, + alertTimestampOverride, + ruleExecutionLogger, + publicBaseUrl, + primaryTimestamp, + secondaryTimestamp, +}: { + events: SignalSourceHit[]; + spaceId: string; + completeRule: CompleteRule; + mergeStrategy: ConfigType['alertMergeStrategy']; + indicesToQuery: string[]; + buildReasonMessage: BuildReasonMessage; + alertTimestampOverride: Date | undefined; + ruleExecutionLogger: IRuleExecutionLogForExecutors; + publicBaseUrl: string | undefined; + primaryTimestamp: string; + secondaryTimestamp?: string; +}): Array> => { + const suppressedBy = completeRule?.ruleParams?.alertSuppression?.groupBy ?? []; + + return events.map((event) => { + const suppressedProps = pick(event.fields, suppressedBy) as Record< + string, + string[] | number[] | undefined + >; + const suppressionTerms = suppressedBy.map((field) => ({ + field, + value: suppressedProps[field] ?? null, + })); + + const id = objectHash([ + event._index, + event._id, + `${spaceId}:${completeRule.alertId}`, + suppressionTerms, + ]); + + const instanceId = objectHash([suppressionTerms, completeRule.alertId, spaceId]); + + const baseAlert: BaseFieldsLatest = buildBulkBody( + spaceId, + completeRule, + event, + mergeStrategy, + [], + true, + buildReasonMessage, + indicesToQuery, + alertTimestampOverride, + ruleExecutionLogger, + id, + publicBaseUrl + ); + + const suppressionTime = new Date( + get(event.fields, primaryTimestamp) ?? + (secondaryTimestamp && get(event.fields, secondaryTimestamp)) ?? + baseAlert[TIMESTAMP] + ); + + return { + _id: id, + _index: '', + _source: { + ...baseAlert, + [ALERT_SUPPRESSION_TERMS]: suppressionTerms, + [ALERT_SUPPRESSION_START]: suppressionTime, + [ALERT_SUPPRESSION_END]: suppressionTime, + [ALERT_SUPPRESSION_DOCS_COUNT]: 0, + [ALERT_INSTANCE_ID]: instanceId, + }, + }; + }); +}; diff --git a/x-pack/test/security_solution_api_integration/config/ess/config.base.ts b/x-pack/test/security_solution_api_integration/config/ess/config.base.ts index a73480c051ee4..6a9232050c3f0 100644 --- a/x-pack/test/security_solution_api_integration/config/ess/config.base.ts +++ b/x-pack/test/security_solution_api_integration/config/ess/config.base.ts @@ -82,6 +82,7 @@ export function createTestConfig(options: CreateTestConfigOptions, testFiles?: s 'riskScoringPersistence', 'riskScoringRoutesEnabled', 'entityAnalyticsAssetCriticalityEnabled', + 'alertSuppressionForIndicatorMatchRuleEnabled', ])}`, '--xpack.task_manager.poll_interval=1000', `--xpack.actions.preconfigured=${JSON.stringify({ diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/configs/serverless.config.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/configs/serverless.config.ts index 68969be91f96f..2e3c1e359d252 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/configs/serverless.config.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/configs/serverless.config.ts @@ -19,6 +19,7 @@ export default createTestConfig({ ])}`, // See tests within the file "ignore_fields.ts" which use these values in "alertIgnoreFields" `--xpack.securitySolution.enableExperimental=${JSON.stringify([ 'entityAnalyticsAssetCriticalityEnabled', + 'alertSuppressionForIndicatorMatchRuleEnabled', ])}`, ], }); diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/index.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/index.ts index c5ef010a1bd2d..31c9b648e9cd3 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/index.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/index.ts @@ -15,7 +15,7 @@ export default ({ loadTestFile }: FtrProviderContext): void => { loadTestFile(require.resolve('./new_terms')); loadTestFile(require.resolve('./saved_query')); loadTestFile(require.resolve('./threat_match')); - loadTestFile(require.resolve('./indicator_match_alert_suppression')); + loadTestFile(require.resolve('./threat_match_alert_suppression')); loadTestFile(require.resolve('./threshold')); loadTestFile(require.resolve('./threshold_alert_suppression')); loadTestFile(require.resolve('./non_ecs_fields')); diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/indicator_match_alert_suppression.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/threat_match_alert_suppression.ts similarity index 56% rename from x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/indicator_match_alert_suppression.ts rename to x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/threat_match_alert_suppression.ts index 2b5ed717468b6..413af489079f0 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/indicator_match_alert_suppression.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/threat_match_alert_suppression.ts @@ -15,7 +15,9 @@ import { ALERT_SUPPRESSION_TERMS, ALERT_LAST_DETECTED, TIMESTAMP, + ALERT_START, } from '@kbn/rule-data-utils'; +import { getSuppressionMaxSignalsWarning as getSuppressionMaxAlertsWarning } from '@kbn/security-solution-plugin/server/lib/detection_engine/rule_types/utils/utils'; import { DETECTION_ENGINE_SIGNALS_STATUS_URL as DETECTION_ENGINE_ALERTS_STATUS_URL } from '@kbn/security-solution-plugin/common/constants'; @@ -215,12 +217,12 @@ export default ({ getService }: FtrProviderContext) => { [ALERT_SUPPRESSION_TERMS]: [ { field: 'host.name', - value: 'host-a', + value: ['host-a'], }, ], - // suppression boundaries equal to alert time, since no alert been suppressed - [ALERT_SUPPRESSION_START]: suppressionStart, - [ALERT_SUPPRESSION_END]: suppressionStart, + // suppression boundaries equal to original event time, since no alert been suppressed + [ALERT_SUPPRESSION_START]: firstTimestamp, + [ALERT_SUPPRESSION_END]: firstTimestamp, [ALERT_ORIGINAL_TIME]: firstTimestamp, [TIMESTAMP]: suppressionStart, [ALERT_SUPPRESSION_DOCS_COUNT]: 0, @@ -258,11 +260,12 @@ export default ({ getService }: FtrProviderContext) => { [ALERT_SUPPRESSION_TERMS]: [ { field: 'host.name', - value: 'host-a', + value: ['host-a'], }, ], [ALERT_ORIGINAL_TIME]: firstTimestamp, // timestamp is the same - [ALERT_SUPPRESSION_START]: suppressionStart, // suppression start is the same + [ALERT_SUPPRESSION_START]: firstTimestamp, // suppression start is the same + [ALERT_SUPPRESSION_END]: secondTimestamp, // suppression end is updated [ALERT_SUPPRESSION_DOCS_COUNT]: 2, // 2 alerts from second rule run, that's why 2 suppressed }) ); @@ -357,7 +360,7 @@ export default ({ getService }: FtrProviderContext) => { [ALERT_SUPPRESSION_TERMS]: [ { field: 'host.name', - value: 'host-a', + value: ['host-a'], }, ], [ALERT_ORIGINAL_TIME]: firstTimestamp, @@ -434,12 +437,12 @@ export default ({ getService }: FtrProviderContext) => { [ALERT_SUPPRESSION_TERMS]: [ { field: 'host.name', - value: 'host-a', + value: ['host-a'], }, ], [TIMESTAMP]: '2020-10-28T06:00:00.000Z', - [ALERT_SUPPRESSION_START]: '2020-10-28T06:00:00.000Z', - [ALERT_SUPPRESSION_END]: '2020-10-28T06:00:00.000Z', + [ALERT_SUPPRESSION_START]: firstTimestamp, + [ALERT_SUPPRESSION_END]: firstTimestamp, [ALERT_SUPPRESSION_DOCS_COUNT]: 0, }) ); @@ -449,12 +452,12 @@ export default ({ getService }: FtrProviderContext) => { [ALERT_SUPPRESSION_TERMS]: [ { field: 'host.name', - value: 'host-a', + value: ['host-a'], }, ], [TIMESTAMP]: '2020-10-28T06:30:00.000Z', - [ALERT_SUPPRESSION_START]: '2020-10-28T06:30:00.000Z', - [ALERT_SUPPRESSION_END]: '2020-10-28T06:30:00.000Z', + [ALERT_SUPPRESSION_START]: secondTimestamp, + [ALERT_SUPPRESSION_END]: secondTimestamp, [ALERT_SUPPRESSION_DOCS_COUNT]: 0, }) ); @@ -536,14 +539,15 @@ export default ({ getService }: FtrProviderContext) => { [ALERT_SUPPRESSION_TERMS]: [ { field: 'host.name', - value: 'host-a', + value: ['host-a'], }, ], [TIMESTAMP]: '2020-10-28T06:00:00.000Z', [ALERT_LAST_DETECTED]: '2020-10-28T07:00:00.000Z', // Note: ALERT_LAST_DETECTED gets updated, timestamp does not + [ALERT_START]: '2020-10-28T06:00:00.000Z', [ALERT_ORIGINAL_TIME]: firstTimestamp, - [ALERT_SUPPRESSION_START]: '2020-10-28T06:00:00.000Z', - [ALERT_SUPPRESSION_END]: '2020-10-28T07:00:00.000Z', + [ALERT_SUPPRESSION_START]: firstTimestamp, + [ALERT_SUPPRESSION_END]: thirdTimestamp, [ALERT_SUPPRESSION_DOCS_COUNT]: 4, // in total 4 alert got suppressed: 1 from the first run, 2 from the second, 1 from the third }); }); @@ -634,18 +638,17 @@ export default ({ getService }: FtrProviderContext) => { [ALERT_SUPPRESSION_TERMS]: [ { field: 'host.name', - value: 'host-a', + value: ['host-a'], }, { field: 'agent.version', - value: '1', + value: ['1'], }, ], [TIMESTAMP]: '2020-10-28T06:00:00.000Z', - [ALERT_LAST_DETECTED]: '2020-10-28T06:30:00.000Z', // Note: ALERT_LAST_DETECTED gets updated, timestamp does not [ALERT_ORIGINAL_TIME]: firstTimestamp, - [ALERT_SUPPRESSION_START]: '2020-10-28T06:00:00.000Z', - [ALERT_SUPPRESSION_END]: '2020-10-28T06:30:00.000Z', + [ALERT_SUPPRESSION_START]: firstTimestamp, + [ALERT_SUPPRESSION_END]: secondTimestamp, [ALERT_SUPPRESSION_DOCS_COUNT]: 3, // 3 alerts suppressed from the second run }); expect(previewAlerts[1]._source).toEqual({ @@ -653,18 +656,18 @@ export default ({ getService }: FtrProviderContext) => { [ALERT_SUPPRESSION_TERMS]: [ { field: 'host.name', - value: 'host-a', + value: ['host-a'], }, { field: 'agent.version', - value: '2', + value: ['2'], }, ], [TIMESTAMP]: '2020-10-28T06:00:00.000Z', [ALERT_LAST_DETECTED]: '2020-10-28T06:00:00.000Z', [ALERT_ORIGINAL_TIME]: firstTimestamp, - [ALERT_SUPPRESSION_START]: '2020-10-28T06:00:00.000Z', - [ALERT_SUPPRESSION_END]: '2020-10-28T06:00:00.000Z', + [ALERT_SUPPRESSION_START]: firstTimestamp, + [ALERT_SUPPRESSION_END]: firstTimestamp, [ALERT_SUPPRESSION_DOCS_COUNT]: 0, // no suppressed alerts }); }); @@ -743,12 +746,12 @@ export default ({ getService }: FtrProviderContext) => { [ALERT_SUPPRESSION_TERMS]: [ { field: 'host.name', - value: 'host-a', + value: ['host-a'], }, ], [ALERT_ORIGINAL_TIME]: firstTimestamp, - [ALERT_SUPPRESSION_START]: '2020-10-28T06:00:00.000Z', - [ALERT_SUPPRESSION_END]: '2020-10-28T06:30:00.000Z', + [ALERT_SUPPRESSION_START]: firstTimestamp, + [ALERT_SUPPRESSION_END]: secondTimestamp, [ALERT_SUPPRESSION_DOCS_COUNT]: 1, }); }); @@ -825,7 +828,7 @@ export default ({ getService }: FtrProviderContext) => { [ALERT_SUPPRESSION_TERMS]: [ { field: 'agent.name', - value: 'agent-0', + value: ['agent-0'], }, ], [ALERT_SUPPRESSION_DOCS_COUNT]: 1, @@ -833,7 +836,7 @@ export default ({ getService }: FtrProviderContext) => { ); }); - it('should deduplicate alerts while suppressing new ones', async () => { + it('should deduplicate multiple alerts while suppressing new ones', async () => { const id = uuidv4(); const firstTimestamp = '2020-10-28T05:45:00.000Z'; const secondTimestamp = '2020-10-28T06:10:00.000Z'; @@ -901,16 +904,95 @@ export default ({ getService }: FtrProviderContext) => { [ALERT_SUPPRESSION_TERMS]: [ { field: 'host.name', - value: 'host-a', + value: ['host-a'], }, ], [ALERT_ORIGINAL_TIME]: firstTimestamp, - [ALERT_SUPPRESSION_START]: '2020-10-28T06:00:00.000Z', - [ALERT_SUPPRESSION_END]: '2020-10-28T06:30:00.000Z', + [ALERT_SUPPRESSION_START]: firstTimestamp, + [ALERT_SUPPRESSION_END]: secondTimestamp, + // TODO: fix count, it should be 4 suppressed [ALERT_SUPPRESSION_DOCS_COUNT]: 4, }); }); + it('should deduplicate single alerts while suppressing new ones', async () => { + const id = uuidv4(); + const firstTimestamp = '2020-10-28T05:45:00.000Z'; + const secondTimestamp = '2020-10-28T06:10:00.000Z'; + const doc1 = { + id, + '@timestamp': firstTimestamp, + host: { name: 'host-a' }, + }; + + const doc2 = { + ...doc1, + '@timestamp': secondTimestamp, + }; + + await eventsFiller({ + id, + count: eventsCount, + timestamp: [firstTimestamp, secondTimestamp], + }); + await threatsFiller({ id, count: threatsCount, timestamp: firstTimestamp }); + + // 3 alerts should be suppressed + await indexListOfSourceDocuments([doc1, doc2, doc2, doc2]); + + await addThreatDocuments({ + id, + timestamp: firstTimestamp, + fields: { + host: { + name: 'host-a', + }, + }, + count: 1, + }); + + const rule: ThreatMatchRuleCreateProps = { + ...indicatorMatchRule(id), + alert_suppression: { + group_by: ['host.name'], + duration: { + value: 300, + unit: 'm', + }, + missing_fields_strategy: 'suppress', + }, + // large look-back time covers all docs + from: 'now-1h', + interval: '30m', + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T06:30:00.000Z'), + invocationCount: 2, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: ['host.name', ALERT_ORIGINAL_TIME], + }); + expect(previewAlerts.length).toEqual(1); + expect(previewAlerts[0]._source).toEqual({ + ...previewAlerts[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: ['host-a'], + }, + ], + [ALERT_ORIGINAL_TIME]: firstTimestamp, + [ALERT_SUPPRESSION_START]: firstTimestamp, + [ALERT_SUPPRESSION_END]: secondTimestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 3, + }); + }); + it('should suppress alerts with missing fields', async () => { const id = uuidv4(); const firstTimestamp = '2020-10-28T05:45:00.000Z'; @@ -995,12 +1077,12 @@ export default ({ getService }: FtrProviderContext) => { [ALERT_SUPPRESSION_TERMS]: [ { field: 'agent.name', - value: 'agent-1', + value: ['agent-1'], }, ], [ALERT_ORIGINAL_TIME]: firstTimestamp, - [ALERT_SUPPRESSION_START]: '2020-10-28T06:00:00.000Z', - [ALERT_SUPPRESSION_END]: '2020-10-28T06:30:00.000Z', + [ALERT_SUPPRESSION_START]: firstTimestamp, + [ALERT_SUPPRESSION_END]: secondTimestamp, [ALERT_SUPPRESSION_DOCS_COUNT]: 2, }); @@ -1013,8 +1095,8 @@ export default ({ getService }: FtrProviderContext) => { }, ], [ALERT_ORIGINAL_TIME]: firstTimestamp, - [ALERT_SUPPRESSION_START]: '2020-10-28T06:00:00.000Z', - [ALERT_SUPPRESSION_END]: '2020-10-28T06:30:00.000Z', + [ALERT_SUPPRESSION_START]: firstTimestamp, + [ALERT_SUPPRESSION_END]: secondTimestamp, [ALERT_SUPPRESSION_DOCS_COUNT]: 2, }); }); @@ -1103,12 +1185,12 @@ export default ({ getService }: FtrProviderContext) => { [ALERT_SUPPRESSION_TERMS]: [ { field: 'agent.name', - value: 'agent-1', + value: ['agent-1'], }, ], [ALERT_ORIGINAL_TIME]: firstTimestamp, - [ALERT_SUPPRESSION_START]: '2020-10-28T06:00:00.000Z', - [ALERT_SUPPRESSION_END]: '2020-10-28T06:30:00.000Z', + [ALERT_SUPPRESSION_START]: firstTimestamp, + [ALERT_SUPPRESSION_END]: secondTimestamp, [ALERT_SUPPRESSION_DOCS_COUNT]: 2, }); @@ -1127,11 +1209,16 @@ export default ({ getService }: FtrProviderContext) => { it('should suppress alerts during rule execution only', async () => { const id = uuidv4(); const timestamp = '2020-10-28T06:45:00.000Z'; + const laterTimestamp = '2020-10-28T06:50:00.000Z'; const doc1 = { id, '@timestamp': timestamp, host: { name: 'host-a' }, }; + const doc1WithLaterTimestamp = { + ...doc1, + '@timestamp': laterTimestamp, + }; // doc2 does not generate alert const doc2 = { ...doc1, @@ -1141,7 +1228,71 @@ export default ({ getService }: FtrProviderContext) => { await eventsFiller({ id, count: eventsCount, timestamp: [timestamp] }); await threatsFiller({ id, count: threatsCount, timestamp }); - await indexListOfSourceDocuments([doc1, doc1, doc1, doc2]); + await indexListOfSourceDocuments([doc1, doc1WithLaterTimestamp, doc1, doc2]); + + await addThreatDocuments({ + id, + timestamp, + fields: { + host: { + name: 'host-a', + }, + }, + count: 1, + }); + + const rule: ThreatMatchRuleCreateProps = { + ...indicatorMatchRule(id), + alert_suppression: { + group_by: ['host.name'], + missing_fields_strategy: 'suppress', + }, + from: 'now-35m', + interval: '30m', + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T07:00:00.000Z'), + invocationCount: 1, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: [ALERT_ORIGINAL_TIME], + }); + expect(previewAlerts.length).toEqual(1); + expect(previewAlerts[0]._source).toEqual({ + ...previewAlerts[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: ['host-a'], + }, + ], + [TIMESTAMP]: '2020-10-28T07:00:00.000Z', + [ALERT_LAST_DETECTED]: '2020-10-28T07:00:00.000Z', + [ALERT_ORIGINAL_TIME]: timestamp, + [ALERT_SUPPRESSION_START]: timestamp, + [ALERT_SUPPRESSION_END]: laterTimestamp, // suppression ends with later timestamp + [ALERT_SUPPRESSION_DOCS_COUNT]: 2, + }); + }); + + it('should suppress alerts during rule execution only for array field', async () => { + const id = uuidv4(); + const timestamp = '2020-10-28T06:45:00.000Z'; + const doc1 = { + id, + '@timestamp': timestamp, + host: { name: ['host-a', 'host-b'] }, + }; + + await eventsFiller({ id, count: eventsCount, timestamp: [timestamp] }); + await threatsFiller({ id, count: threatsCount, timestamp }); + + await indexListOfSourceDocuments([doc1, doc1, doc1]); await addThreatDocuments({ id, @@ -1181,18 +1332,152 @@ export default ({ getService }: FtrProviderContext) => { [ALERT_SUPPRESSION_TERMS]: [ { field: 'host.name', - value: 'host-a', + value: ['host-a', 'host-b'], }, ], [TIMESTAMP]: '2020-10-28T07:00:00.000Z', [ALERT_LAST_DETECTED]: '2020-10-28T07:00:00.000Z', [ALERT_ORIGINAL_TIME]: timestamp, - [ALERT_SUPPRESSION_START]: '2020-10-28T07:00:00.000Z', - [ALERT_SUPPRESSION_END]: '2020-10-28T07:00:00.000Z', + [ALERT_SUPPRESSION_START]: timestamp, + [ALERT_SUPPRESSION_END]: timestamp, [ALERT_SUPPRESSION_DOCS_COUNT]: 2, }); }); + it('should suppress alerts with missing fields during rule execution only for multiple suppress by fields', async () => { + const id = uuidv4(); + const timestamp = '2020-10-28T06:45:00.000Z'; + const noMissingFieldsDoc = { + id, + '@timestamp': timestamp, + host: { name: 'host-a' }, + agent: { name: 'agent-a', version: 10 }, + }; + + const missingNameFieldsDoc = { + ...noMissingFieldsDoc, + agent: { version: 10 }, + }; + + const missingVersionFieldsDoc = { + ...noMissingFieldsDoc, + agent: { name: 'agent-a' }, + }; + + const missingAgentFieldsDoc = { + ...noMissingFieldsDoc, + agent: undefined, + }; + + await eventsFiller({ id, count: eventsCount, timestamp: [timestamp] }); + await threatsFiller({ id, count: threatsCount, timestamp }); + + // 4 alerts should be suppressed: 1 for each pair of documents + await indexListOfSourceDocuments([ + noMissingFieldsDoc, + noMissingFieldsDoc, + missingNameFieldsDoc, + missingNameFieldsDoc, + missingVersionFieldsDoc, + missingVersionFieldsDoc, + missingAgentFieldsDoc, + missingAgentFieldsDoc, + ]); + + await addThreatDocuments({ + id, + timestamp, + fields: { + host: { + name: 'host-a', + }, + }, + count: 1, + }); + + const rule: ThreatMatchRuleCreateProps = { + ...indicatorMatchRule(id), + alert_suppression: { + group_by: ['agent.name', 'agent.version'], + missing_fields_strategy: 'suppress', + }, + from: 'now-35m', + interval: '30m', + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T07:00:00.000Z'), + invocationCount: 1, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: ['agent.name', 'agent.version', ALERT_ORIGINAL_TIME], + }); + expect(previewAlerts.length).toEqual(4); + expect(previewAlerts[0]._source).toEqual({ + ...previewAlerts[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'agent.name', + value: ['agent-a'], + }, + { + field: 'agent.version', + value: ['10'], + }, + ], + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }); + + expect(previewAlerts[1]._source).toEqual({ + ...previewAlerts[1]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'agent.name', + value: ['agent-a'], + }, + { + field: 'agent.version', + value: null, + }, + ], + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }); + + expect(previewAlerts[2]._source).toEqual({ + ...previewAlerts[2]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'agent.name', + value: null, + }, + { + field: 'agent.version', + value: ['10'], + }, + ], + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }); + + expect(previewAlerts[3]._source).toEqual({ + ...previewAlerts[3]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'agent.name', + value: null, + }, + { + field: 'agent.version', + value: null, + }, + ], + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }); + }); + it('should suppress alerts with missing fields during rule execution only', async () => { const id = uuidv4(); const timestamp = '2020-10-28T06:45:00.000Z'; @@ -1251,14 +1536,14 @@ export default ({ getService }: FtrProviderContext) => { [ALERT_SUPPRESSION_TERMS]: [ { field: 'agent.name', - value: 'agent-1', + value: ['agent-1'], }, ], [TIMESTAMP]: '2020-10-28T07:00:00.000Z', [ALERT_LAST_DETECTED]: '2020-10-28T07:00:00.000Z', [ALERT_ORIGINAL_TIME]: timestamp, - [ALERT_SUPPRESSION_START]: '2020-10-28T07:00:00.000Z', - [ALERT_SUPPRESSION_END]: '2020-10-28T07:00:00.000Z', + [ALERT_SUPPRESSION_START]: timestamp, + [ALERT_SUPPRESSION_END]: timestamp, [ALERT_SUPPRESSION_DOCS_COUNT]: 1, }); @@ -1273,8 +1558,8 @@ export default ({ getService }: FtrProviderContext) => { [TIMESTAMP]: '2020-10-28T07:00:00.000Z', [ALERT_LAST_DETECTED]: '2020-10-28T07:00:00.000Z', [ALERT_ORIGINAL_TIME]: timestamp, - [ALERT_SUPPRESSION_START]: '2020-10-28T07:00:00.000Z', - [ALERT_SUPPRESSION_END]: '2020-10-28T07:00:00.000Z', + [ALERT_SUPPRESSION_START]: timestamp, + [ALERT_SUPPRESSION_END]: timestamp, [ALERT_SUPPRESSION_DOCS_COUNT]: 1, }); }); @@ -1337,14 +1622,14 @@ export default ({ getService }: FtrProviderContext) => { [ALERT_SUPPRESSION_TERMS]: [ { field: 'agent.name', - value: 'agent-1', + value: ['agent-1'], }, ], [TIMESTAMP]: '2020-10-28T07:00:00.000Z', [ALERT_LAST_DETECTED]: '2020-10-28T07:00:00.000Z', [ALERT_ORIGINAL_TIME]: timestamp, - [ALERT_SUPPRESSION_START]: '2020-10-28T07:00:00.000Z', - [ALERT_SUPPRESSION_END]: '2020-10-28T07:00:00.000Z', + [ALERT_SUPPRESSION_START]: timestamp, + [ALERT_SUPPRESSION_END]: timestamp, [ALERT_SUPPRESSION_DOCS_COUNT]: 1, }); @@ -1358,6 +1643,628 @@ export default ({ getService }: FtrProviderContext) => { expect(source).not.toHaveProperty(ALERT_SUPPRESSION_DOCS_COUNT); }); }); + + it('should not suppress alerts with missing fields during rule execution only if configured so for multiple suppress by fields', async () => { + const id = uuidv4(); + const timestamp = '2020-10-28T06:45:00.000Z'; + const noMissingFieldsDoc = { + id, + '@timestamp': timestamp, + host: { name: 'host-a' }, + agent: { name: 'agent-a', version: 10 }, + }; + + const missingNameFieldsDoc = { + ...noMissingFieldsDoc, + agent: { version: 10 }, + }; + + const missingVersionFieldsDoc = { + ...noMissingFieldsDoc, + agent: { name: 'agent-a' }, + }; + + const missingAgentFieldsDoc = { + ...noMissingFieldsDoc, + agent: undefined, + }; + + await eventsFiller({ id, count: eventsCount, timestamp: [timestamp] }); + await threatsFiller({ id, count: threatsCount, timestamp }); + + // 1 alert should be suppressed: 1 doc only + await indexListOfSourceDocuments([ + noMissingFieldsDoc, + noMissingFieldsDoc, + missingNameFieldsDoc, + missingNameFieldsDoc, + missingVersionFieldsDoc, + missingVersionFieldsDoc, + missingAgentFieldsDoc, + ]); + + await addThreatDocuments({ + id, + timestamp, + fields: { + host: { + name: 'host-a', + }, + }, + count: 1, + }); + + const rule: ThreatMatchRuleCreateProps = { + ...indicatorMatchRule(id), + alert_suppression: { + group_by: ['agent.name', 'agent.version'], + missing_fields_strategy: 'doNotSuppress', + }, + from: 'now-35m', + interval: '30m', + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T07:00:00.000Z'), + invocationCount: 1, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: ['agent.name', 'agent.version', ALERT_ORIGINAL_TIME], + }); + // from 7 injected, only one should be suppressed + expect(previewAlerts.length).toEqual(6); + expect(previewAlerts[0]._source).toEqual({ + ...previewAlerts[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'agent.name', + value: ['agent-a'], + }, + { + field: 'agent.version', + value: ['10'], + }, + ], + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }); + + // rest of alerts are not suppressed and do not have suppress properties + previewAlerts.slice(1).forEach((previewAlert) => { + const source = previewAlert._source; + expect(source).toHaveProperty('id', id); + expect(source).not.toHaveProperty(ALERT_SUPPRESSION_DOCS_COUNT); + expect(source).not.toHaveProperty(ALERT_SUPPRESSION_END); + expect(source).not.toHaveProperty(ALERT_SUPPRESSION_TERMS); + expect(source).not.toHaveProperty(ALERT_SUPPRESSION_DOCS_COUNT); + }); + }); + + it('should deduplicate multiple alerts while suppressing on rule interval only', async () => { + const id = uuidv4(); + const firstTimestamp = '2020-10-28T05:45:00.000Z'; + const secondTimestamp = '2020-10-28T06:10:00.000Z'; + const doc1 = { + id, + '@timestamp': firstTimestamp, + host: { name: 'host-a' }, + }; + + const doc2 = { + ...doc1, + '@timestamp': secondTimestamp, + }; + + await eventsFiller({ + id, + count: eventsCount, + timestamp: [firstTimestamp, secondTimestamp], + }); + await threatsFiller({ id, count: threatsCount, timestamp: firstTimestamp }); + + // 4 alert should be suppressed + await indexListOfSourceDocuments([doc1, doc1, doc2, doc2, doc2]); + + await addThreatDocuments({ + id, + timestamp: firstTimestamp, + fields: { + host: { + name: 'host-a', + }, + }, + count: 1, + }); + + const rule: ThreatMatchRuleCreateProps = { + ...indicatorMatchRule(id), + alert_suppression: { + group_by: ['host.name'], + missing_fields_strategy: 'suppress', + }, + // large look-back time covers all docs + from: 'now-50m', + interval: '30m', + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T06:30:00.000Z'), + invocationCount: 2, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: ['host.name', ALERT_ORIGINAL_TIME], + }); + expect(previewAlerts.length).toEqual(2); + expect(previewAlerts[0]._source).toEqual({ + ...previewAlerts[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: ['host-a'], + }, + ], + [ALERT_ORIGINAL_TIME]: firstTimestamp, + [ALERT_SUPPRESSION_START]: firstTimestamp, + [ALERT_SUPPRESSION_END]: firstTimestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }); + + expect(previewAlerts[1]._source).toEqual({ + ...previewAlerts[1]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: ['host-a'], + }, + ], + [ALERT_ORIGINAL_TIME]: secondTimestamp, + [ALERT_SUPPRESSION_START]: secondTimestamp, + [ALERT_SUPPRESSION_END]: secondTimestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 2, + }); + }); + + it('should deduplicate single alert while suppressing new ones on rule execution', async () => { + const id = uuidv4(); + const firstTimestamp = '2020-10-28T05:45:00.000Z'; + const secondTimestamp = '2020-10-28T06:10:00.000Z'; + const doc1 = { + id, + '@timestamp': firstTimestamp, + host: { name: 'host-a' }, + }; + + const doc2 = { + ...doc1, + '@timestamp': secondTimestamp, + }; + + await eventsFiller({ + id, + count: eventsCount, + timestamp: [firstTimestamp, secondTimestamp], + }); + await threatsFiller({ id, count: threatsCount, timestamp: firstTimestamp }); + + // 3 alerts should be suppressed + await indexListOfSourceDocuments([doc1, doc2, doc2, doc2]); + + await addThreatDocuments({ + id, + timestamp: firstTimestamp, + fields: { + host: { + name: 'host-a', + }, + }, + count: 1, + }); + + const rule: ThreatMatchRuleCreateProps = { + ...indicatorMatchRule(id), + alert_suppression: { + group_by: ['host.name'], + missing_fields_strategy: 'suppress', + }, + // large look-back time covers all docs + from: 'now-1h', + interval: '30m', + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T06:30:00.000Z'), + invocationCount: 2, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: ['host.name', ALERT_ORIGINAL_TIME], + }); + expect(previewAlerts.length).toEqual(2); + expect(previewAlerts[0]._source).toEqual({ + ...previewAlerts[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: ['host-a'], + }, + ], + [ALERT_ORIGINAL_TIME]: firstTimestamp, + [ALERT_SUPPRESSION_START]: firstTimestamp, + [ALERT_SUPPRESSION_END]: firstTimestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 0, + }); + expect(previewAlerts[1]._source).toEqual({ + ...previewAlerts[1]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: ['host-a'], + }, + ], + [ALERT_ORIGINAL_TIME]: secondTimestamp, + [ALERT_SUPPRESSION_START]: secondTimestamp, + [ALERT_SUPPRESSION_END]: secondTimestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 2, + }); + }); + + it('should create and suppress alert on rule execution when alert created on previous execution', async () => { + const id = uuidv4(); + const firstTimestamp = '2020-10-28T05:45:00.000Z'; + const secondTimestamp = '2020-10-28T06:10:00.000Z'; + const doc1 = { + id, + '@timestamp': firstTimestamp, + host: { name: 'host-a' }, + }; + + const doc2 = { + ...doc1, + '@timestamp': secondTimestamp, + }; + + await eventsFiller({ + id, + count: eventsCount, + timestamp: [firstTimestamp, secondTimestamp], + }); + await threatsFiller({ id, count: threatsCount, timestamp: firstTimestamp }); + + // 1 created + 1 suppressed on first run + // 1 created + 2 suppressed on second run + await indexListOfSourceDocuments([doc1, doc1, doc2, doc2, doc2]); + + await addThreatDocuments({ + id, + timestamp: firstTimestamp, + fields: { + host: { + name: 'host-a', + }, + }, + count: 1, + }); + + const rule: ThreatMatchRuleCreateProps = { + ...indicatorMatchRule(id), + alert_suppression: { + group_by: ['host.name'], + missing_fields_strategy: 'suppress', + }, + from: 'now-35m', + interval: '30m', + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T06:30:00.000Z'), + invocationCount: 2, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: ['host.name', ALERT_ORIGINAL_TIME], + }); + expect(previewAlerts.length).toEqual(2); + expect(previewAlerts[0]._source).toEqual({ + ...previewAlerts[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: ['host-a'], + }, + ], + [ALERT_ORIGINAL_TIME]: firstTimestamp, + [ALERT_SUPPRESSION_START]: firstTimestamp, + [ALERT_SUPPRESSION_END]: firstTimestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }); + expect(previewAlerts[1]._source).toEqual({ + ...previewAlerts[1]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: ['host-a'], + }, + ], + [ALERT_ORIGINAL_TIME]: secondTimestamp, + [ALERT_SUPPRESSION_START]: secondTimestamp, + [ALERT_SUPPRESSION_END]: secondTimestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 2, + }); + }); + + it('should not suppress more than limited number (max_signals x5)', async () => { + const id = uuidv4(); + const timestamp = '2020-10-28T06:45:00.000Z'; + + await eventsFiller({ id, count: 100 * eventsCount, timestamp: [timestamp] }); + await threatsFiller({ id, count: 100 * threatsCount, timestamp }); + + await indexGeneratedSourceDocuments({ + docsCount: 700, + seed: (index) => ({ + id, + '@timestamp': `2020-10-28T06:50:00.${index}Z`, + host: { + name: `host-a`, + }, + agent: { name: 'agent-a' }, + }), + }); + + await addThreatDocuments({ + id, + timestamp, + fields: { + host: { + name: 'host-a', + }, + }, + count: 1, + }); + + const rule: ThreatMatchRuleCreateProps = { + ...indicatorMatchRule(id), + alert_suppression: { + group_by: ['agent.name'], + missing_fields_strategy: 'suppress', + }, + from: 'now-35m', + interval: '30m', + }; + + const { previewId, logs } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T07:00:00.000Z'), + invocationCount: 1, + }); + + expect(logs[0].warnings).toEqual( + expect.arrayContaining([getSuppressionMaxAlertsWarning()]) + ); + + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: ['agent.name', ALERT_ORIGINAL_TIME], + }); + expect(previewAlerts.length).toEqual(1); + expect(previewAlerts[0]._source).toEqual({ + ...previewAlerts[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'agent.name', + value: ['agent-a'], + }, + ], + [ALERT_SUPPRESSION_DOCS_COUNT]: 499, + }); + }); + + // 9,000 is the size of chunk that is processed in IM rule + // when number of documents in either of index exceeds this number it may leads to unexpected behavior + // this test added to ensure these cases covered + it('should not suppress more than limited number (max_signals x5) for number of events/threats greater than 9,000', async () => { + const id = uuidv4(); + const timestamp = '2020-10-28T06:45:00.000Z'; + + await eventsFiller({ id, count: 10000 * eventsCount, timestamp: [timestamp] }); + await threatsFiller({ id, count: 10000 * threatsCount, timestamp }); + + await indexGeneratedSourceDocuments({ + docsCount: 15000, + seed: (index) => ({ + id, + '@timestamp': `2020-10-28T06:50:00.${index}Z`, + host: { + name: `host-a`, + }, + agent: { name: 'agent-a' }, + }), + }); + + await addThreatDocuments({ + id, + timestamp, + fields: { + host: { + name: 'host-a', + }, + }, + count: 1, + }); + + const rule: ThreatMatchRuleCreateProps = { + ...indicatorMatchRule(id), + alert_suppression: { + group_by: ['agent.name'], + missing_fields_strategy: 'suppress', + }, + from: 'now-35m', + interval: '30m', + }; + + const { previewId, logs } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T07:00:00.000Z'), + invocationCount: 1, + }); + + expect(logs[0].warnings).toEqual( + expect.arrayContaining([getSuppressionMaxAlertsWarning()]) + ); + + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: ['agent.name', ALERT_ORIGINAL_TIME], + }); + expect(previewAlerts.length).toEqual(1); + expect(previewAlerts[0]._source).toEqual({ + ...previewAlerts[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'agent.name', + value: ['agent-a'], + }, + ], + [ALERT_SUPPRESSION_DOCS_COUNT]: 499, + }); + }); + + it('should detect threats beyond max_signals if large number of alerts suppressed', async () => { + const id = uuidv4(); + const timestamp = '2020-10-28T06:45:00.000Z'; + const doc1 = { + id, + '@timestamp': timestamp, + host: { name: 'host-a' }, + agent: { name: 'agent-b' }, + }; + + const doc2 = { + id, + '@timestamp': timestamp, + host: { name: 'host-c' }, + agent: { name: 'agent-c' }, + }; + + await eventsFiller({ id, count: 20 * eventsCount, timestamp: [timestamp] }); + await threatsFiller({ id, count: 20 * threatsCount, timestamp }); + + await indexGeneratedSourceDocuments({ + docsCount: 150, + seed: (index) => ({ + id, + '@timestamp': `2020-10-28T06:50:00.${100 + index}Z`, + host: { + name: `host-a`, + }, + agent: { name: 'agent-a' }, + }), + }); + + await indexListOfSourceDocuments([doc1, doc1, doc1, doc2, doc2]); + + await addThreatDocuments({ + id, + timestamp, + fields: { + host: { + name: 'host-a', + }, + }, + count: 1, + }); + + await addThreatDocuments({ + id, + timestamp, + fields: { + host: { + name: 'host-c', + }, + }, + count: 1, + }); + + const rule: ThreatMatchRuleCreateProps = { + ...indicatorMatchRule(id), + alert_suppression: { + group_by: ['agent.name'], + missing_fields_strategy: 'suppress', + }, + from: 'now-35m', + interval: '30m', + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T07:00:00.000Z'), + invocationCount: 1, + }); + + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: ['agent.name', ALERT_ORIGINAL_TIME], + }); + + // 3 alerts in total + // 1 + 149 suppressed host-a threat, 'agent-a' suppressed by + // 1 + 2 suppressed host-a threat, 'agent-b' suppressed by + // 1 + 1 suppressed host-c threat, 'agent-c' suppressed by + expect(previewAlerts.length).toEqual(3); + expect(previewAlerts[0]._source).toEqual({ + ...previewAlerts[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'agent.name', + value: ['agent-a'], + }, + ], + [ALERT_SUPPRESSION_START]: '2020-10-28T06:50:00.100Z', + [ALERT_SUPPRESSION_END]: '2020-10-28T06:50:00.249Z', // the largest suppression end boundary + [ALERT_SUPPRESSION_DOCS_COUNT]: 149, + }); + + expect(previewAlerts[1]._source).toEqual({ + ...previewAlerts[1]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'agent.name', + value: ['agent-b'], + }, + ], + [ALERT_SUPPRESSION_DOCS_COUNT]: 2, + }); + + expect(previewAlerts[2]._source).toEqual({ + ...previewAlerts[2]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'agent.name', + value: ['agent-c'], + }, + ], + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }); + }); }); }); }); From e73875004dcb39adfd1f2d643f1ffe31d6d62c81 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com> Date: Mon, 29 Jan 2024 17:24:33 +0000 Subject: [PATCH 16/28] PR review (#175784) ## Summary Summarize your PR. If it involves visual changes include a screenshot or gif. ### Checklist Delete any items that are not applicable to this PR. - [ ] Any text added follows [EUI's writing guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses sentence case text and includes [i18n support](https://github.com/elastic/kibana/blob/main/packages/kbn-i18n/README.md) - [ ] [Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html) was added for features that require explanation or tutorials - [ ] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios - [ ] [Flaky Test Runner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1) was used on any tests changed - [ ] Any UI touched in this PR is usable by keyboard only (learn more about [keyboard accessibility](https://webaim.org/techniques/keyboard/)) - [ ] Any UI touched in this PR does not create any new axe failures (run axe in browser: [FF](https://addons.mozilla.org/en-US/firefox/addon/axe-devtools/), [Chrome](https://chrome.google.com/webstore/detail/axe-web-accessibility-tes/lhdoppojpmngadmnindnejefpokejbdd?hl=en-US)) - [ ] If a plugin configuration key changed, check if it needs to be allowlisted in the cloud and added to the [docker list](https://github.com/elastic/kibana/blob/main/src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker) - [ ] This renders correctly on smaller devices using a responsive layout. (You can test this [in your browser](https://www.browserstack.com/guide/responsive-testing-on-local-server)) - [ ] This was checked for [cross-browser compatibility](https://www.elastic.co/support/matrix#matrix_browsers) ### Risk Matrix Delete this section if it is not applicable to this PR. Before closing this PR, invite QA, stakeholders, and other developers to identify risks that should be tested prior to the change/feature release. When forming the risk matrix, consider some of the following examples and how they may potentially impact the change: | Risk | Probability | Severity | Mitigation/Notes | |---------------------------|-------------|----------|-------------------------| | Multiple Spaces—unexpected behavior in non-default Kibana Space. | Low | High | Integration tests will verify that all features are still supported in non-default Kibana Space and when user switches between spaces. | | Multiple nodes—Elasticsearch polling might have race conditions when multiple Kibana nodes are polling for the same tasks. | High | Low | Tasks are idempotent, so executing them multiple times will not result in logical error, but will degrade performance. To test for this case we add plenty of unit tests around this logic and document manual testing procedure. | | Code should gracefully handle cases when feature X or plugin Y are disabled. | Medium | High | Unit tests will verify that any feature flag or plugin combination still results in our service operational. | | [See more potential risk examples](https://github.com/elastic/kibana/blob/main/RISK_MATRIX.mdx) | ### For maintainers - [ ] This was checked for breaking API changes and was [labeled appropriately](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process) --- .../create_persistence_rule_type_wrapper.ts | 13 +- .../detection_engine/rule_types/constants.ts | 5 + .../threat_mapping/create_event_signal.ts | 60 ++--- .../threat_mapping/create_threat_signal.ts | 60 ++--- .../threat_mapping/create_threat_signals.ts | 27 +- .../partition_missing_fields_events.test.ts | 113 +++++++++ .../utils/partition_missing_fields_events.ts | 7 +- .../utils/search_after_bulk_create_factory.ts | 4 - ...rch_after_bulk_create_suppressed_alerts.ts | 5 +- .../utils/wrap_suppressed_alerts.test.ts | 236 ++++++++++++++++++ .../utils/wrap_suppressed_alerts.ts | 13 +- .../threat_match_alert_suppression.ts | 10 +- 12 files changed, 447 insertions(+), 106 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/partition_missing_fields_events.test.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.test.ts diff --git a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts index 4798f29062c6d..e621890d815a7 100644 --- a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts +++ b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts @@ -33,7 +33,7 @@ import { errorAggregator } from './utils'; import { AlertWithSuppressionFields870 } from '../../common/schemas/8.7.0'; /** - * alerts returned from BE have date type coerce to ISO strings + * alerts returned from BE have date type coerced to ISO strings */ export type BackendAlertWithSuppressionFields870 = Omit< AlertWithSuppressionFields870, @@ -78,6 +78,9 @@ const mapAlertsToBulkCreate = (alerts: Array<{ _id: string; _source: T }>) => return alerts.flatMap((alert) => [{ create: { _id: alert._id } }, alert._source]); }; +/** + * finds if any of alerts has duplicate and filter them out + */ const filterDuplicateAlerts = async ({ alerts, spaceId, @@ -450,11 +453,14 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper return acc; }, {}); + // filter out alerts that were already suppressed + // alert was suppressed if its suppression ends is older than suppression end of existing alert + // if existing alert was created earlier during the same rule execution - then alerts can be counted as not suppressed yet + // as they are processed for the first against this existing alert const nonSuppressedAlerts = filteredDuplicates.filter((alert) => { const existingAlert = existingAlertsByInstanceId[alert._source[ALERT_INSTANCE_ID]]; - // if existing alert was generated earlier during rule execution, it means new ones are not suppressed yet if ( !existingAlert || existingAlert?._source?.[ALERT_RULE_EXECUTION_UUID] === options.executionId @@ -476,7 +482,8 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper const existingAlert = existingAlertsByInstanceId[alert._source[ALERT_INSTANCE_ID]]; - // if suppression enabled only on rule execution, we need to account for alerts created earlier + // if suppression enabled only on rule execution, we need to suppress alerts only against + // alert created in the same rule execution. Otherwise, we need to create a new alert to accommodate per rule execution suppression if (isRuleExecutionOnly) { return ( existingAlert?._source?.[ALERT_RULE_EXECUTION_UUID] === options.executionId diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/constants.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/constants.ts index 0cf5cf1a303a5..ea7240c8a106c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/constants.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/constants.ts @@ -6,3 +6,8 @@ */ export const TIMESTAMP_RUNTIME_FIELD = 'kibana.combined_timestamp' as const; + +/** + * When suppression is enabled, allow to to suppress more than max signals alerts + */ +export const MAX_SIGNALS_SUPPRESSION_MULTIPLIER = 5; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/create_event_signal.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/create_event_signal.ts index 8a487270d5957..211466e17c8f7 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/create_event_signal.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/create_event_signal.ts @@ -139,6 +139,26 @@ export const createEventSignal = async ({ ); let createResult: SearchAfterAndBulkCreateReturnType; + const searchAfterBulkCreateParams = { + buildReasonMessage: buildReasonMessageForThreatMatchAlert, + bulkCreate, + enrichment, + eventsTelemetry, + exceptionsList: unprocessedExceptions, + filter: esFilter, + inputIndexPattern: inputIndex, + listClient, + pageSize: searchAfterSize, + ruleExecutionLogger, + services, + sortOrder: 'desc' as const, + trackTotalHits: false, + tuple, + wrapHits, + runtimeMappings, + primaryTimestamp, + secondaryTimestamp, + }; if ( isAlertSuppressionEnabled && @@ -146,50 +166,14 @@ export const createEventSignal = async ({ hasPlatinumLicense ) { createResult = await searchAfterAndBulkCreateSuppressedAlerts({ - buildReasonMessage: buildReasonMessageForThreatMatchAlert, - bulkCreate, - enrichment, - eventsTelemetry, - exceptionsList: unprocessedExceptions, - filter: esFilter, - inputIndexPattern: inputIndex, - listClient, - pageSize: searchAfterSize, - ruleExecutionLogger, - services, - sortOrder: 'desc', - trackTotalHits: false, - tuple, - wrapHits, + ...searchAfterBulkCreateParams, wrapSuppressedHits, - runtimeMappings, - primaryTimestamp, - secondaryTimestamp, alertTimestampOverride: runOpts.alertTimestampOverride, alertWithSuppression: runOpts.alertWithSuppression, alertSuppression: completeRule.ruleParams.alertSuppression, }); } else { - createResult = await searchAfterAndBulkCreate({ - buildReasonMessage: buildReasonMessageForThreatMatchAlert, - bulkCreate, - enrichment, - eventsTelemetry, - exceptionsList: unprocessedExceptions, - filter: esFilter, - inputIndexPattern: inputIndex, - listClient, - pageSize: searchAfterSize, - ruleExecutionLogger, - services, - sortOrder: 'desc', - trackTotalHits: false, - tuple, - wrapHits, - runtimeMappings, - primaryTimestamp, - secondaryTimestamp, - }); + createResult = await searchAfterAndBulkCreate(searchAfterBulkCreateParams); } ruleExecutionLogger.debug( `${ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/create_threat_signal.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/create_threat_signal.ts index e7f99aa6469d6..c5071605841f5 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/create_threat_signal.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/create_threat_signal.ts @@ -112,6 +112,26 @@ export const createThreatSignal = async ({ ); let result: SearchAfterAndBulkCreateReturnType; + const searchAfterBulkCreateParams = { + buildReasonMessage: buildReasonMessageForThreatMatchAlert, + bulkCreate, + enrichment: threatEnrichment, + eventsTelemetry, + exceptionsList: unprocessedExceptions, + filter: esFilter, + inputIndexPattern: inputIndex, + listClient, + pageSize: searchAfterSize, + ruleExecutionLogger, + services, + sortOrder: 'desc' as const, + trackTotalHits: false, + tuple, + wrapHits, + runtimeMappings, + primaryTimestamp, + secondaryTimestamp, + }; if ( isAlertSuppressionEnabled && @@ -119,50 +139,14 @@ export const createThreatSignal = async ({ hasPlatinumLicense ) { result = await searchAfterAndBulkCreateSuppressedAlerts({ - buildReasonMessage: buildReasonMessageForThreatMatchAlert, - bulkCreate, - enrichment: threatEnrichment, - eventsTelemetry, - exceptionsList: unprocessedExceptions, - filter: esFilter, - inputIndexPattern: inputIndex, - listClient, - pageSize: searchAfterSize, - ruleExecutionLogger, - services, - sortOrder: 'desc', - trackTotalHits: false, - tuple, - wrapHits, + ...searchAfterBulkCreateParams, wrapSuppressedHits, - runtimeMappings, - primaryTimestamp, - secondaryTimestamp, alertTimestampOverride: runOpts.alertTimestampOverride, alertWithSuppression: runOpts.alertWithSuppression, alertSuppression: completeRule.ruleParams.alertSuppression, }); } else { - result = await searchAfterAndBulkCreate({ - buildReasonMessage: buildReasonMessageForThreatMatchAlert, - bulkCreate, - enrichment: threatEnrichment, - eventsTelemetry, - exceptionsList: unprocessedExceptions, - filter: esFilter, - inputIndexPattern: inputIndex, - listClient, - pageSize: searchAfterSize, - ruleExecutionLogger, - services, - sortOrder: 'desc', - trackTotalHits: false, - tuple, - wrapHits, - runtimeMappings, - primaryTimestamp, - secondaryTimestamp, - }); + result = await searchAfterAndBulkCreate(searchAfterBulkCreateParams); } ruleExecutionLogger.debug( diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/create_threat_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/create_threat_signals.ts index 3cd63cd3d9a65..5cb9dddc9b42e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/create_threat_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/create_threat_signals.ts @@ -17,6 +17,7 @@ import type { import { createThreatSignal } from './create_threat_signal'; import { createEventSignal } from './create_event_signal'; import type { SearchAfterAndBulkCreateReturnType } from '../../types'; +import { MAX_SIGNALS_SUPPRESSION_MULTIPLIER } from '../../constants'; import { buildExecutionIntervalValidator, combineConcurrentResults, @@ -181,18 +182,9 @@ export const createThreatSignals = async ({ `bulk create times ${results.bulkCreateTimes}ms,`, `all successes are ${results.success}` ); + // if alerts suppressed it means suppression enabled, so suppression alert limit should be applied (5 * max_signals) - if ( - results.suppressedAlertsCount && - results.suppressedAlertsCount > 0 && - results.suppressedAlertsCount + results.createdSignalsCount >= 5 * params.maxSignals - ) { - // warning should be already set - ruleExecutionLogger.debug( - `Indicator match has reached its max signals count ${params.maxSignals}. Additional documents not checked are ${documentCount}` - ); - break; - } else if (results.createdSignalsCount >= params.maxSignals) { + if (results.createdSignalsCount >= params.maxSignals) { if (results.warningMessages.includes(getMaxSignalsWarning())) { results.warningMessages = uniq(results.warningMessages); } else if (documentCount > 0) { @@ -202,6 +194,19 @@ export const createThreatSignals = async ({ `Indicator match has reached its max signals count ${params.maxSignals}. Additional documents not checked are ${documentCount}` ); break; + } else if ( + results.suppressedAlertsCount && + results.suppressedAlertsCount > 0 && + results.suppressedAlertsCount + results.createdSignalsCount >= + MAX_SIGNALS_SUPPRESSION_MULTIPLIER * params.maxSignals + ) { + // warning should be already set + ruleExecutionLogger.debug( + `Indicator match has reached its max signals count ${ + MAX_SIGNALS_SUPPRESSION_MULTIPLIER * params.maxSignals + }. Additional documents not checked are ${documentCount}` + ); + break; } ruleExecutionLogger.debug(`Documents items left to check are ${documentCount}`); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/partition_missing_fields_events.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/partition_missing_fields_events.test.ts new file mode 100644 index 0000000000000..8e515cf46e1c6 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/partition_missing_fields_events.test.ts @@ -0,0 +1,113 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { partitionMissingFieldsEvents } from './partition_missing_fields_events'; + +describe('partitionMissingFieldsEvents', () => { + it('should partition if one field is empty', () => { + expect( + partitionMissingFieldsEvents( + [ + { + fields: { + 'agent.host': 'host-1', + 'agent.type': ['test-1', 'test-2'], + 'agent.version': 2, + }, + _id: '1', + _index: 'index-0', + }, + { + fields: { + 'agent.host': 'host-1', + 'agent.type': ['test-1', 'test-2'], + }, + _id: '1', + _index: 'index-0', + }, + ], + ['agent.host', 'agent.type', 'agent.version'] + ) + ).toEqual([ + [ + { + fields: { + 'agent.host': 'host-1', + 'agent.type': ['test-1', 'test-2'], + 'agent.version': 2, + }, + _id: '1', + _index: 'index-0', + }, + ], + [ + { + fields: { + 'agent.host': 'host-1', + 'agent.type': ['test-1', 'test-2'], + }, + _id: '1', + _index: 'index-0', + }, + ], + ]); + }); + it('should partition if two fields are empty', () => { + expect( + partitionMissingFieldsEvents( + [ + { + fields: { + 'agent.type': ['test-1', 'test-2'], + }, + _id: '1', + _index: 'index-0', + }, + ], + ['agent.host', 'agent.type', 'agent.version'] + ) + ).toEqual([ + [], + [ + { + fields: { + 'agent.type': ['test-1', 'test-2'], + }, + _id: '1', + _index: 'index-0', + }, + ], + ]); + }); + it('should partition if all fields are empty', () => { + expect( + partitionMissingFieldsEvents( + [ + { + fields: { + 'some.field': 0, + }, + _id: '1', + _index: 'index-0', + }, + ], + ['agent.host', 'agent.type', 'agent.version'] + ) + ).toEqual([ + [], + [ + { + fields: { + 'some.field': 0, + }, + _id: '1', + _index: 'index-0', + }, + ], + ]); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/partition_missing_fields_events.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/partition_missing_fields_events.ts index 91cbe6ed0ed73..6506a291da561 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/partition_missing_fields_events.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/partition_missing_fields_events.ts @@ -11,10 +11,9 @@ import partition from 'lodash/partition'; import type { SignalSourceHit } from '../types'; /** - * TODO: add description - * @param events - * @param suppressedBy - * @returns + * partition events in 2 arrays: + * 1. first one, where no suppressed by field has empty value + * 2. where any of fields is empty */ export const partitionMissingFieldsEvents = ( events: SignalSourceHit[], diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/search_after_bulk_create_factory.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/search_after_bulk_create_factory.ts index 2eceb6b6ed072..cdbb0530a0452 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/search_after_bulk_create_factory.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/search_after_bulk_create_factory.ts @@ -34,10 +34,6 @@ export interface SearchAfterAndBulkCreateFactoryParams extends SearchAfterAndBul enrichedEvents: SignalSourceHit[]; toReturn: SearchAfterAndBulkCreateReturnType; }) => Promise>; - // }) => Promise<{ - // bulkCreateResult: GenericBulkCreateResponse; - // alertsWereTruncated: boolean; - // }>; getWarningMessage: () => string; } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/search_after_bulk_create_suppressed_alerts.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/search_after_bulk_create_suppressed_alerts.ts index 83a14acf8b625..73f7bfd4a6ba4 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/search_after_bulk_create_suppressed_alerts.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/search_after_bulk_create_suppressed_alerts.ts @@ -14,6 +14,7 @@ import type { SearchAfterAndBulkCreateReturnType, WrapSuppressedHits, } from '../types'; +import { MAX_SIGNALS_SUPPRESSION_MULTIPLIER } from '../constants'; import { createEnrichEventsFunction } from './enrichments'; import { AlertSuppressionMissingFieldsStrategyEnum } from '../../../../../common/api/detection_engine/model/rule_schema'; @@ -57,7 +58,7 @@ export const searchAfterAndBulkCreateSuppressedAlerts = async ( // max signals for suppression includes suppressed and created alerts // this allows to lift max signals limitation to higher value // and can detects threats beyond default max_signals value - const suppressionMaxSignals = 5 * tuple.maxSignals; + const suppressionMaxSignals = MAX_SIGNALS_SUPPRESSION_MULTIPLIER * tuple.maxSignals; const suppressionDuration = alertSuppression?.duration; const suppressionWindow = suppressionDuration @@ -108,7 +109,7 @@ export const searchAfterAndBulkCreateSuppressedAlerts = async ( ...bulkCreateResult, alertsWereTruncated: (toReturn.suppressedAlertsCount ?? 0) + toReturn.createdSignalsCount >= - suppressionMaxSignals, + suppressionMaxSignals || toReturn.createdSignalsCount >= tuple.maxSignals, }; }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.test.ts new file mode 100644 index 0000000000000..5baa84ff913b2 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.test.ts @@ -0,0 +1,236 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { wrapSuppressedAlerts } from './wrap_suppressed_alerts'; + +import { + ALERT_SUPPRESSION_DOCS_COUNT, + ALERT_INSTANCE_ID, + ALERT_SUPPRESSION_TERMS, + ALERT_SUPPRESSION_START, + ALERT_SUPPRESSION_END, +} from '@kbn/rule-data-utils'; + +import type { CompleteRule, ThreatRuleParams } from '../../rule_schema'; +import { buildBulkBody } from '../factories/utils/build_bulk_body'; + +import { ruleExecutionLogMock } from '../../rule_monitoring/mocks'; + +jest.mock('../factories/utils/build_bulk_body', () => ({ buildBulkBody: jest.fn() })); + +const buildBulkBodyMock = buildBulkBody as jest.Mock; + +const ruleExecutionLogger = ruleExecutionLogMock.forExecutors.create(); + +const completeRuleMock: CompleteRule = { + ruleConfig: { + name: 'ALert suppression IM test rule', + tags: [], + consumer: 'siem', + schedule: { + interval: '30m', + }, + enabled: true, + actions: [], + id: 'c1436b3e-e2a6-412a-92ff-ef7e86b926fe', + createdAt: new Date('2024-01-29T13:16:55.678Z'), + createdBy: 'elastic', + producer: 'preview-producer', + revision: 0, + ruleTypeId: 'siem.indicatorRule', + ruleTypeName: 'Indicator Match Rule', + updatedAt: new Date('2024-01-29T13:16:55.678Z'), + updatedBy: 'elastic', + muteAll: false, + snoozeSchedule: [], + }, + ruleParams: { + author: [], + description: 'Tests a simple query', + ruleId: 'threat-match-rule', + falsePositives: [], + from: 'now-35m', + immutable: false, + outputIndex: '', + maxSignals: 100, + riskScore: 1, + riskScoreMapping: [], + severity: 'high', + severityMapping: [], + threat: [], + to: 'now', + references: [], + version: 1, + exceptionsList: [], + relatedIntegrations: [], + requiredFields: [], + setup: '', + type: 'threat_match', + language: 'kuery', + index: ['ecs_compliant'], + query: 'id:a517ae81-eb16-4232-a794-aa81f0ed0302 and NOT agent.type:threat', + threatQuery: 'id:a517ae81-eb16-4232-a794-aa81f0ed0302 and agent.type:threat', + threatMapping: [ + { + entries: [ + { + field: 'host.name', + type: 'mapping', + value: 'host.name', + }, + ], + }, + ], + threatLanguage: 'kuery', + threatIndex: ['ecs_compliant'], + threatIndicatorPath: 'threat.indicator', + alertSuppression: { + groupBy: ['host.name'], + missingFieldsStrategy: 'suppress', + }, + }, + alertId: 'c1436b3e-e2a6-412a-92ff-ef7e86b926fe', +}; + +const wrappedParams = { + spaceId: 'default', + completeRule: { + ...completeRuleMock, + ruleParams: { + ...completeRuleMock.ruleParams, + alertSuppression: { + groupBy: ['agent.name', 'user.name'], + }, + }, + }, + mergeStrategy: 'missingFields' as const, + indicesToQuery: ['test*'], + buildReasonMessage: () => 'mock', + alertTimestampOverride: undefined, + ruleExecutionLogger, + publicBaseUrl: 'public-url-mock', + primaryTimestamp: '@timestamp', + secondaryTimestamp: 'event.ingested', +}; + +describe('wrapSuppressedAlerts', () => { + buildBulkBodyMock.mockReturnValue({ 'mock-props': true }); + + it('should wrap event with alert fields and correctly set suppression fields', () => { + const expectedTimestamp = '2020-10-28T06:30:00.000Z'; + const wrappedAlerts = wrapSuppressedAlerts({ + events: [ + { + fields: { + '@timestamp': [expectedTimestamp], + 'agent.name': ['agent-0'], + 'user.name': ['user-1', 'user-2'], + }, + _id: '1', + _index: 'test*', + }, + ], + ...wrappedParams, + }); + + expect(buildBulkBodyMock).toHaveBeenCalledWith( + 'default', + wrappedParams.completeRule, + { + fields: { + '@timestamp': [expectedTimestamp], + 'agent.name': ['agent-0'], + 'user.name': ['user-1', 'user-2'], + }, + _id: '1', + _index: 'test*', + }, + 'missingFields', + [], + true, + wrappedParams.buildReasonMessage, + ['test*'], + undefined, + ruleExecutionLogger, + expect.any(String), + 'public-url-mock' + ); + expect(wrappedAlerts[0]._source).toEqual( + expect.objectContaining({ + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'agent.name', + value: ['agent-0'], + }, + { + field: 'user.name', + value: ['user-1', 'user-2'], + }, + ], + [ALERT_SUPPRESSION_START]: new Date(expectedTimestamp), + [ALERT_SUPPRESSION_END]: new Date(expectedTimestamp), + [ALERT_SUPPRESSION_DOCS_COUNT]: 0, + 'mock-props': true, + }) + ); + }); + + it('should set instance id of the same value for unsorted identical arrays', () => { + const expectedTimestamp = '2020-10-28T06:30:00.000Z'; + const wrappedAlerts = wrapSuppressedAlerts({ + events: [ + { + fields: { + '@timestamp': [expectedTimestamp], + 'agent.name': ['agent-0'], + 'user.name': ['user-1', 'user-2'], + }, + _id: '1', + _index: 'test*', + }, + { + fields: { + '@timestamp': [expectedTimestamp], + 'agent.name': ['agent-0'], + 'user.name': ['user-2', 'user-1'], + }, + _id: '1', + _index: 'test*', + }, + ], + ...wrappedParams, + }); + + expect(wrappedAlerts[0]._source[ALERT_INSTANCE_ID]).toBe( + wrappedAlerts[1]._source[ALERT_INSTANCE_ID] + ); + }); + it('should set suppression boundaries from secondary timestamp event.ingested if primary is absent', () => { + const expectedTimestamp = '2020-10-28T06:30:00.000Z'; + const wrappedAlerts = wrapSuppressedAlerts({ + events: [ + { + fields: { + 'agent.name': ['agent-0'], + 'user.name': ['user-1', 'user-2'], + 'event.ingested': expectedTimestamp, + }, + _id: '1', + _index: 'test*', + }, + ], + ...wrappedParams, + }); + + expect(wrappedAlerts[0]._source).toEqual( + expect.objectContaining({ + [ALERT_SUPPRESSION_START]: new Date(expectedTimestamp), + [ALERT_SUPPRESSION_END]: new Date(expectedTimestamp), + }) + ); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts index 58b3763136829..f6f1d347fc481 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts @@ -8,6 +8,7 @@ import objectHash from 'object-hash'; import pick from 'lodash/pick'; import get from 'lodash/get'; +import sortBy from 'lodash/sortBy'; import type { SuppressionFieldsLatest } from '@kbn/rule-registry-plugin/common/schemas'; import { @@ -69,10 +70,14 @@ export const wrapSuppressedAlerts = ({ string, string[] | number[] | undefined >; - const suppressionTerms = suppressedBy.map((field) => ({ - field, - value: suppressedProps[field] ?? null, - })); + const suppressionTerms = suppressedBy.map((field) => { + const value = suppressedProps[field] ?? null; + const sortedValue = Array.isArray(value) ? (sortBy(value) as string[] | number[]) : value; + return { + field, + value: sortedValue, + }; + }); const id = objectHash([ event._index, diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/threat_match_alert_suppression.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/threat_match_alert_suppression.ts index 413af489079f0..dae785fe5bee8 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/threat_match_alert_suppression.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/threat_match_alert_suppression.ts @@ -140,6 +140,7 @@ export default ({ getService }: FtrProviderContext) => { name: 'ALert suppression IM test rule', }); + // cases to cover 2 execution paths of IM const cases = [ { eventsCount: 10, @@ -772,7 +773,7 @@ export default ({ getService }: FtrProviderContext) => { await Promise.all( [firstTimestamp, secondTimestamp].map((t) => indexGeneratedSourceDocuments({ - docsCount: expectedMaxSignals, + docsCount: expectedMaxSignals + 15, seed: (index) => ({ id, '@timestamp': t, @@ -810,12 +811,17 @@ export default ({ getService }: FtrProviderContext) => { max_signals: expectedMaxSignals, }; - const { previewId } = await previewRule({ + const { previewId, logs } = await previewRule({ supertest, rule, timeframeEnd: new Date('2020-10-28T06:30:00.000Z'), invocationCount: 2, }); + + expect(logs[0].warnings).toEqual( + expect.arrayContaining([getSuppressionMaxAlertsWarning()]) + ); + const previewAlerts = await getPreviewAlerts({ es, previewId, From 91d40d74dbdb72bcf6bbbf0a4b7d214e7e9990dd Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com> Date: Tue, 30 Jan 2024 08:53:37 +0000 Subject: [PATCH 17/28] test changes --- .../execution_logic/threat_match_alert_suppression.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/threat_match_alert_suppression.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/threat_match_alert_suppression.ts index dae785fe5bee8..4c9558627d51b 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/threat_match_alert_suppression.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/threat_match_alert_suppression.ts @@ -758,7 +758,6 @@ export default ({ getService }: FtrProviderContext) => { }); it('should generate and update up to max_signals alerts', async () => { - const expectedMaxSignals = 40; const id = uuidv4(); const firstTimestamp = '2020-10-28T05:45:00.000Z'; const secondTimestamp = '2020-10-28T06:10:00.000Z'; @@ -773,7 +772,7 @@ export default ({ getService }: FtrProviderContext) => { await Promise.all( [firstTimestamp, secondTimestamp].map((t) => indexGeneratedSourceDocuments({ - docsCount: expectedMaxSignals + 15, + docsCount: 115, seed: (index) => ({ id, '@timestamp': t, @@ -808,7 +807,6 @@ export default ({ getService }: FtrProviderContext) => { }, from: 'now-35m', interval: '30m', - max_signals: expectedMaxSignals, }; const { previewId, logs } = await previewRule({ @@ -828,7 +826,7 @@ export default ({ getService }: FtrProviderContext) => { size: 1000, sort: ['agent.name', ALERT_ORIGINAL_TIME], }); - expect(previewAlerts.length).toEqual(expectedMaxSignals); + expect(previewAlerts.length).toEqual(100); expect(previewAlerts[0]._source).toEqual( expect.objectContaining({ [ALERT_SUPPRESSION_TERMS]: [ From f591ed6fa8905b4d4ca496ac09d9be0cb3b2ecef Mon Sep 17 00:00:00 2001 From: Wafaa Nasr Date: Tue, 30 Jan 2024 11:35:19 +0200 Subject: [PATCH 18/28] [Security Solution][Detection Engine] Refactor Alert Suppression FE (#175089) ## Summary - Include use_alert_suppression to verify the activation of alert suppression for a rule type, considering the existence and activation status of the feature flag if available. - Introduce the SuppressibleAlertRules enum to consolidate all rule types capable of alert suppression. - Implement utility functions to assess the suppression configurations. --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- .../common/detection_engine/constants.ts | 4 + .../common/detection_engine/utils.test.ts | 75 +++++++++++++++++++ .../common/detection_engine/utils.ts | 13 ++++ .../components/description_step/index.tsx | 15 ++-- .../components/step_define_rule/index.tsx | 12 +-- .../rule_details/rule_definition_section.tsx | 17 ++--- .../hooks/use_alert_suppression.test.tsx | 50 +++++++++++++ .../hooks/use_alert_suppression.tsx | 33 ++++++++ 8 files changed, 191 insertions(+), 28 deletions(-) create mode 100644 x-pack/plugins/security_solution/public/detection_engine/rule_management/hooks/use_alert_suppression.test.tsx create mode 100644 x-pack/plugins/security_solution/public/detection_engine/rule_management/hooks/use_alert_suppression.tsx diff --git a/x-pack/plugins/security_solution/common/detection_engine/constants.ts b/x-pack/plugins/security_solution/common/detection_engine/constants.ts index a2a07209997bf..7ef8f2ab13a01 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/constants.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/constants.ts @@ -5,6 +5,8 @@ * 2.0. */ +import type { Type } from '@kbn/securitysolution-io-ts-alerting-types'; + export enum RULE_PREVIEW_INVOCATION_COUNT { HOUR = 12, DAY = 24, @@ -36,3 +38,5 @@ export const ELASTIC_SECURITY_RULE_ID = '9a1a2dae-0b5f-4c3d-8305-a268d404c306'; export const DEFAULT_SUPPRESSION_MISSING_FIELDS_STRATEGY = 'suppress' as const; export const MINIMUM_LICENSE_FOR_SUPPRESSION = 'platinum' as const; + +export const SuppressibleAlertRules: Type[] = ['threshold', 'saved_query', 'query', 'threat_match']; diff --git a/x-pack/plugins/security_solution/common/detection_engine/utils.test.ts b/x-pack/plugins/security_solution/common/detection_engine/utils.test.ts index 8477c115e4e57..11b25889e83c6 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/utils.test.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/utils.test.ts @@ -13,11 +13,17 @@ import { normalizeThresholdField, isMlRule, isEsqlRule, + isSuppressibleAlertRule, + isSuppressionRuleConfiguredWithDuration, + isSuppressionRuleConfiguredWithGroupBy, + isSuppressionRuleConfiguredWithMissingFields, } from './utils'; +import type { Type } from '@kbn/securitysolution-io-ts-alerting-types'; import { hasLargeValueList } from '@kbn/securitysolution-list-utils'; import type { EntriesArray } from '@kbn/securitysolution-io-ts-list-types'; +import { SuppressibleAlertRules } from './constants'; describe('#hasLargeValueList', () => { test('it returns false if empty array', () => { @@ -217,3 +223,72 @@ describe('normalizeMachineLearningJobIds', () => { ]); }); }); +describe('Alert Suppression Rules', () => { + describe('isSuppressibleAlertRule', () => { + test('should return true for a suppressible rule type', () => { + const suppressibleRules: Type[] = Object.values(SuppressibleAlertRules); + suppressibleRules.forEach((rule) => { + const result = isSuppressibleAlertRule(rule); + expect(result).toBe(true); + }); + }); + + test('should return false for a non-suppressible rule type', () => { + const ruleType = '123' as Type; + const result = isSuppressibleAlertRule(ruleType); + expect(result).toBe(false); + }); + }); + + describe('isSuppressionRuleConfiguredWithDuration', () => { + test('should return true for a suppressible rule type', () => { + const suppressibleRules: Type[] = Object.values(SuppressibleAlertRules); + suppressibleRules.forEach((rule) => { + const result = isSuppressionRuleConfiguredWithDuration(rule); + expect(result).toBe(true); + }); + }); + + test('should return false for a non-suppressible rule type', () => { + const ruleType = '123' as Type; + const result = isSuppressionRuleConfiguredWithDuration(ruleType); + expect(result).toBe(false); + }); + }); + + describe('isSuppressionRuleConfiguredWithGroupBy', () => { + test('should return true for a suppressible rule type with groupBy', () => { + const result = isSuppressionRuleConfiguredWithGroupBy('saved_query'); + expect(result).toBe(true); + }); + + test('should return false for a threshold rule type', () => { + const result = isSuppressionRuleConfiguredWithGroupBy('threshold'); + expect(result).toBe(false); + }); + + test('should return false for a non-suppressible rule type', () => { + const ruleType = '123' as Type; + const result = isSuppressionRuleConfiguredWithGroupBy(ruleType); + expect(result).toBe(false); + }); + }); + + describe('isSuppressionRuleConfiguredWithMissingFields', () => { + test('should return true for a suppressible rule type with missing fields', () => { + const result = isSuppressionRuleConfiguredWithMissingFields('query'); + expect(result).toBe(true); + }); + + test('should return false for a threshold rule type', () => { + const result = isSuppressionRuleConfiguredWithMissingFields('threshold'); + expect(result).toBe(false); + }); + + test('should return false for a non-suppressible rule type', () => { + const ruleType = '123' as Type; + const result = isSuppressionRuleConfiguredWithMissingFields(ruleType); + expect(result).toBe(false); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/common/detection_engine/utils.ts b/x-pack/plugins/security_solution/common/detection_engine/utils.ts index 90641e88bafc6..8b9ac375639ab 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/utils.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/utils.ts @@ -17,6 +17,7 @@ import type { Type } from '@kbn/securitysolution-io-ts-alerting-types'; import { hasLargeValueList } from '@kbn/securitysolution-list-utils'; import type { Threshold, ThresholdNormalized } from '../api/detection_engine/model/rule_schema'; +import { SuppressibleAlertRules } from './constants'; export const hasLargeValueItem = ( exceptionItems: Array @@ -68,3 +69,15 @@ export const normalizeThresholdObject = (threshold: Threshold): ThresholdNormali export const normalizeMachineLearningJobIds = (value: string | string[]): string[] => Array.isArray(value) ? value : [value]; + +export const isSuppressibleAlertRule = (ruleType: Type): boolean => { + return SuppressibleAlertRules.includes(ruleType); +}; +export const isSuppressionRuleConfiguredWithDuration = (ruleType: Type) => + isSuppressibleAlertRule(ruleType); + +export const isSuppressionRuleConfiguredWithGroupBy = (ruleType: Type) => + !isThresholdRule(ruleType) && isSuppressibleAlertRule(ruleType); + +export const isSuppressionRuleConfiguredWithMissingFields = (ruleType: Type) => + !isThresholdRule(ruleType) && isSuppressibleAlertRule(ruleType); diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/description_step/index.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/description_step/index.tsx index e662d4aaabf10..ebc25cadfa118 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/description_step/index.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/description_step/index.tsx @@ -58,8 +58,9 @@ import { useLicense } from '../../../../common/hooks/use_license'; import type { LicenseService } from '../../../../../common/license'; import { isThresholdRule, - isQueryRule, - isThreatMatchRule, + isSuppressionRuleConfiguredWithMissingFields, + isSuppressionRuleConfiguredWithGroupBy, + isSuppressionRuleConfiguredWithDuration, } from '../../../../../common/detection_engine/utils'; const DescriptionListContainer = styled(EuiDescriptionList)` @@ -210,7 +211,8 @@ export const getDescriptionItem = ( return []; } else if (field === 'groupByFields') { const ruleType: Type = get('ruleType', data); - const ruleCanHaveGroupByFields = isQueryRule(ruleType) || isThreatMatchRule(ruleType); + + const ruleCanHaveGroupByFields = isSuppressionRuleConfiguredWithGroupBy(ruleType); if (!ruleCanHaveGroupByFields) { return []; } @@ -220,8 +222,8 @@ export const getDescriptionItem = ( return []; } else if (field === 'groupByDuration') { const ruleType: Type = get('ruleType', data); - const ruleCanHaveDuration = - isQueryRule(ruleType) || isThresholdRule(ruleType) || isThreatMatchRule(ruleType); + + const ruleCanHaveDuration = isSuppressionRuleConfiguredWithDuration(ruleType); if (!ruleCanHaveDuration) { return []; } @@ -245,7 +247,8 @@ export const getDescriptionItem = ( } else if (field === 'suppressionMissingFields') { const ruleType: Type = get('ruleType', data); const ruleCanHaveSuppressionMissingFields = - isQueryRule(ruleType) || isThreatMatchRule(ruleType); + isSuppressionRuleConfiguredWithMissingFields(ruleType); + if (!ruleCanHaveSuppressionMissingFields) { return []; } diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.tsx index 6d4e4c6cb381b..cd2f08cab5359 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.tsx @@ -88,7 +88,7 @@ import { AlertSuppressionMissingFieldsStrategyEnum } from '../../../../../common import { DurationInput } from '../duration_input'; import { MINIMUM_LICENSE_FOR_SUPPRESSION } from '../../../../../common/detection_engine/constants'; import { useUpsellingMessage } from '../../../../common/hooks/use_upselling'; -import { useIsExperimentalFeatureEnabled } from '../../../../common/hooks/use_experimental_features'; +import { useAlertSuppression } from '../../../rule_management/hooks/use_alert_suppression'; const CommonUseField = getUseField({ component: Field }); @@ -178,6 +178,7 @@ const StepDefineRuleComponent: FC = ({ thresholdFields, enableThresholdSuppression, }) => { + const { isSuppressionEnabled: isAlertSuppressionEnabled } = useAlertSuppression(ruleType); const mlCapabilities = useMlCapabilities(); const [openTimelineSearch, setOpenTimelineSearch] = useState(false); const [indexModified, setIndexModified] = useState(false); @@ -186,10 +187,6 @@ const StepDefineRuleComponent: FC = ({ const esqlQueryRef = useRef(undefined); - const isAlertSuppressionForIndicatorMatchRuleEnabled = useIsExperimentalFeatureEnabled( - 'alertSuppressionForIndicatorMatchRuleEnabled' - ); - const isAlertSuppressionLicenseValid = license.isAtLeast(MINIMUM_LICENSE_FOR_SUPPRESSION); const isThresholdRule = getIsThresholdRule(ruleType); @@ -813,11 +810,6 @@ const StepDefineRuleComponent: FC = ({ [isUpdateView, mlCapabilities] ); - const isAlertSuppressionEnabled = - isQueryRule(ruleType) || - isThresholdRule || - (isAlertSuppressionForIndicatorMatchRuleEnabled && isThreatMatchRule(ruleType)); - return ( <> diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_management/components/rule_details/rule_definition_section.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_management/components/rule_details/rule_definition_section.tsx index b14bd8cce214e..475d2e0de69a3 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_management/components/rule_details/rule_definition_section.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_management/components/rule_details/rule_definition_section.tsx @@ -54,8 +54,7 @@ import { TechnicalPreviewBadge } from '../../../../common/components/technical_p import { BadgeList } from './badge_list'; import { DEFAULT_DESCRIPTION_LIST_COLUMN_WIDTHS } from './constants'; import * as i18n from './translations'; -import { useIsExperimentalFeatureEnabled } from '../../../../common/hooks/use_experimental_features'; -import type { ExperimentalFeatures } from '../../../../../common/experimental_features'; +import { useAlertSuppression } from '../../hooks/use_alert_suppression'; interface SavedQueryNameProps { savedQueryName: string; @@ -427,7 +426,7 @@ const prepareDefinitionSectionListItems = ( rule: Partial, isInteractive: boolean, savedQuery: SavedQuery | undefined, - { alertSuppressionForIndicatorMatchRuleEnabled }: Partial = {} + isSuppressionEnabled: boolean ): EuiDescriptionListProps['listItems'] => { const definitionSectionListItems: EuiDescriptionListProps['listItems'] = []; @@ -657,11 +656,7 @@ const prepareDefinitionSectionListItems = ( }); } - const isSuppressionEnabled = - (rule.type === 'threat_match' && alertSuppressionForIndicatorMatchRuleEnabled) || - (rule.type && (['query', 'saved_query', 'threshold'] as Type[]).includes(rule.type)); - - if ('alert_suppression' in rule && rule.alert_suppression && isSuppressionEnabled) { + if (isSuppressionEnabled && 'alert_suppression' in rule && rule.alert_suppression) { if ('group_by' in rule.alert_suppression) { definitionSectionListItems.push({ title: ( @@ -743,15 +738,13 @@ export const RuleDefinitionSection = ({ ruleType: rule.type, }); - const alertSuppressionForIndicatorMatchRuleEnabled = useIsExperimentalFeatureEnabled( - 'alertSuppressionForIndicatorMatchRuleEnabled' - ); + const { isSuppressionEnabled } = useAlertSuppression(rule.type); const definitionSectionListItems = prepareDefinitionSectionListItems( rule, isInteractive, savedQuery, - { alertSuppressionForIndicatorMatchRuleEnabled } + isSuppressionEnabled ); return ( diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_management/hooks/use_alert_suppression.test.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_management/hooks/use_alert_suppression.test.tsx new file mode 100644 index 0000000000000..f59b56257ddee --- /dev/null +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_management/hooks/use_alert_suppression.test.tsx @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { renderHook } from '@testing-library/react-hooks'; +import type { Type } from '@kbn/securitysolution-io-ts-alerting-types'; +import * as useIsExperimentalFeatureEnabledMock from '../../../common/hooks/use_experimental_features'; +import { useAlertSuppression } from './use_alert_suppression'; + +describe('useAlertSuppression', () => { + it('should return the correct isSuppressionEnabled value if rule Type exists in SuppressibleAlertRules and Feature Flag is enabled', () => { + jest + .spyOn(useIsExperimentalFeatureEnabledMock, 'useIsExperimentalFeatureEnabled') + .mockImplementation((featureFlagName: string) => { + return featureFlagName === 'alertSuppressionForIndicatorMatchRuleEnabled'; + }); + const { result } = renderHook(() => useAlertSuppression('threat_match')); + + expect(result.current.isSuppressionEnabled).toBe(true); + }); + it('should return the correct isSuppressionEnabled value if rule Type exists in SuppressibleAlertRules and Feature Flag is disabled', () => { + jest + .spyOn(useIsExperimentalFeatureEnabledMock, 'useIsExperimentalFeatureEnabled') + .mockImplementation((featureFlagName: string) => { + return featureFlagName !== 'alertSuppressionForIndicatorMatchRuleEnabled'; + }); + const { result } = renderHook(() => useAlertSuppression('threat_match')); + + expect(result.current.isSuppressionEnabled).toBe(false); + }); + + it('should return the correct isSuppressionEnabled value if rule Type exists in SuppressibleAlertRules', () => { + const { result } = renderHook(() => useAlertSuppression('query')); + + expect(result.current.isSuppressionEnabled).toBe(true); + }); + + it('should return false if rule type is not set', () => { + const { result } = renderHook(() => useAlertSuppression()); + expect(result.current.isSuppressionEnabled).toBe(false); + }); + + it('should return false if rule type is not THREAT_MATCH', () => { + const { result } = renderHook(() => useAlertSuppression('OTHER_RULE_TYPE' as Type)); + + expect(result.current.isSuppressionEnabled).toBe(false); + }); +}); diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_management/hooks/use_alert_suppression.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_management/hooks/use_alert_suppression.tsx new file mode 100644 index 0000000000000..b03340ec70fa9 --- /dev/null +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_management/hooks/use_alert_suppression.tsx @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { useCallback } from 'react'; +import type { Type } from '@kbn/securitysolution-io-ts-alerting-types'; +import { isSuppressibleAlertRule } from '../../../../common/detection_engine/utils'; +import { useIsExperimentalFeatureEnabled } from '../../../common/hooks/use_experimental_features'; +export interface UseAlertSuppressionReturn { + isSuppressionEnabled: boolean; +} + +export const useAlertSuppression = (ruleType?: Type): UseAlertSuppressionReturn => { + const isThreatMatchRuleFFEnabled = useIsExperimentalFeatureEnabled( + 'alertSuppressionForIndicatorMatchRuleEnabled' + ); + + const isSuppressionEnabledForRuleType = useCallback(() => { + if (!ruleType) return false; + + // Remove this condition when the Feature Flag for enabling Suppression in the Indicator Match rule is removed. + if (ruleType === 'threat_match') + return isSuppressibleAlertRule(ruleType) && isThreatMatchRuleFFEnabled; + + return isSuppressibleAlertRule(ruleType); + }, [ruleType, isThreatMatchRuleFFEnabled]); + + return { + isSuppressionEnabled: isSuppressionEnabledForRuleType(), + }; +}; From cd6be83c26bd520fe23bc3fdffc2d268c9c9d403 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com> Date: Tue, 30 Jan 2024 09:56:06 +0000 Subject: [PATCH 19/28] Update threat_match_alert_suppression.ts --- .../execution_logic/threat_match_alert_suppression.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/threat_match_alert_suppression.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/threat_match_alert_suppression.ts index 4c9558627d51b..26ea0ee9303ab 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/threat_match_alert_suppression.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/threat_match_alert_suppression.ts @@ -772,7 +772,7 @@ export default ({ getService }: FtrProviderContext) => { await Promise.all( [firstTimestamp, secondTimestamp].map((t) => indexGeneratedSourceDocuments({ - docsCount: 115, + docsCount: 100, seed: (index) => ({ id, '@timestamp': t, From b608483948e52b44de57290f5427dec66501ffaa Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com> Date: Tue, 30 Jan 2024 10:31:39 +0000 Subject: [PATCH 20/28] rewrite flaky tests --- .../threat_match_alert_suppression.ts | 156 ++++++++---------- 1 file changed, 73 insertions(+), 83 deletions(-) diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/threat_match_alert_suppression.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/threat_match_alert_suppression.ts index 4c9558627d51b..09b430063fe31 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/threat_match_alert_suppression.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/threat_match_alert_suppression.ts @@ -757,89 +757,6 @@ export default ({ getService }: FtrProviderContext) => { }); }); - it('should generate and update up to max_signals alerts', async () => { - const id = uuidv4(); - const firstTimestamp = '2020-10-28T05:45:00.000Z'; - const secondTimestamp = '2020-10-28T06:10:00.000Z'; - - await eventsFiller({ - id, - count: eventsCount * 20, - timestamp: [firstTimestamp, secondTimestamp], - }); - await threatsFiller({ id, count: threatsCount * 20, timestamp: firstTimestamp }); - - await Promise.all( - [firstTimestamp, secondTimestamp].map((t) => - indexGeneratedSourceDocuments({ - docsCount: 115, - seed: (index) => ({ - id, - '@timestamp': t, - host: { - name: `host-a`, - }, - 'agent.name': `agent-${index}`, - }), - }) - ) - ); - await addThreatDocuments({ - id, - timestamp: firstTimestamp, - fields: { - host: { - name: 'host-a', - }, - }, - count: 1, - }); - - const rule: ThreatMatchRuleCreateProps = { - ...indicatorMatchRule(id), - alert_suppression: { - group_by: ['agent.name'], - duration: { - value: 300, - unit: 'm', - }, - missing_fields_strategy: 'suppress', - }, - from: 'now-35m', - interval: '30m', - }; - - const { previewId, logs } = await previewRule({ - supertest, - rule, - timeframeEnd: new Date('2020-10-28T06:30:00.000Z'), - invocationCount: 2, - }); - - expect(logs[0].warnings).toEqual( - expect.arrayContaining([getSuppressionMaxAlertsWarning()]) - ); - - const previewAlerts = await getPreviewAlerts({ - es, - previewId, - size: 1000, - sort: ['agent.name', ALERT_ORIGINAL_TIME], - }); - expect(previewAlerts.length).toEqual(100); - expect(previewAlerts[0]._source).toEqual( - expect.objectContaining({ - [ALERT_SUPPRESSION_TERMS]: [ - { - field: 'agent.name', - value: ['agent-0'], - }, - ], - [ALERT_SUPPRESSION_DOCS_COUNT]: 1, - }) - ); - }); - it('should deduplicate multiple alerts while suppressing new ones', async () => { const id = uuidv4(); const firstTimestamp = '2020-10-28T05:45:00.000Z'; @@ -2150,6 +2067,79 @@ export default ({ getService }: FtrProviderContext) => { }); }); + it('should generate to max_signals alerts' + ' ' + i, async () => { + const id = uuidv4(); + const firstTimestamp = '2020-10-28T06:05:00.000Z'; + const secondTimestamp = '2020-10-28T06:10:00.000Z'; + + await eventsFiller({ + id, + count: eventsCount * 20, + timestamp: [firstTimestamp, secondTimestamp], + }); + await threatsFiller({ id, count: threatsCount * 20, timestamp: firstTimestamp }); + + await Promise.all( + [firstTimestamp, secondTimestamp].map((t) => + indexGeneratedSourceDocuments({ + docsCount: 115, + seed: (index) => ({ + id, + '@timestamp': t, + host: { + name: `host-a`, + }, + 'agent.name': `agent-${index}`, + }), + }) + ) + ); + + await addThreatDocuments({ + id, + timestamp: firstTimestamp, + fields: { + host: { + name: 'host-a', + }, + }, + count: 1, + }); + + const rule: ThreatMatchRuleCreateProps = { + ...indicatorMatchRule(id), + alert_suppression: { + group_by: ['agent.name'], + duration: { + value: 300, + unit: 'm', + }, + missing_fields_strategy: 'suppress', + }, + from: 'now-35m', + interval: '30m', + }; + + const { previewId, logs } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T06:30:00.000Z'), + invocationCount: 1, + }); + + expect(logs[0].warnings).toEqual( + expect.arrayContaining([getSuppressionMaxAlertsWarning()]) + ); + + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + size: 1000, + sort: ['agent.name', ALERT_ORIGINAL_TIME], + }); + expect(previewAlerts.length).toEqual(100); + }); + it('should detect threats beyond max_signals if large number of alerts suppressed', async () => { const id = uuidv4(); const timestamp = '2020-10-28T06:45:00.000Z'; From e53d5445f08a4f644f1cac531f7b70651ced53c5 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com> Date: Tue, 30 Jan 2024 10:37:46 +0000 Subject: [PATCH 21/28] follow constants naming convention --- .../security_solution/common/detection_engine/constants.ts | 7 ++++++- .../common/detection_engine/utils.test.ts | 6 +++--- .../security_solution/common/detection_engine/utils.ts | 4 ++-- .../rule_management/hooks/use_alert_suppression.test.tsx | 6 +++--- 4 files changed, 14 insertions(+), 9 deletions(-) diff --git a/x-pack/plugins/security_solution/common/detection_engine/constants.ts b/x-pack/plugins/security_solution/common/detection_engine/constants.ts index 7ef8f2ab13a01..08af29c8f69c7 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/constants.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/constants.ts @@ -39,4 +39,9 @@ export const DEFAULT_SUPPRESSION_MISSING_FIELDS_STRATEGY = 'suppress' as const; export const MINIMUM_LICENSE_FOR_SUPPRESSION = 'platinum' as const; -export const SuppressibleAlertRules: Type[] = ['threshold', 'saved_query', 'query', 'threat_match']; +export const SUPPRESSIBLE_ALERT_RULES: Type[] = [ + 'threshold', + 'saved_query', + 'query', + 'threat_match', +]; diff --git a/x-pack/plugins/security_solution/common/detection_engine/utils.test.ts b/x-pack/plugins/security_solution/common/detection_engine/utils.test.ts index 11b25889e83c6..db51e91cf94d3 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/utils.test.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/utils.test.ts @@ -23,7 +23,7 @@ import type { Type } from '@kbn/securitysolution-io-ts-alerting-types'; import { hasLargeValueList } from '@kbn/securitysolution-list-utils'; import type { EntriesArray } from '@kbn/securitysolution-io-ts-list-types'; -import { SuppressibleAlertRules } from './constants'; +import { SUPPRESSIBLE_ALERT_RULES } from './constants'; describe('#hasLargeValueList', () => { test('it returns false if empty array', () => { @@ -226,7 +226,7 @@ describe('normalizeMachineLearningJobIds', () => { describe('Alert Suppression Rules', () => { describe('isSuppressibleAlertRule', () => { test('should return true for a suppressible rule type', () => { - const suppressibleRules: Type[] = Object.values(SuppressibleAlertRules); + const suppressibleRules: Type[] = Object.values(SUPPRESSIBLE_ALERT_RULES); suppressibleRules.forEach((rule) => { const result = isSuppressibleAlertRule(rule); expect(result).toBe(true); @@ -242,7 +242,7 @@ describe('Alert Suppression Rules', () => { describe('isSuppressionRuleConfiguredWithDuration', () => { test('should return true for a suppressible rule type', () => { - const suppressibleRules: Type[] = Object.values(SuppressibleAlertRules); + const suppressibleRules: Type[] = Object.values(SUPPRESSIBLE_ALERT_RULES); suppressibleRules.forEach((rule) => { const result = isSuppressionRuleConfiguredWithDuration(rule); expect(result).toBe(true); diff --git a/x-pack/plugins/security_solution/common/detection_engine/utils.ts b/x-pack/plugins/security_solution/common/detection_engine/utils.ts index 8b9ac375639ab..12342116addab 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/utils.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/utils.ts @@ -17,7 +17,7 @@ import type { Type } from '@kbn/securitysolution-io-ts-alerting-types'; import { hasLargeValueList } from '@kbn/securitysolution-list-utils'; import type { Threshold, ThresholdNormalized } from '../api/detection_engine/model/rule_schema'; -import { SuppressibleAlertRules } from './constants'; +import { SUPPRESSIBLE_ALERT_RULES } from './constants'; export const hasLargeValueItem = ( exceptionItems: Array @@ -71,7 +71,7 @@ export const normalizeMachineLearningJobIds = (value: string | string[]): string Array.isArray(value) ? value : [value]; export const isSuppressibleAlertRule = (ruleType: Type): boolean => { - return SuppressibleAlertRules.includes(ruleType); + return SUPPRESSIBLE_ALERT_RULES.includes(ruleType); }; export const isSuppressionRuleConfiguredWithDuration = (ruleType: Type) => isSuppressibleAlertRule(ruleType); diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_management/hooks/use_alert_suppression.test.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_management/hooks/use_alert_suppression.test.tsx index f59b56257ddee..dc664b89fa446 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_management/hooks/use_alert_suppression.test.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_management/hooks/use_alert_suppression.test.tsx @@ -10,7 +10,7 @@ import * as useIsExperimentalFeatureEnabledMock from '../../../common/hooks/use_ import { useAlertSuppression } from './use_alert_suppression'; describe('useAlertSuppression', () => { - it('should return the correct isSuppressionEnabled value if rule Type exists in SuppressibleAlertRules and Feature Flag is enabled', () => { + it('should return the correct isSuppressionEnabled value if rule Type exists in SUPPRESSIBLE_ALERT_RULES and Feature Flag is enabled', () => { jest .spyOn(useIsExperimentalFeatureEnabledMock, 'useIsExperimentalFeatureEnabled') .mockImplementation((featureFlagName: string) => { @@ -20,7 +20,7 @@ describe('useAlertSuppression', () => { expect(result.current.isSuppressionEnabled).toBe(true); }); - it('should return the correct isSuppressionEnabled value if rule Type exists in SuppressibleAlertRules and Feature Flag is disabled', () => { + it('should return the correct isSuppressionEnabled value if rule Type exists in SUPPRESSIBLE_ALERT_RULES and Feature Flag is disabled', () => { jest .spyOn(useIsExperimentalFeatureEnabledMock, 'useIsExperimentalFeatureEnabled') .mockImplementation((featureFlagName: string) => { @@ -31,7 +31,7 @@ describe('useAlertSuppression', () => { expect(result.current.isSuppressionEnabled).toBe(false); }); - it('should return the correct isSuppressionEnabled value if rule Type exists in SuppressibleAlertRules', () => { + it('should return the correct isSuppressionEnabled value if rule Type exists in SUPPRESSIBLE_ALERT_RULES', () => { const { result } = renderHook(() => useAlertSuppression('query')); expect(result.current.isSuppressionEnabled).toBe(true); From 3ccd2f8fc2a2baeb900c8c545aa4f6dd921c0f79 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com> Date: Thu, 1 Feb 2024 16:30:00 +0000 Subject: [PATCH 22/28] PR feedback --- .../rule_creation/indicator_match_rule_suppression.cy.ts | 7 ++----- .../indicator_match_rule_suppression_ess_basic.cy.ts | 2 ++ ...ator_match_rule_suppression_serverless_essentials.cy.ts | 1 - 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/indicator_match_rule_suppression.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/indicator_match_rule_suppression.cy.ts index 4c95996841663..d91c26e1990a3 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/indicator_match_rule_suppression.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/indicator_match_rule_suppression.cy.ts @@ -58,12 +58,12 @@ describe( deleteAlertsAndRules(); login(); visit(CREATE_RULE_URL); - }); - it('creates rule with per rule execution suppression', () => { selectIndicatorMatchType(); fillDefineIndicatorMatchRule(rule); + }); + it('creates rule with per rule execution suppression', () => { // selecting only suppression fields, the rest options would be default fillAlertSuppressionFields(SUPPRESS_BY_FIELDS); continueFromDefineStep(); @@ -98,9 +98,6 @@ describe( it('creates rule rule with time interval suppression', () => { const expectedSuppressByFields = SUPPRESS_BY_FIELDS.slice(0, 1); - selectIndicatorMatchType(); - fillDefineIndicatorMatchRule(rule); - // fill suppress by fields and select non-default suppression options fillAlertSuppressionFields(expectedSuppressByFields); selectAlertSuppressionPerInterval(); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/indicator_match_rule_suppression_ess_basic.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/indicator_match_rule_suppression_ess_basic.cy.ts index f3d724ec49a11..17c1e57fd85ed 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/indicator_match_rule_suppression_ess_basic.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/indicator_match_rule_suppression_ess_basic.cy.ts @@ -29,6 +29,7 @@ import { CREATE_RULE_URL } from '../../../../urls/navigation'; import { TOOLTIP } from '../../../../screens/common'; import { ruleDetailsUrl } from '../../../../urls/rule_details'; import { getDetails } from '../../../../tasks/rule_details'; +import { deleteAlertsAndRules } from '../../../../tasks/api_calls/common'; const SUPPRESS_BY_FIELDS = ['myhash.mysha256', 'source.ip.keyword']; @@ -40,6 +41,7 @@ describe( () => { describe('Create rule form', () => { beforeEach(() => { + deleteAlertsAndRules(); login(); visit(CREATE_RULE_URL); startBasicLicense(); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/indicator_match_rule_suppression_serverless_essentials.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/indicator_match_rule_suppression_serverless_essentials.cy.ts index f5c845f344289..cc99d905f4406 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/indicator_match_rule_suppression_serverless_essentials.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/indicator_match_rule_suppression_serverless_essentials.cy.ts @@ -56,7 +56,6 @@ describe( }); it('creates rule with per rule execution suppression for essentials license', () => { - login(); visit(CREATE_RULE_URL); selectIndicatorMatchType(); fillDefineIndicatorMatchRule(rule); From 3aa86a66974c3ac99841bbe175a45d9f458220ba Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com> Date: Thu, 1 Feb 2024 17:30:22 +0000 Subject: [PATCH 23/28] PR feedback --- .../rule_schema/rule_request_schema.test.ts | 186 +++++++++--------- .../common/detection_engine/utils.test.ts | 66 +++++-- .../components/step_define_rule/index.tsx | 2 +- .../rule_details/rule_definition_section.tsx | 2 +- .../use_alert_suppression.test.tsx | 4 +- .../use_alert_suppression.tsx | 2 +- 6 files changed, 149 insertions(+), 113 deletions(-) rename x-pack/plugins/security_solution/public/detection_engine/rule_management/{hooks => logic}/use_alert_suppression.test.tsx (94%) rename x-pack/plugins/security_solution/public/detection_engine/rule_management/{hooks => logic}/use_alert_suppression.tsx (92%) diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_request_schema.test.ts b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_request_schema.test.ts index d99beea8acc90..7393d3ad52fa4 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_request_schema.test.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_request_schema.test.ts @@ -1214,121 +1214,129 @@ describe('rules schema', () => { }); describe('alerts suppression', () => { - test('should drop suppression fields apart from duration for "threshold" rule type', () => { - const payload = { - ...getCreateThresholdRulesSchemaMock(), - alert_suppression: { - group_by: ['host.name'], - duration: { value: 5, unit: 'm' }, - missing_field_strategy: 'suppress', - }, - }; - - const result = RuleCreateProps.safeParse(payload); - expectParseSuccess(result); - expect(result.data).toEqual({ - ...payload, - alert_suppression: { - duration: { value: 5, unit: 'm' }, - }, - }); - }); - test('should validate only suppression duration for "threshold" rule type', () => { - const payload = { - ...getCreateThresholdRulesSchemaMock(), - alert_suppression: { - duration: { value: 5, unit: 'm' }, - }, - }; - - const result = RuleCreateProps.safeParse(payload); - expectParseSuccess(result); - expect(result.data).toEqual(payload); - }); - test('should throw error if alert suppression duration is absent for "threshold" rule type', () => { - const payload = { - ...getCreateThresholdRulesSchemaMock(), - alert_suppression: { - group_by: ['host.name'], - missing_field_strategy: 'suppress', - }, - }; - - const result = RuleCreateProps.safeParse(payload); - expectParseError(result); - expect(stringifyZodError(result.error)).toMatchInlineSnapshot( - `"alert_suppression.duration: Required"` - ); - }); - // behaviour common for multiple rule types - const cases = [{ ruleType: 'threat_match', ruleMock: getCreateThreatMatchRulesSchemaMock() }]; - - cases.forEach(({ ruleType, ruleMock }) => { - test(`should validate suppression fields for "${ruleType}" rule type`, () => { + describe(`alert suppression validation for "threshold" rule type`, () => { + test('should drop suppression fields apart from duration for "threshold" rule type', () => { const payload = { - ...ruleMock, + ...getCreateThresholdRulesSchemaMock(), alert_suppression: { - group_by: ['agent.name'], + group_by: ['host.name'], duration: { value: 5, unit: 'm' }, - missing_fields_strategy: 'suppress', + missing_field_strategy: 'suppress', }, }; const result = RuleCreateProps.safeParse(payload); expectParseSuccess(result); - expect(result.data).toEqual(payload); + expect(result.data).toEqual({ + ...payload, + alert_suppression: { + duration: { value: 5, unit: 'm' }, + }, + }); }); - - test(`should throw error if suppression fields not valid for "${ruleType}" rule`, () => { + test('should validate only suppression duration for "threshold" rule type', () => { const payload = { - ...ruleMock, + ...getCreateThresholdRulesSchemaMock(), alert_suppression: { - group_by: 'not an array', - missing_fields_strategy: 'suppress', + duration: { value: 5, unit: 'm' }, }, }; const result = RuleCreateProps.safeParse(payload); - expectParseError(result); - expect(stringifyZodError(result.error)).toEqual( - 'alert_suppression.group_by: Expected array, received string' - ); + expectParseSuccess(result); + expect(result.data).toEqual(payload); }); - - test(`should throw error if suppression required field is missing for "${ruleType}" rule`, () => { + test('should throw error if alert suppression duration is absent for "threshold" rule type', () => { const payload = { - ...ruleMock, + ...getCreateThresholdRulesSchemaMock(), alert_suppression: { - duration: { value: 5, unit: 'm' }, - missing_fields_strategy: 'suppress', + group_by: ['host.name'], + missing_field_strategy: 'suppress', }, }; const result = RuleCreateProps.safeParse(payload); expectParseError(result); - expect(stringifyZodError(result.error)).toEqual('alert_suppression.group_by: Required'); + expect(stringifyZodError(result.error)).toMatchInlineSnapshot( + `"alert_suppression.duration: Required"` + ); }); + }); + // behaviour common for multiple rule types + const cases = [ + { ruleType: 'threat_match', ruleMock: getCreateThreatMatchRulesSchemaMock() }, + { ruleType: 'query', ruleMock: getCreateRulesSchemaMock() }, + { ruleType: 'saved_query', ruleMock: getCreateSavedQueryRulesSchemaMock() }, + ]; - test(`should drop fields that are not in suppression schema for "${ruleType}" rule`, () => { - const payload = { - ...ruleMock, - alert_suppression: { - group_by: ['agent.name'], - duration: { value: 5, unit: 'm' }, - missing_fields_strategy: 'suppress', - random_field: 1, - }, - }; + cases.forEach(({ ruleType, ruleMock }) => { + describe(`alert suppression validation for "${ruleType}" rule type`, () => { + test(`should validate suppression fields for "${ruleType}" rule type`, () => { + const payload = { + ...ruleMock, + alert_suppression: { + group_by: ['agent.name'], + duration: { value: 5, unit: 'm' }, + missing_fields_strategy: 'suppress', + }, + }; - const result = RuleCreateProps.safeParse(payload); - expectParseSuccess(result); - expect(result.data).toEqual({ - ...ruleMock, - alert_suppression: { - group_by: ['agent.name'], - duration: { value: 5, unit: 'm' }, - missing_fields_strategy: 'suppress', - }, + const result = RuleCreateProps.safeParse(payload); + expectParseSuccess(result); + expect(result.data).toEqual(payload); + }); + + test(`should throw error if suppression fields not valid for "${ruleType}" rule`, () => { + const payload = { + ...ruleMock, + alert_suppression: { + group_by: 'not an array', + missing_fields_strategy: 'suppress', + }, + }; + + const result = RuleCreateProps.safeParse(payload); + expectParseError(result); + expect(stringifyZodError(result.error)).toEqual( + 'alert_suppression.group_by: Expected array, received string' + ); + }); + + test(`should throw error if suppression required field is missing for "${ruleType}" rule`, () => { + const payload = { + ...ruleMock, + alert_suppression: { + duration: { value: 5, unit: 'm' }, + missing_fields_strategy: 'suppress', + }, + }; + + const result = RuleCreateProps.safeParse(payload); + expectParseError(result); + expect(stringifyZodError(result.error)).toEqual('alert_suppression.group_by: Required'); + }); + + test(`should drop fields that are not in suppression schema for "${ruleType}" rule`, () => { + const payload = { + ...ruleMock, + alert_suppression: { + group_by: ['agent.name'], + duration: { value: 5, unit: 'm' }, + missing_fields_strategy: 'suppress', + random_field: 1, + }, + }; + + const result = RuleCreateProps.safeParse(payload); + expectParseSuccess(result); + expect(result.data).toEqual({ + ...ruleMock, + alert_suppression: { + group_by: ['agent.name'], + duration: { value: 5, unit: 'm' }, + missing_fields_strategy: 'suppress', + }, + }); }); }); }); diff --git a/x-pack/plugins/security_solution/common/detection_engine/utils.test.ts b/x-pack/plugins/security_solution/common/detection_engine/utils.test.ts index db51e91cf94d3..1db75db42a7d8 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/utils.test.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/utils.test.ts @@ -23,7 +23,6 @@ import type { Type } from '@kbn/securitysolution-io-ts-alerting-types'; import { hasLargeValueList } from '@kbn/securitysolution-list-utils'; import type { EntriesArray } from '@kbn/securitysolution-io-ts-list-types'; -import { SUPPRESSIBLE_ALERT_RULES } from './constants'; describe('#hasLargeValueList', () => { test('it returns false if empty array', () => { @@ -223,17 +222,24 @@ describe('normalizeMachineLearningJobIds', () => { ]); }); }); + describe('Alert Suppression Rules', () => { describe('isSuppressibleAlertRule', () => { test('should return true for a suppressible rule type', () => { - const suppressibleRules: Type[] = Object.values(SUPPRESSIBLE_ALERT_RULES); - suppressibleRules.forEach((rule) => { - const result = isSuppressibleAlertRule(rule); - expect(result).toBe(true); - }); + // Rule types that support alert suppression: + expect(isSuppressibleAlertRule('threshold')).toBe(true); + expect(isSuppressibleAlertRule('saved_query')).toBe(true); + expect(isSuppressibleAlertRule('query')).toBe(true); + expect(isSuppressibleAlertRule('threat_match')).toBe(true); + + // Rule types that don't support alert suppression: + expect(isSuppressibleAlertRule('eql')).toBe(false); + expect(isSuppressibleAlertRule('machine_learning')).toBe(false); + expect(isSuppressibleAlertRule('new_terms')).toBe(false); + expect(isSuppressibleAlertRule('esql')).toBe(false); }); - test('should return false for a non-suppressible rule type', () => { + test('should return false for an unknown rule type', () => { const ruleType = '123' as Type; const result = isSuppressibleAlertRule(ruleType); expect(result).toBe(false); @@ -242,14 +248,20 @@ describe('Alert Suppression Rules', () => { describe('isSuppressionRuleConfiguredWithDuration', () => { test('should return true for a suppressible rule type', () => { - const suppressibleRules: Type[] = Object.values(SUPPRESSIBLE_ALERT_RULES); - suppressibleRules.forEach((rule) => { - const result = isSuppressionRuleConfiguredWithDuration(rule); - expect(result).toBe(true); - }); + // Rule types that support alert suppression: + expect(isSuppressionRuleConfiguredWithDuration('threshold')).toBe(true); + expect(isSuppressionRuleConfiguredWithDuration('saved_query')).toBe(true); + expect(isSuppressionRuleConfiguredWithDuration('query')).toBe(true); + expect(isSuppressionRuleConfiguredWithDuration('threat_match')).toBe(true); + + // Rule types that don't support alert suppression: + expect(isSuppressionRuleConfiguredWithDuration('eql')).toBe(false); + expect(isSuppressionRuleConfiguredWithDuration('machine_learning')).toBe(false); + expect(isSuppressionRuleConfiguredWithDuration('new_terms')).toBe(false); + expect(isSuppressionRuleConfiguredWithDuration('esql')).toBe(false); }); - test('should return false for a non-suppressible rule type', () => { + test('should return false for an unknown rule type', () => { const ruleType = '123' as Type; const result = isSuppressionRuleConfiguredWithDuration(ruleType); expect(result).toBe(false); @@ -258,8 +270,16 @@ describe('Alert Suppression Rules', () => { describe('isSuppressionRuleConfiguredWithGroupBy', () => { test('should return true for a suppressible rule type with groupBy', () => { - const result = isSuppressionRuleConfiguredWithGroupBy('saved_query'); - expect(result).toBe(true); + // Rule types that support alert suppression groupBy: + expect(isSuppressionRuleConfiguredWithGroupBy('saved_query')).toBe(true); + expect(isSuppressionRuleConfiguredWithGroupBy('query')).toBe(true); + expect(isSuppressionRuleConfiguredWithGroupBy('threat_match')).toBe(true); + + // Rule types that don't support alert suppression: + expect(isSuppressionRuleConfiguredWithGroupBy('eql')).toBe(false); + expect(isSuppressionRuleConfiguredWithGroupBy('machine_learning')).toBe(false); + expect(isSuppressionRuleConfiguredWithGroupBy('new_terms')).toBe(false); + expect(isSuppressionRuleConfiguredWithGroupBy('esql')).toBe(false); }); test('should return false for a threshold rule type', () => { @@ -267,7 +287,7 @@ describe('Alert Suppression Rules', () => { expect(result).toBe(false); }); - test('should return false for a non-suppressible rule type', () => { + test('should return false for an unknown rule type', () => { const ruleType = '123' as Type; const result = isSuppressionRuleConfiguredWithGroupBy(ruleType); expect(result).toBe(false); @@ -276,8 +296,16 @@ describe('Alert Suppression Rules', () => { describe('isSuppressionRuleConfiguredWithMissingFields', () => { test('should return true for a suppressible rule type with missing fields', () => { - const result = isSuppressionRuleConfiguredWithMissingFields('query'); - expect(result).toBe(true); + // Rule types that support alert suppression groupBy: + expect(isSuppressionRuleConfiguredWithMissingFields('saved_query')).toBe(true); + expect(isSuppressionRuleConfiguredWithMissingFields('query')).toBe(true); + expect(isSuppressionRuleConfiguredWithMissingFields('threat_match')).toBe(true); + + // Rule types that don't support alert suppression: + expect(isSuppressionRuleConfiguredWithMissingFields('eql')).toBe(false); + expect(isSuppressionRuleConfiguredWithMissingFields('machine_learning')).toBe(false); + expect(isSuppressionRuleConfiguredWithMissingFields('new_terms')).toBe(false); + expect(isSuppressionRuleConfiguredWithMissingFields('esql')).toBe(false); }); test('should return false for a threshold rule type', () => { @@ -285,7 +313,7 @@ describe('Alert Suppression Rules', () => { expect(result).toBe(false); }); - test('should return false for a non-suppressible rule type', () => { + test('should return false for an unknown rule type', () => { const ruleType = '123' as Type; const result = isSuppressionRuleConfiguredWithMissingFields(ruleType); expect(result).toBe(false); diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.tsx index cd2f08cab5359..87551441136ca 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.tsx @@ -88,7 +88,7 @@ import { AlertSuppressionMissingFieldsStrategyEnum } from '../../../../../common import { DurationInput } from '../duration_input'; import { MINIMUM_LICENSE_FOR_SUPPRESSION } from '../../../../../common/detection_engine/constants'; import { useUpsellingMessage } from '../../../../common/hooks/use_upselling'; -import { useAlertSuppression } from '../../../rule_management/hooks/use_alert_suppression'; +import { useAlertSuppression } from '../../../rule_management/logic/use_alert_suppression'; const CommonUseField = getUseField({ component: Field }); diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_management/components/rule_details/rule_definition_section.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_management/components/rule_details/rule_definition_section.tsx index 475d2e0de69a3..449a9c58d89fe 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_management/components/rule_details/rule_definition_section.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_management/components/rule_details/rule_definition_section.tsx @@ -54,7 +54,7 @@ import { TechnicalPreviewBadge } from '../../../../common/components/technical_p import { BadgeList } from './badge_list'; import { DEFAULT_DESCRIPTION_LIST_COLUMN_WIDTHS } from './constants'; import * as i18n from './translations'; -import { useAlertSuppression } from '../../hooks/use_alert_suppression'; +import { useAlertSuppression } from '../../logic/use_alert_suppression'; interface SavedQueryNameProps { savedQueryName: string; diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_management/hooks/use_alert_suppression.test.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_management/logic/use_alert_suppression.test.tsx similarity index 94% rename from x-pack/plugins/security_solution/public/detection_engine/rule_management/hooks/use_alert_suppression.test.tsx rename to x-pack/plugins/security_solution/public/detection_engine/rule_management/logic/use_alert_suppression.test.tsx index dc664b89fa446..1f817aa4dd6a5 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_management/hooks/use_alert_suppression.test.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_management/logic/use_alert_suppression.test.tsx @@ -37,8 +37,8 @@ describe('useAlertSuppression', () => { expect(result.current.isSuppressionEnabled).toBe(true); }); - it('should return false if rule type is not set', () => { - const { result } = renderHook(() => useAlertSuppression()); + it('should return false if rule type is undefined', () => { + const { result } = renderHook(() => useAlertSuppression(undefined)); expect(result.current.isSuppressionEnabled).toBe(false); }); diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_management/hooks/use_alert_suppression.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_management/logic/use_alert_suppression.tsx similarity index 92% rename from x-pack/plugins/security_solution/public/detection_engine/rule_management/hooks/use_alert_suppression.tsx rename to x-pack/plugins/security_solution/public/detection_engine/rule_management/logic/use_alert_suppression.tsx index b03340ec70fa9..fb8fbaa8756cc 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_management/hooks/use_alert_suppression.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_management/logic/use_alert_suppression.tsx @@ -12,7 +12,7 @@ export interface UseAlertSuppressionReturn { isSuppressionEnabled: boolean; } -export const useAlertSuppression = (ruleType?: Type): UseAlertSuppressionReturn => { +export const useAlertSuppression = (ruleType: Type | undefined): UseAlertSuppressionReturn => { const isThreatMatchRuleFFEnabled = useIsExperimentalFeatureEnabled( 'alertSuppressionForIndicatorMatchRuleEnabled' ); From f0ad4f23415328ad5ddeccdf9acfa31c663e2826 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com> Date: Fri, 2 Feb 2024 12:42:23 +0000 Subject: [PATCH 24/28] address PR feedback --- .../rule_creation/indicator_match_rule_suppression.cy.ts | 1 + ...indicator_match_rule_suppression_serverless_essentials.cy.ts | 2 +- .../detection_engine/rule_edit/indicator_match_rule.cy.ts | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/indicator_match_rule_suppression.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/indicator_match_rule_suppression.cy.ts index d91c26e1990a3..1f544363d547e 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/indicator_match_rule_suppression.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/indicator_match_rule_suppression.cy.ts @@ -40,6 +40,7 @@ describe( 'Detection rules, Indicator Match, Alert Suppression', { tags: ['@ess', '@serverless'], + // alertSuppressionForIndicatorMatchRuleEnabled feature flag is also enabled in a global config env: { ftrConfig: { kbnServerArgs: [ diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/indicator_match_rule_suppression_serverless_essentials.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/indicator_match_rule_suppression_serverless_essentials.cy.ts index cc99d905f4406..59bf7e6d0d23b 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/indicator_match_rule_suppression_serverless_essentials.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/indicator_match_rule_suppression_serverless_essentials.cy.ts @@ -37,7 +37,7 @@ describe( { product_line: 'security', product_tier: 'essentials' }, { product_line: 'endpoint', product_tier: 'essentials' }, ], - + // alertSuppressionForIndicatorMatchRuleEnabled feature flag is also enabled in a global config kbnServerArgs: [ `--xpack.securitySolution.enableExperimental=${JSON.stringify([ 'alertSuppressionForIndicatorMatchRuleEnabled', diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_edit/indicator_match_rule.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_edit/indicator_match_rule.cy.ts index 71e072f5c3af2..66e697d7743e5 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_edit/indicator_match_rule.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_edit/indicator_match_rule.cy.ts @@ -48,6 +48,7 @@ describe( 'Detection rules, Indicator Match, Edit', { tags: ['@ess', '@serverless'], + // alertSuppressionForIndicatorMatchRuleEnabled feature flag is also enabled in a global config env: { ftrConfig: { kbnServerArgs: [ From c3e6fb9b9d27a6be18a9ba2aa914590ecf90f0e2 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com> Date: Fri, 2 Feb 2024 15:19:35 +0000 Subject: [PATCH 25/28] Update x-pack/plugins/rule_registry/common/schemas/8.13.0/index.ts Co-authored-by: Ievgen Sorokopud --- x-pack/plugins/rule_registry/common/schemas/8.13.0/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/rule_registry/common/schemas/8.13.0/index.ts b/x-pack/plugins/rule_registry/common/schemas/8.13.0/index.ts index a3ed01d861e4e..14262d7f9fd7e 100644 --- a/x-pack/plugins/rule_registry/common/schemas/8.13.0/index.ts +++ b/x-pack/plugins/rule_registry/common/schemas/8.13.0/index.ts @@ -10,7 +10,7 @@ import { AlertWithCommonFields880 } from '../8.8.0'; import { SuppressionFields870 } from '../8.7.0'; -/* DO NOT MODIFY THIS SCHEMA TO ADD NEW FIELDS. These types represent the alerts that shipped in 8.7.0. +/* DO NOT MODIFY THIS SCHEMA TO ADD NEW FIELDS. These types represent the alerts that shipped in 8.13.0. Any changes to these types should be bug fixes so the types more accurately represent the alerts from 8.7.0. If you are adding new fields for a new release of Kibana, create a new sibling folder to this one From 73d482ca94fdb1b19f90ddd517106ed89f293561 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com> Date: Fri, 2 Feb 2024 15:19:45 +0000 Subject: [PATCH 26/28] Update x-pack/plugins/rule_registry/common/schemas/8.13.0/index.ts Co-authored-by: Ievgen Sorokopud --- x-pack/plugins/rule_registry/common/schemas/8.13.0/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/rule_registry/common/schemas/8.13.0/index.ts b/x-pack/plugins/rule_registry/common/schemas/8.13.0/index.ts index 14262d7f9fd7e..70b0b0d6b5793 100644 --- a/x-pack/plugins/rule_registry/common/schemas/8.13.0/index.ts +++ b/x-pack/plugins/rule_registry/common/schemas/8.13.0/index.ts @@ -11,7 +11,7 @@ import { AlertWithCommonFields880 } from '../8.8.0'; import { SuppressionFields870 } from '../8.7.0'; /* DO NOT MODIFY THIS SCHEMA TO ADD NEW FIELDS. These types represent the alerts that shipped in 8.13.0. -Any changes to these types should be bug fixes so the types more accurately represent the alerts from 8.7.0. +Any changes to these types should be bug fixes so the types more accurately represent the alerts from 8.13.0. If you are adding new fields for a new release of Kibana, create a new sibling folder to this one for the version to be released and add the field(s) to the schema in that folder. From e4239b6d73cc0fc465d94a377a2c28fe35a78abf Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com> Date: Fri, 2 Feb 2024 15:36:02 +0000 Subject: [PATCH 27/28] PR feedback --- .../utils/create_persistence_rule_type_wrapper.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts index e621890d815a7..8f6628298b494 100644 --- a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts +++ b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts @@ -33,7 +33,11 @@ import { errorAggregator } from './utils'; import { AlertWithSuppressionFields870 } from '../../common/schemas/8.7.0'; /** - * alerts returned from BE have date type coerced to ISO strings + * Alerts returned from BE have date type coerced to ISO strings + * + * We use BackendAlertWithSuppressionFields870 explicitly here as the type instead of + * AlertWithSuppressionFieldsLatest since we're reading alerts rather than writing, + * so future versions of Kibana may read 8.7.0 version alerts and need to update them */ export type BackendAlertWithSuppressionFields870 = Omit< AlertWithSuppressionFields870, @@ -407,7 +411,7 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper { terms: { [ALERT_INSTANCE_ID]: filteredDuplicates.map( - (alert) => alert._source['kibana.alert.instance.id'] + (alert) => alert._source[ALERT_INSTANCE_ID] ), }, }, @@ -436,9 +440,6 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper }, }; - // We use BackendAlertWithSuppressionFields870 explicitly here as the type instead of - // AlertWithSuppressionFieldsLatest since we're reading alerts rather than writing, - // so future versions of Kibana may read 8.7.0 version alerts and need to update them const response = await ruleDataClient .getReader({ namespace: options.spaceId }) .search< From a0122e9e1a7a9433a7111d2f9f22050795cd54c8 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com> Date: Fri, 2 Feb 2024 16:16:01 +0000 Subject: [PATCH 28/28] PR feedback --- .../rule_creation_ui/pages/rule_creation/helpers.ts | 9 ++++++--- .../detections/pages/detection_engine/rules/types.ts | 3 +++ .../rule_types/utils/wrap_suppressed_alerts.ts | 5 ++--- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_creation/helpers.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_creation/helpers.ts index 4e0077871fcd0..924dc4a62fd70 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_creation/helpers.ts +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_creation/helpers.ts @@ -45,7 +45,10 @@ import { DataSourceType, GroupByOptions, } from '../../../../detections/pages/detection_engine/rules/types'; -import type { RuleCreateProps } from '../../../../../common/api/detection_engine/model/rule_schema'; +import type { + RuleCreateProps, + AlertSuppression, +} from '../../../../../common/api/detection_engine/model/rule_schema'; import { stepActionsDefaultValue } from '../../../rule_creation/components/step_rule_actions'; import { DEFAULT_SUPPRESSION_MISSING_FIELDS_STRATEGY } from '../../../../../common/detection_engine/constants'; @@ -416,8 +419,8 @@ export const formatDefineStepData = (defineStepData: DefineStepRule): DefineStep ruleFields.groupByRadioSelection === GroupByOptions.PerTimePeriod ? ruleFields.groupByDuration : undefined, - missing_fields_strategy: - ruleFields.suppressionMissingFields || DEFAULT_SUPPRESSION_MISSING_FIELDS_STRATEGY, + missing_fields_strategy: (ruleFields.suppressionMissingFields || + DEFAULT_SUPPRESSION_MISSING_FIELDS_STRATEGY) as AlertSuppression['missing_fields_strategy'], }, } : {}; diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/types.ts b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/types.ts index 1077407ea4756..f57184a3a490b 100644 --- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/types.ts +++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/types.ts @@ -35,6 +35,8 @@ import type { AlertSuppressionMissingFieldsStrategy, InvestigationFields, RuleAction, + AlertSuppression, + ThresholdAlertSuppression, } from '../../../../../common/api/detection_engine/model/rule_schema'; import type { SortOrder } from '../../../../../common/api/detection_engine'; import type { EqlOptionsSelected } from '../../../../../common/search_strategy'; @@ -219,6 +221,7 @@ export interface DefineStepRuleJson { timestamp_field?: string; event_category_override?: string; tiebreaker_field?: string; + alert_suppression?: AlertSuppression | ThresholdAlertSuppression; } export interface AboutStepRuleJson { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts index f6f1d347fc481..b8ccacb9e515f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts @@ -33,9 +33,8 @@ import { buildBulkBody } from '../factories/utils/build_bulk_body'; import type { BuildReasonMessage } from './reason_formatters'; /** - * wraps suppressed threshold alerts - * first, transforms aggregation threshold buckets to hits - * creates instanceId hash, which is used to search suppressed on time interval alerts + * wraps suppressed alerts + * creates instanceId hash, which is used to search on time interval alerts * populates alert's suppression fields */ export const wrapSuppressedAlerts = ({