[Security Solution][Detection Engine] adds alerts Suppression to thre…

…shold rule (#171423) ## Summary - addresses milestone 1 of elastic/security-team#7773 epic - adds alerts suppression capabilities to threshold rule type - to enable alerts suppression for threshold rule type use experimental feature flag `alertSuppressionForThresholdRuleEnabled` in kibana.yml ``` xpack.securitySolution.enableExperimental: - alertSuppressionForThresholdRuleEnabled ``` - similarly to query rule Platinum license is required ### UI Few changes in comparison with custom query alerts suppression 1. Suppress by fields removed, since suppression is performed on Threshold Groups By fields 2. Instead, we show checkbox - so user can opt-in for alert suppression (either by selected threshold fields or w/o any) 3. Only time interval is radio button is available, suppression in rule execution is disabled(Threshold rule itself 'suppress' by grouping during rule execution) Demo video, shows suppression on interval when users select threshold group by fields and when do not https://github.com/elastic/kibana/assets/92328789/7dc476ad-0d0f-4e40-8042-d4dd552759d9 <details> <summary> Suppression is enabled, threshold fields selected </summary> <img width="1056" alt="Screenshot 2023-11-27 at 16 44 04" src="https://github.com/elastic/kibana/assets/92328789/c654a7b2-6f70-4a04-8a85-48b2a2445014"> </details> <details> <summary> Suppression is not enabled, threshold fields selected </summary> <img width="1036" alt="Screenshot 2023-11-27 at 16 44 27" src="https://github.com/elastic/kibana/assets/92328789/1cd4145f-df17-4b41-954b-c64de9eac0ff"> </details> <details> <summary> Suppression is not enabled, threshold fields not selected </summary> <img width="1050" alt="Screenshot 2023-11-27 at 16 44 42" src="https://github.com/elastic/kibana/assets/92328789/8b64a65b-4abd-4334-a1a5-e2b00fe7d8a5"> </details> ### Checklist - [x] Functional changes are hidden behind a feature flag Feature flag `alertSuppressionForThresholdRuleEnabled` - [x] Functional changes are covered with a test plan and automated tests. Test plan in progress(cc @vgomez-el), unit/ftr/cypress tests added to cover alert suppression functionality added - [x] Stability of new and changed tests is verified using the [Flaky Test Runner](https://ci-stats.kibana.dev/trigger_flaky_test_runner). [FTR ESS & Serverless tests](https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/4057) [Cypress ESS](https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/4058) [Cypress Serverless](https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/4059) - [ ] Comprehensive manual testing is done by two engineers: the PR author and one of the PR reviewers. Changes are tested in both ESS and Serverless. - [x] Mapping changes are accompanied by a technical design document. It can be a GitHub issue or an RFC explaining the changes. The design document is shared with and approved by the appropriate teams and individual stakeholders. Existing AlertSuppression schema field is used for Threshold rule, similarly to Query. But only `duration` field is applicable and required - [x] Functional changes are communicated to the Docs team. A ticket or PR is opened in https://github.com/elastic/security-docs. The following information is included: any feature flags used, affected environments (Serverless, ESS, or both). elastic/security-docs#4315 --------- Co-authored-by: kibanamachine <[email protected]>
elastic · Dec 4, 2023 · be7f6cf · be7f6cf
1 parent 96b8c5f
commit be7f6cf
Show file tree

Hide file tree

Showing 61 changed files with 2,043 additions and 313 deletions.
diff --git a/.../security_solution/common/api/detection_engine/model/rule_schema/common_attributes.gen.ts b/.../security_solution/common/api/detection_engine/model/rule_schema/common_attributes.gen.ts
@@ -432,3 +432,9 @@ export const RuleExceptionList = z.object({
    */
   namespace_type: z.enum(['agnostic', 'single']),
 });
+
+export type AlertSuppressionDuration = z.infer<typeof AlertSuppressionDuration>;
+export const AlertSuppressionDuration = z.object({
+  value: z.number().int().min(1),
+  unit: z.enum(['s', 'm', 'h']),
+});
diff --git a/...rity_solution/common/api/detection_engine/model/rule_schema/common_attributes.schema.yaml b/...rity_solution/common/api/detection_engine/model/rule_schema/common_attributes.schema.yaml
@@ -471,3 +471,19 @@ components:
         - list_id
         - type
         - namespace_type
+
+    AlertSuppressionDuration:
+      type: object
+      properties:
+        value:
+          type: integer
+          minimum: 1
+        unit:
+          type: string
+          enum:
+            - s
+            - m
+            - h
+      required:
+        - value
+        - unit
diff --git a/...curity_solution/common/api/detection_engine/model/rule_schema/rule_request_schema.test.ts b/...curity_solution/common/api/detection_engine/model/rule_schema/rule_request_schema.test.ts
@@ -1212,4 +1212,53 @@ describe('rules schema', () => {
       expect(stringifyZodError(result.error)).toEqual('investigation_fields.field_names: Required');
     });
   });
+
+  describe('alerts suppression', () => {
+    test('should drop suppression fields apart from duration for "threshold" rule type', () => {
+      const payload = {
+        ...getCreateThresholdRulesSchemaMock(),
+        alert_suppression: {
+          group_by: ['host.name'],
+          duration: { value: 5, unit: 'm' },
+          missing_field_strategy: 'suppress',
+        },
+      };
+
+      const result = RuleCreateProps.safeParse(payload);
+      expectParseSuccess(result);
+      expect(result.data).toEqual({
+        ...payload,
+        alert_suppression: {
+          duration: { value: 5, unit: 'm' },
+        },
+      });
+    });
+    test('should validate only suppression duration for "threshold" rule type', () => {
+      const payload = {
+        ...getCreateThresholdRulesSchemaMock(),
+        alert_suppression: {
+          duration: { value: 5, unit: 'm' },
+        },
+      };
+
+      const result = RuleCreateProps.safeParse(payload);
+      expectParseSuccess(result);
+      expect(result.data).toEqual(payload);
+    });
+    test('should throw error if alert suppression duration is absent for "threshold" rule type', () => {
+      const payload = {
+        ...getCreateThresholdRulesSchemaMock(),
+        alert_suppression: {
+          group_by: ['host.name'],
+          missing_field_strategy: 'suppress',
+        },
+      };
+
+      const result = RuleCreateProps.safeParse(payload);
+      expectParseError(result);
+      expect(stringifyZodError(result.error)).toMatchInlineSnapshot(
+        `"alert_suppression.duration: Required"`
+      );
+    });
+  });
 });
diff --git a/...ugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.gen.ts b/...ugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.gen.ts
@@ -69,7 +69,10 @@ import {
 } from './specific_attributes/eql_attributes.gen';
 import { ResponseAction } from '../rule_response_actions/response_actions.gen';
 import { AlertSuppression } from './specific_attributes/query_attributes.gen';
-import { Threshold } from './specific_attributes/threshold_attributes.gen';
+import {
+  Threshold,
+  ThresholdAlertSuppression,
+} from './specific_attributes/threshold_attributes.gen';
 import {
   ThreatQuery,
   ThreatMapping,
@@ -353,6 +356,7 @@ export const ThresholdRuleOptionalFields = z.object({
   data_view_id: DataViewId.optional(),
   filters: RuleFilterArray.optional(),
   saved_id: SavedQueryId.optional(),
+  alert_suppression: ThresholdAlertSuppression.optional(),
 });
 
 export type ThresholdRuleDefaultableFields = z.infer<typeof ThresholdRuleDefaultableFields>;

diff --git a/.../security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.schema.yaml b/.../security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.schema.yaml
@@ -508,6 +508,8 @@ components:
           $ref: './common_attributes.schema.yaml#/components/schemas/RuleFilterArray'
         saved_id:
           $ref: './common_attributes.schema.yaml#/components/schemas/SavedQueryId'
+        alert_suppression:
+          $ref: './specific_attributes/threshold_attributes.schema.yaml#/components/schemas/ThresholdAlertSuppression'
 
     ThresholdRuleDefaultableFields:
       type: object

diff --git a/...common/api/detection_engine/model/rule_schema/specific_attributes/query_attributes.gen.ts b/...common/api/detection_engine/model/rule_schema/specific_attributes/query_attributes.gen.ts
@@ -12,6 +12,8 @@ import { z } from 'zod';
  * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator.
  */
 
+import { AlertSuppressionDuration } from '../common_attributes.gen';
+
 /**
   * Describes how alerts will be generated for documents with missing suppress by fields:
 doNotSuppress - per each document a separate alert will be created
@@ -28,12 +30,6 @@ export const AlertSuppressionMissingFieldsStrategyEnum = AlertSuppressionMissing
 export type AlertSuppressionGroupBy = z.infer<typeof AlertSuppressionGroupBy>;
 export const AlertSuppressionGroupBy = z.array(z.string()).min(1).max(3);
 
-export type AlertSuppressionDuration = z.infer<typeof AlertSuppressionDuration>;
-export const AlertSuppressionDuration = z.object({
-  value: z.number().int().min(1),
-  unit: z.enum(['s', 'm', 'h']),
-});
-
 export type AlertSuppression = z.infer<typeof AlertSuppression>;
 export const AlertSuppression = z.object({
   group_by: AlertSuppressionGroupBy,

diff --git a/...n/api/detection_engine/model/rule_schema/specific_attributes/query_attributes.schema.yaml b/...n/api/detection_engine/model/rule_schema/specific_attributes/query_attributes.schema.yaml
@@ -23,29 +23,13 @@ components:
       minItems: 1
       maxItems: 3
 
-    AlertSuppressionDuration:
-      type: object
-      properties:
-        value:
-          type: integer
-          minimum: 1
-        unit:
-          type: string
-          enum:
-            - s
-            - m
-            - h
-      required:
-        - value
-        - unit
-
     AlertSuppression:
       type: object
       properties:
         group_by:
           $ref: '#/components/schemas/AlertSuppressionGroupBy'
         duration:
-          $ref: '#/components/schemas/AlertSuppressionDuration'
+          $ref: '../common_attributes.schema.yaml#/components/schemas/AlertSuppressionDuration'
         missing_fields_strategy:
           $ref: '#/components/schemas/AlertSuppressionMissingFieldsStrategy'
       required:
@@ -57,7 +41,7 @@ components:
         groupBy:
           $ref: '#/components/schemas/AlertSuppressionGroupBy'
         duration:
-          $ref: '#/components/schemas/AlertSuppressionDuration'
+          $ref: '../common_attributes.schema.yaml#/components/schemas/AlertSuppressionDuration'
         missingFieldsStrategy:
           $ref: '#/components/schemas/AlertSuppressionMissingFieldsStrategy'
       required:

diff --git a/...on/api/detection_engine/model/rule_schema/specific_attributes/threshold_attributes.gen.ts b/...on/api/detection_engine/model/rule_schema/specific_attributes/threshold_attributes.gen.ts
@@ -12,6 +12,8 @@ import { z } from 'zod';
  * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator.
  */
 
+import { AlertSuppressionDuration } from '../common_attributes.gen';
+
 export type ThresholdCardinality = z.infer<typeof ThresholdCardinality>;
 export const ThresholdCardinality = z.array(
   z.object({
@@ -58,3 +60,8 @@ export const ThresholdWithCardinality = z.object({
   value: ThresholdValue,
   cardinality: ThresholdCardinality,
 });
+
+export type ThresholdAlertSuppression = z.infer<typeof ThresholdAlertSuppression>;
+export const ThresholdAlertSuppression = z.object({
+  duration: AlertSuppressionDuration,
+});
diff --git a/...i/detection_engine/model/rule_schema/specific_attributes/threshold_attributes.schema.yaml b/...i/detection_engine/model/rule_schema/specific_attributes/threshold_attributes.schema.yaml
@@ -78,3 +78,11 @@ components:
         - field
         - value
         - cardinality
+
+    ThresholdAlertSuppression:
+      type: object
+      properties:
+        duration: 
+          $ref: '../common_attributes.schema.yaml#/components/schemas/AlertSuppressionDuration'
+      required:
+        - duration
diff --git a/x-pack/plugins/security_solution/common/experimental_features.ts b/x-pack/plugins/security_solution/common/experimental_features.ts
@@ -115,6 +115,11 @@ export const allowedExperimentalValues = Object.freeze({
    */
   protectionUpdatesEnabled: true,
 
+  /**
+   * Enables alerts suppression for threshold rules
+   */
+  alertSuppressionForThresholdRuleEnabled: false,
+
   /**
    * Disables the timeline save tour.
    * This flag is used to disable the tour in cypress tests.

diff --git a/...security_solution/public/detection_engine/rule_creation_ui/pages/rule_creation/helpers.ts b/...security_solution/public/detection_engine/rule_creation_ui/pages/rule_creation/helpers.ts
@@ -434,6 +434,9 @@ export const formatDefineStepData = (defineStepData: DefineStepRule): DefineStep
                   ]
                 : [],
           },
+          ...(ruleFields.enableThresholdSuppression && {
+            alert_suppression: { duration: ruleFields.groupByDuration },
+          }),
         }),
       }
     : isThreatMatchFields(ruleFields)
@@ -505,6 +508,7 @@ export const formatDefineStepData = (defineStepData: DefineStepRule): DefineStep
             saved_id: ruleFields.queryBar.saved_id,
           }),
       };
+
   return {
     ...baseFields,
     ...typeFields,

diff --git a/.../security_solution/public/detection_engine/rule_creation_ui/pages/rule_creation/index.tsx b/.../security_solution/public/detection_engine/rule_creation_ui/pages/rule_creation/index.tsx
@@ -523,6 +523,8 @@ const CreateRulePageComponent: React.FC = () => {
             shouldLoadQueryDynamically={defineStepData.shouldLoadQueryDynamically}
             queryBarTitle={defineStepData.queryBar.title}
             queryBarSavedId={defineStepData.queryBar.saved_id}
+            thresholdFields={defineStepData.threshold.field}
+            enableThresholdSuppression={defineStepData.enableThresholdSuppression}
           />
           <NextStep
             dataTestSubj="define-continue"
@@ -557,6 +559,8 @@ const CreateRulePageComponent: React.FC = () => {
       memoDefineStepReadOnly,
       setEqlOptionsSelected,
       threatIndicesConfig,
+      defineStepData.threshold.field,
+      defineStepData.enableThresholdSuppression,
     ]
   );
   const memoDefineStepExtraAction = useMemo(

diff --git a/...s/security_solution/public/detection_engine/rule_creation_ui/pages/rule_editing/index.tsx b/...s/security_solution/public/detection_engine/rule_creation_ui/pages/rule_editing/index.tsx
@@ -261,6 +261,8 @@ const EditRulePageComponent: FC<{ rule: RuleResponse }> = ({ rule }) => {
                   shouldLoadQueryDynamically={defineStepData.shouldLoadQueryDynamically}
                   queryBarTitle={defineStepData.queryBar.title}
                   queryBarSavedId={defineStepData.queryBar.saved_id}
+                  thresholdFields={defineStepData.threshold.field}
+                  enableThresholdSuppression={defineStepData.enableThresholdSuppression}
                 />
               )}
               <EuiSpacer />

diff --git a/...blic/detection_engine/rule_management/components/rule_details/rule_definition_section.tsx b/...blic/detection_engine/rule_management/components/rule_details/rule_definition_section.tsx
@@ -55,6 +55,8 @@ import { TechnicalPreviewBadge } from '../../../../detections/components/rules/t
 import { BadgeList } from './badge_list';
 import { DESCRIPTION_LIST_COLUMN_WIDTHS } from './constants';
 import * as i18n from './translations';
+import { useIsExperimentalFeatureEnabled } from '../../../../common/hooks/use_experimental_features';
+import type { ExperimentalFeatures } from '../../../../../common/experimental_features';
 
 interface SavedQueryNameProps {
   savedQueryName: string;
@@ -427,7 +429,8 @@ const HistoryWindowSize = ({ historyWindowStart }: HistoryWindowSizeProps) => {
 const prepareDefinitionSectionListItems = (
   rule: Partial<RuleResponse>,
   isInteractive: boolean,
-  savedQuery?: SavedQuery
+  savedQuery: SavedQuery | undefined,
+  { alertSuppressionForThresholdRuleEnabled }: Partial<ExperimentalFeatures>
 ): EuiDescriptionListProps['listItems'] => {
   const definitionSectionListItems: EuiDescriptionListProps['listItems'] = [];
 
@@ -658,36 +661,42 @@ const prepareDefinitionSectionListItems = (
   }
 
   if ('alert_suppression' in rule && rule.alert_suppression) {
-    definitionSectionListItems.push({
-      title: (
-        <span data-test-subj="alertSuppressionGroupByPropertyTitle">
-          <AlertSuppressionTitle title={i18n.SUPPRESS_ALERTS_BY_FIELD_LABEL} />
-        </span>
-      ),
-      description: <SuppressAlertsByField fields={rule.alert_suppression.group_by} />,
-    });
+    if ('group_by' in rule.alert_suppression) {
+      definitionSectionListItems.push({
+        title: (
+          <span data-test-subj="alertSuppressionGroupByPropertyTitle">
+            <AlertSuppressionTitle title={i18n.SUPPRESS_ALERTS_BY_FIELD_LABEL} />
+          </span>
+        ),
+        description: <SuppressAlertsByField fields={rule.alert_suppression.group_by} />,
+      });
+    }
 
-    definitionSectionListItems.push({
-      title: (
-        <span data-test-subj="alertSuppressionDurationPropertyTitle">
-          <AlertSuppressionTitle title={i18n.SUPPRESS_ALERTS_DURATION_FIELD_LABEL} />
-        </span>
-      ),
-      description: <SuppressAlertsDuration duration={rule.alert_suppression.duration} />,
-    });
+    if (rule.type !== 'threshold' || alertSuppressionForThresholdRuleEnabled) {
+      definitionSectionListItems.push({
+        title: (
+          <span data-test-subj="alertSuppressionDurationPropertyTitle">
+            <AlertSuppressionTitle title={i18n.SUPPRESS_ALERTS_DURATION_FIELD_LABEL} />
+          </span>
+        ),
+        description: <SuppressAlertsDuration duration={rule.alert_suppression.duration} />,
+      });
+    }
 
-    definitionSectionListItems.push({
-      title: (
-        <span data-test-subj="alertSuppressionSuppressionFieldPropertyTitle">
-          <AlertSuppressionTitle title={i18n.SUPPRESSION_FIELD_MISSING_FIELD_LABEL} />
-        </span>
-      ),
-      description: (
-        <MissingFieldsStrategy
-          missingFieldsStrategy={rule.alert_suppression.missing_fields_strategy}
-        />
-      ),
-    });
+    if ('missing_fields_strategy' in rule.alert_suppression) {
+      definitionSectionListItems.push({
+        title: (
+          <span data-test-subj="alertSuppressionSuppressionFieldPropertyTitle">
+            <AlertSuppressionTitle title={i18n.SUPPRESSION_FIELD_MISSING_FIELD_LABEL} />
+          </span>
+        ),
+        description: (
+          <MissingFieldsStrategy
+            missingFieldsStrategy={rule.alert_suppression.missing_fields_strategy}
+          />
+        ),
+      });
+    }
   }
 
   if ('new_terms_fields' in rule && rule.new_terms_fields && rule.new_terms_fields.length > 0) {
@@ -733,10 +742,15 @@ export const RuleDefinitionSection = ({
     ruleType: rule.type,
   });
 
+  const alertSuppressionForThresholdRuleEnabled = useIsExperimentalFeatureEnabled(
+    'alertSuppressionForThresholdRuleEnabled'
+  );
+
   const definitionSectionListItems = prepareDefinitionSectionListItems(
     rule,
     isInteractive,
-    savedQuery
+    savedQuery,
+    { alertSuppressionForThresholdRuleEnabled }
   );
 
   return (

diff --git a/...ution/public/detection_engine/rule_management_ui/components/rules_table/__mocks__/mock.ts b/...ution/public/detection_engine/rule_management_ui/components/rules_table/__mocks__/mock.ts
@@ -241,6 +241,7 @@ export const mockDefineStepRule = (): DefineStepRule => ({
     unit: 'm',
     value: 5,
   },
+  enableThresholdSuppression: false,
 });
 
 export const mockScheduleStepRule = (): ScheduleStepRule => ({

diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/duration_input/index.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/duration_input/index.tsx
@@ -85,6 +85,7 @@ const DurationInputComponent: React.FC<DurationInputProps> = ({
     { value: 'm', text: I18n.MINUTES },
     { value: 'h', text: I18n.HOURS },
   ],
+  ...props
 }: DurationInputProps) => {
   const { isInvalid, errorMessage } = getFieldValidityAndErrorMessage(durationValueField);
   const { value: durationValue, setValue: setDurationValue } = durationValueField;
@@ -106,7 +107,7 @@ const DurationInputComponent: React.FC<DurationInputProps> = ({
   );
 
   // EUI missing some props
-  const rest = { disabled: isDisabled };
+  const rest = { disabled: isDisabled, ...props };
 
   return (
     <StyledEuiFormRow error={errorMessage} isInvalid={isInvalid}>