Added ignoreFields feature

src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker

@@ -383,6 +383,7 @@ kibana_vars=(
     xpack.security.session.lifespan
     xpack.security.sessionTimeout
     xpack.securitySolution.alertMergeStrategy
+    xpack.securitySolution.alertIgnoreFields
     xpack.securitySolution.endpointResultListDefaultFirstPageIndex
     xpack.securitySolution.endpointResultListDefaultPageSize
     xpack.securitySolution.maxRuleImportExportSize

x-pack/plugins/security_solution/server/config.ts

@@ -27,6 +27,9 @@ export const configSchema = schema.object({
       defaultValue: 'missingFields',
     }
   ),
+  alertIgnoreFields: schema.arrayOf(schema.string(), {
+    defaultValue: [],
+  }),
   [SIGNALS_INDEX_KEY]: schema.string({ defaultValue: DEFAULT_SIGNALS_INDEX }),
 
   /**

x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/index.ts

@@ -26,6 +26,7 @@ export const createMockConfig = (): ConfigType => ({
   endpointResultListDefaultPageSize: 10,
   packagerTaskInterval: '60s',
   alertMergeStrategy: 'missingFields',
+  alertIgnoreFields: [],
   prebuiltRulesFromFileSystem: true,
   prebuiltRulesFromSavedObjects: false,
 });

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_factory.ts

@@ -40,6 +40,7 @@ export const createSecurityRuleTypeFactory: CreateSecurityRuleTypeFactory = ({
   lists,
   logger,
   mergeStrategy,
+  ignoreFields,
   ruleDataClient,
   ruleDataService,
 }) => (type) => {
@@ -208,6 +209,7 @@ export const createSecurityRuleTypeFactory: CreateSecurityRuleTypeFactory = ({
 
         const wrapHits = wrapHitsFactory({
           logger,
+          ignoreFields,
           mergeStrategy,
           ruleSO,
           spaceId,

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_bulk_body.ts

@@ -36,10 +36,11 @@ export const buildBulkBody = (
   ruleSO: SavedObject<AlertAttributes>,
   doc: SignalSourceHit,
   mergeStrategy: ConfigType['alertMergeStrategy'],
+  ignoreFields: ConfigType['alertIgnoreFields'],
   applyOverrides: boolean,
   buildReasonMessage: BuildReasonMessage
 ): RACAlert => {
-  const mergedDoc = getMergeStrategy(mergeStrategy)({ doc });
+  const mergedDoc = getMergeStrategy(mergeStrategy)({ doc, ignoreFields });
   const rule = applyOverrides
     ? buildRuleWithOverrides(ruleSO, mergedDoc._source ?? {})
     : buildRuleWithoutOverrides(ruleSO);

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/wrap_hits_factory.ts

@@ -7,7 +7,7 @@
 
 import { Logger } from 'kibana/server';
 
-import { SearchAfterAndBulkCreateParams, SignalSourceHit, WrapHits } from '../../signals/types';
+import { SearchAfterAndBulkCreateParams, WrapHits } from '../../signals/types';
 import { buildBulkBody } from './utils/build_bulk_body';
 import { generateId } from '../../signals/utils';
 import { filterDuplicateSignals } from '../../signals/filter_duplicate_signals';
@@ -16,13 +16,15 @@ import { WrappedRACAlert } from '../types';
 
 export const wrapHitsFactory = ({
   logger,
+  ignoreFields,
   mergeStrategy,
   ruleSO,
   spaceId,
 }: {
   logger: Logger;
   ruleSO: SearchAfterAndBulkCreateParams['ruleSO'];
   mergeStrategy: ConfigType['alertMergeStrategy'];
+  ignoreFields: ConfigType['alertIgnoreFields'];
   spaceId: string | null | undefined;
 }): WrapHits => (events, buildReasonMessage) => {
   try {
@@ -38,8 +40,9 @@ export const wrapHitsFactory = ({
         _source: buildBulkBody(
           spaceId,
           ruleSO,
-          doc as SignalSourceHit,
+          doc,
           mergeStrategy,
+          ignoreFields,
           true,
           buildReasonMessage
         ),

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/create_indicator_match_alert_type.test.ts

@@ -56,6 +56,7 @@ describe('Indicator Match Alerts', () => {
       experimentalFeatures: allowedExperimentalValues,
       lists: dependencies.lists,
       logger: dependencies.logger,
+      ignoreFields: [],
       mergeStrategy: 'allFields',
       ruleDataClient: dependencies.ruleDataClient,
       ruleDataService: dependencies.ruleDataService,
@@ -97,6 +98,7 @@ describe('Indicator Match Alerts', () => {
       lists: dependencies.lists,
       logger: dependencies.logger,
       mergeStrategy: 'allFields',
+      ignoreFields: [],
       ruleDataClient: dependencies.ruleDataClient,
       ruleDataService: dependencies.ruleDataService,
       version: '1.0.0',
@@ -135,6 +137,7 @@ describe('Indicator Match Alerts', () => {
       lists: dependencies.lists,
       logger: dependencies.logger,
       mergeStrategy: 'allFields',
+      ignoreFields: [],
       ruleDataClient: dependencies.ruleDataClient,
       ruleDataService: dependencies.ruleDataService,
       version: '1.0.0',

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/create_indicator_match_alert_type.ts

@@ -19,6 +19,7 @@ export const createIndicatorMatchAlertType = (createOptions: CreateRuleOptions)
     lists,
     logger,
     mergeStrategy,
+    ignoreFields,
     ruleDataClient,
     version,
     ruleDataService,
@@ -27,6 +28,7 @@ export const createIndicatorMatchAlertType = (createOptions: CreateRuleOptions)
     lists,
     logger,
     mergeStrategy,
+    ignoreFields,
     ruleDataClient,
     ruleDataService,
   });

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/ml/create_ml_alert_type.test.ts

@@ -98,6 +98,7 @@ describe('Machine Learning Alerts', () => {
       lists: dependencies.lists,
       logger: dependencies.logger,
       mergeStrategy: 'allFields',
+      ignoreFields: [],
       ml: mlMock,
       ruleDataClient: dependencies.ruleDataClient,
       ruleDataService: dependencies.ruleDataService,

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/ml/create_ml_alert_type.ts

@@ -14,11 +14,20 @@ import { createSecurityRuleTypeFactory } from '../create_security_rule_type_fact
 import { CreateRuleOptions } from '../types';
 
 export const createMlAlertType = (createOptions: CreateRuleOptions) => {
-  const { lists, logger, mergeStrategy, ml, ruleDataClient, ruleDataService } = createOptions;
+  const {
+    lists,
+    logger,
+    mergeStrategy,
+    ignoreFields,
+    ml,
+    ruleDataClient,
+    ruleDataService,
+  } = createOptions;
   const createSecurityRuleType = createSecurityRuleTypeFactory({
     lists,
     logger,
     mergeStrategy,
+    ignoreFields,
     ruleDataClient,
     ruleDataService,
   });

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/create_query_alert_type.test.ts

@@ -32,6 +32,7 @@ describe('Custom query alerts', () => {
       lists: dependencies.lists,
       logger: dependencies.logger,
       mergeStrategy: 'allFields',
+      ignoreFields: [],
       ruleDataClient: dependencies.ruleDataClient,
       ruleDataService: dependencies.ruleDataService,
       version: '1.0.0',
@@ -79,6 +80,7 @@ describe('Custom query alerts', () => {
       lists: dependencies.lists,
       logger: dependencies.logger,
       mergeStrategy: 'allFields',
+      ignoreFields: [],
       ruleDataClient: dependencies.ruleDataClient,
       ruleDataService: dependencies.ruleDataService,
       version: '1.0.0',

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/create_query_alert_type.ts

@@ -19,6 +19,7 @@ export const createQueryAlertType = (createOptions: CreateRuleOptions) => {
     lists,
     logger,
     mergeStrategy,
+    ignoreFields,
     ruleDataClient,
     version,
     ruleDataService,
@@ -27,6 +28,7 @@ export const createQueryAlertType = (createOptions: CreateRuleOptions) => {
     lists,
     logger,
     mergeStrategy,
+    ignoreFields,
     ruleDataClient,
     ruleDataService,
   });

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts

@@ -96,6 +96,7 @@ export type CreateSecurityRuleTypeFactory = (options: {
   lists: SetupPlugins['lists'];
   logger: Logger;
   mergeStrategy: ConfigType['alertMergeStrategy'];
+  ignoreFields: ConfigType['alertIgnoreFields'];
   ruleDataClient: IRuleDataClient;
   ruleDataService: IRuleDataPluginService;
 }) => <
@@ -124,6 +125,7 @@ export interface CreateRuleOptions {
   lists: SetupPlugins['lists'];
   logger: Logger;
   mergeStrategy: ConfigType['alertMergeStrategy'];
+  ignoreFields: ConfigType['alertIgnoreFields'];
   ml?: SetupPlugins['ml'];
   ruleDataClient: IRuleDataClient;
   version: string;

x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts

@@ -43,6 +43,7 @@ describe('buildBulkBody', () => {
       ruleSO,
       doc,
       'missingFields',
+      [],
       buildReasonMessage
     );
     // Timestamp will potentially always be different so remove it for the test
@@ -114,6 +115,7 @@ describe('buildBulkBody', () => {
       ruleSO,
       doc,
       'missingFields',
+      [],
       buildReasonMessage
     );
     // Timestamp will potentially always be different so remove it for the test
@@ -199,6 +201,7 @@ describe('buildBulkBody', () => {
       ruleSO,
       doc,
       'missingFields',
+      [],
       buildReasonMessage
     );
     // Timestamp will potentially always be different so remove it for the test
@@ -270,6 +273,7 @@ describe('buildBulkBody', () => {
       ruleSO,
       doc,
       'missingFields',
+      [],
       buildReasonMessage
     );
     // Timestamp will potentially always be different so remove it for the test
@@ -338,6 +342,7 @@ describe('buildBulkBody', () => {
       ruleSO,
       doc,
       'missingFields',
+      [],
       buildReasonMessage
     );
     // Timestamp will potentially always be different so remove it for the test
@@ -405,6 +410,7 @@ describe('buildBulkBody', () => {
       ruleSO,
       doc,
       'missingFields',
+      [],
       buildReasonMessage
     );
     const expected: Omit<SignalHit, '@timestamp'> & { someKey: string } = {
@@ -468,6 +474,7 @@ describe('buildBulkBody', () => {
       ruleSO,
       doc,
       'missingFields',
+      [],
       buildReasonMessage
     );
     const expected: Omit<SignalHit, '@timestamp'> & { someKey: string } = {
@@ -712,6 +719,7 @@ describe('buildSignalFromEvent', () => {
       ruleSO,
       true,
       'missingFields',
+      [],
       buildReasonMessage
     );
 

x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts

@@ -37,9 +37,10 @@ export const buildBulkBody = (
   ruleSO: SavedObject<AlertAttributes>,
   doc: SignalSourceHit,
   mergeStrategy: ConfigType['alertMergeStrategy'],
+  ignoreFields: ConfigType['alertIgnoreFields'],
   buildReasonMessage: BuildReasonMessage
 ): SignalHit => {
-  const mergedDoc = getMergeStrategy(mergeStrategy)({ doc });
+  const mergedDoc = getMergeStrategy(mergeStrategy)({ doc, ignoreFields });
   const rule = buildRuleWithOverrides(ruleSO, mergedDoc._source ?? {});
   const timestamp = new Date().toISOString();
   const reason = buildReasonMessage({ mergedDoc, rule });
@@ -76,11 +77,19 @@ export const buildSignalGroupFromSequence = (
   ruleSO: SavedObject<AlertAttributes>,
   outputIndex: string,
   mergeStrategy: ConfigType['alertMergeStrategy'],
+  ignoreFields: ConfigType['alertIgnoreFields'],
   buildReasonMessage: BuildReasonMessage
 ): WrappedSignalHit[] => {
   const wrappedBuildingBlocks = wrapBuildingBlocks(
     sequence.events.map((event) => {
-      const signal = buildSignalFromEvent(event, ruleSO, false, mergeStrategy, buildReasonMessage);
+      const signal = buildSignalFromEvent(
+        event,
+        ruleSO,
+        false,
+        mergeStrategy,
+        ignoreFields,
+        buildReasonMessage
+      );
       signal.signal.rule.building_block_type = 'default';
       return signal;
     }),
@@ -147,9 +156,10 @@ export const buildSignalFromEvent = (
   ruleSO: SavedObject<AlertAttributes>,
   applyOverrides: boolean,
   mergeStrategy: ConfigType['alertMergeStrategy'],
+  ignoreFields: ConfigType['alertIgnoreFields'],
   buildReasonMessage: BuildReasonMessage
 ): SignalHit => {
-  const mergedEvent = getMergeStrategy(mergeStrategy)({ doc: event });
+  const mergedEvent = getMergeStrategy(mergeStrategy)({ doc: event, ignoreFields });
   const rule = applyOverrides
     ? buildRuleWithOverrides(ruleSO, mergedEvent._source ?? {})
     : buildRuleWithoutOverrides(ruleSO);

x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts

@@ -74,6 +74,7 @@ describe('searchAfterAndBulkCreate', () => {
       ruleSO,
       signalsIndex: DEFAULT_SIGNALS_INDEX,
       mergeStrategy: 'missingFields',
+      ignoreFields: [],
     });
   });
 

x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.test.ts

@@ -195,6 +195,7 @@ describe('signal_rule_alert_type', () => {
       ml: mlMock,
       lists: listMock.createSetup(),
       mergeStrategy: 'missingFields',
+      ignoreFields: [],
       ruleDataService,
     });
 

x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts

@@ -82,6 +82,7 @@ export const signalRulesAlertType = ({
   ml,
   lists,
   mergeStrategy,
+  ignoreFields,
   ruleDataService,
 }: {
   logger: Logger;
@@ -91,6 +92,7 @@ export const signalRulesAlertType = ({
   ml: SetupPlugins['ml'];
   lists: SetupPlugins['lists'] | undefined;
   mergeStrategy: ConfigType['alertMergeStrategy'];
+  ignoreFields: ConfigType['alertIgnoreFields'];
   ruleDataService: IRuleDataPluginService;
 }): SignalRuleAlertTypeDefinition => {
   return {
@@ -275,12 +277,14 @@ export const signalRulesAlertType = ({
           ruleSO: savedObject,
           signalsIndex: params.outputIndex,
           mergeStrategy,
+          ignoreFields,
         });
 
         const wrapSequences = wrapSequencesFactory({
           ruleSO: savedObject,
           signalsIndex: params.outputIndex,
           mergeStrategy,
+          ignoreFields,
         });
 
         if (isMlRule(type)) {