[Detection Engine][Rules] - Adds custom highlighted fields option

packages/kbn-securitysolution-utils/src/transform_data_to_ndjson/index.test.ts

@@ -20,6 +20,7 @@ const getRulesSchemaMock = (anchorDate: string = ANCHOR_DATE) => ({
   enabled: true,
   false_positives: ['false positive 1', 'false positive 2'],
   from: 'now-6m',
+  custom_highlighted_fields: ['custom.field1', 'custom.field2'],
   immutable: false,
   name: 'Query with a rule id',
   query: 'user.name: root or user.name: admin',

x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/common_attributes.ts

@@ -55,6 +55,9 @@ export const RuleAuthorArray = t.array(t.string); // should be non-empty strings
 export type RuleFalsePositiveArray = t.TypeOf<typeof RuleFalsePositiveArray>;
 export const RuleFalsePositiveArray = t.array(t.string); // should be non-empty strings?
 
+export type RuleCustomHighlightedFieldArray = t.TypeOf<typeof RuleCustomHighlightedFieldArray>;
+export const RuleCustomHighlightedFieldArray = t.array(NonEmptyString);
+
 export type RuleReferenceArray = t.TypeOf<typeof RuleReferenceArray>;
 export const RuleReferenceArray = t.array(t.string); // should be non-empty strings?
 

x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_request_schema.test.ts

@@ -1289,6 +1289,36 @@ describe('rules schema', () => {
       expect(message.schema).toEqual({});
       expect(getPaths(left(message.errors))).toEqual(['invalid keys "data_view_id"']);
     });
+
+    test('You can optionally send in an array of custom_highlighted_fields', () => {
+      const payload: RuleCreateProps = {
+        ...getCreateRulesSchemaMock(),
+        custom_highlighted_fields: ['field1', 'field2'],
+      };
+
+      const decoded = RuleCreateProps.decode(payload);
+      const checked = exactCheck(payload, decoded);
+      const message = pipe(checked, foldLeftRight);
+      expect(getPaths(left(message.errors))).toEqual([]);
+      expect(message.schema).toEqual(payload);
+    });
+
+    test('You cannot send in an array of custom_highlighted_fields that are numbers', () => {
+      const payload = {
+        ...getCreateRulesSchemaMock(),
+        custom_highlighted_fields: [0, 1, 2],
+      };
+
+      const decoded = RuleCreateProps.decode(payload);
+      const checked = exactCheck(payload, decoded);
+      const message = pipe(checked, foldLeftRight);
+      expect(getPaths(left(message.errors))).toEqual([
+        'Invalid value "0" supplied to "custom_highlighted_fields"',
+        'Invalid value "1" supplied to "custom_highlighted_fields"',
+        'Invalid value "2" supplied to "custom_highlighted_fields"',
+      ]);
+      expect(message.schema).toEqual({});
+    });
   });
 
   describe('response', () => {

x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_response_schema.mock.ts

@@ -64,6 +64,7 @@ const getResponseBaseParams = (anchorDate: string = ANCHOR_DATE): SharedResponse
   timestamp_override: undefined,
   timestamp_override_fallback_disabled: undefined,
   namespace: undefined,
+  custom_highlighted_fields: undefined,
 });
 
 export const getRulesSchemaMock = (anchorDate: string = ANCHOR_DATE): QueryRule => ({
@@ -77,6 +78,7 @@ export const getRulesSchemaMock = (anchorDate: string = ANCHOR_DATE): QueryRule
   saved_id: undefined,
   response_actions: undefined,
   alert_suppression: undefined,
+  custom_highlighted_fields: undefined,
 });
 
 export const getSavedQuerySchemaMock = (anchorDate: string = ANCHOR_DATE): SavedQueryRule => ({

x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_response_schema.test.ts

@@ -232,4 +232,65 @@ describe('Rule response schema', () => {
       expect(message.schema).toEqual({});
     });
   });
+
+  describe('custom_highlighted_fields', () => {
+    test('it should validate rule with empty array for "custom_highlighted_fields"', () => {
+      const payload = getRulesSchemaMock();
+      payload.custom_highlighted_fields = [];
+
+      const decoded = RuleResponse.decode(payload);
+      const checked = exactCheck(payload, decoded);
+      const message = pipe(checked, foldLeftRight);
+      const expected = { ...getRulesSchemaMock(), custom_highlighted_fields: [] };
+
+      expect(getPaths(left(message.errors))).toEqual([]);
+      expect(message.schema).toEqual(expected);
+    });
+
+    test('it should validate rule with "custom_highlighted_fields"', () => {
+      const payload = getRulesSchemaMock();
+      payload.custom_highlighted_fields = ['foo', 'bar'];
+
+      const decoded = RuleResponse.decode(payload);
+      const checked = exactCheck(payload, decoded);
+      const message = pipe(checked, foldLeftRight);
+      const expected = { ...getRulesSchemaMock(), custom_highlighted_fields: ['foo', 'bar'] };
+
+      expect(getPaths(left(message.errors))).toEqual([]);
+      expect(message.schema).toEqual(expected);
+    });
+
+    test('it should validate undefined for "custom_highlighted_fields"', () => {
+      const payload: RuleResponse = {
+        ...getRulesSchemaMock(),
+        custom_highlighted_fields: undefined,
+      };
+
+      const decoded = RuleResponse.decode(payload);
+      const checked = exactCheck(payload, decoded);
+      const message = pipe(checked, foldLeftRight);
+      const expected = { ...getRulesSchemaMock(), custom_highlighted_fields: undefined };
+
+      expect(getPaths(left(message.errors))).toEqual([]);
+      expect(message.schema).toEqual(expected);
+    });
+
+    test('it should NOT validate a string for "custom_highlighted_fields"', () => {
+      const payload: Omit<RuleResponse, 'custom_highlighted_fields'> & {
+        custom_highlighted_fields: string;
+      } = {
+        ...getRulesSchemaMock(),
+        custom_highlighted_fields: 'foo',
+      };
+
+      const decoded = RuleResponse.decode(payload);
+      const checked = exactCheck(payload, decoded);
+      const message = pipe(checked, foldLeftRight);
+
+      expect(getPaths(left(message.errors))).toEqual([
+        'Invalid value "foo" supplied to "custom_highlighted_fields"',
+      ]);
+      expect(message.schema).toEqual({});
+    });
+  });
 });

x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.ts

@@ -53,6 +53,7 @@ import {
   RelatedIntegrationArray,
   RequiredFieldArray,
   RuleAuthorArray,
+  RuleCustomHighlightedFieldArray,
   RuleDescription,
   RuleFalsePositiveArray,
   RuleFilterArray,
@@ -116,6 +117,7 @@ export const baseSchema = buildRuleSchemas({
     output_index: AlertsIndex,
     namespace: AlertsIndexNamespace,
     meta: RuleMetadata,
+    custom_highlighted_fields: RuleCustomHighlightedFieldArray,
     // Throttle
     throttle: RuleActionThrottle,
   },

x-pack/plugins/security_solution/cypress/e2e/exceptions/alerts_table_flow/rule_exceptions/auto_populate_with_alert_data.cy.ts

@@ -49,7 +49,7 @@ describe('Auto populate exception with Alert data', () => {
     cy.task('esArchiverResetKibana');
     cy.task('esArchiverLoad', 'endpoint');
     login();
-    createRule(getEndpointRule());
+    createRule({ ...getEndpointRule(), custom_highlighted_fields: ['message'] });
     visitWithoutDateRange(DETECTIONS_RULE_MANAGEMENT_URL);
     goToRuleDetails();
     waitForAlertsToPopulate();
@@ -67,6 +67,7 @@ describe('Auto populate exception with Alert data', () => {
     addExceptionFromFirstAlert();
 
     const highlightedFieldsBasedOnAlertDoc = [
+      'message',
       'host.name',
       'agent.id',
       'user.name',

x-pack/plugins/security_solution/cypress/objects/rule.ts

@@ -532,6 +532,7 @@ export const expectedExportedRule = (ruleResponse: Cypress.Response<RuleResponse
     immutable,
     related_integrations: relatedIntegrations,
     setup,
+    custom_highlighted_fields: customHighlightedFields,
   } = ruleResponse.body;
 
   let query: string | undefined;
@@ -558,6 +559,7 @@ export const expectedExportedRule = (ruleResponse: Cypress.Response<RuleResponse
     severity,
     note,
     output_index: '',
+    custom_highlighted_fields: customHighlightedFields,
     author,
     false_positives: falsePositives,
     from,

x-pack/plugins/security_solution/public/common/components/event_details/get_alert_summary_rows.tsx

@@ -215,6 +215,15 @@ function getFieldsByRuleType(ruleType?: string): EventSummaryField[] {
   }
 }
 
+/**
+ * Gets the fields to display based on custom rules and configuration
+ * @param customs The list of custom-defined fields to display
+ * @returns The list of custom-defined fields to display
+ */
+function getHighlightedFieldsOverride(customs: string[]): EventSummaryField[] {
+  return customs.map((field) => ({ id: field }));
+}
+
 /**
   This function is exported because it is used in the Exception Component to
   populate the conditions with the Highlighted Fields. Additionally, the new
@@ -229,12 +238,15 @@ export function getEventFieldsToDisplay({
   eventCategories,
   eventCode,
   eventRuleType,
+  highlightedFieldsOverride,
 }: {
   eventCategories: EventCategories;
   eventCode?: string;
   eventRuleType?: string;
+  highlightedFieldsOverride: string[];
 }): EventSummaryField[] {
   const fields = [
+    ...getHighlightedFieldsOverride(highlightedFieldsOverride),
     ...alwaysDisplayedFields,
     ...getFieldsByCategory(eventCategories),
     ...getFieldsByEventCode(eventCode, eventCategories),
@@ -281,11 +293,13 @@ export const getSummaryRows = ({
   eventId,
   isDraggable = false,
   isReadOnly = false,
+  customHighlightedFields,
 }: {
   data: TimelineEventsDetailsItem[];
   browserFields: BrowserFields;
   scopeId: string;
   eventId: string;
+  customHighlightedFields?: string[];
   isDraggable?: boolean;
   isReadOnly?: boolean;
 }) => {
@@ -306,6 +320,7 @@ export const getSummaryRows = ({
     eventCategories,
     eventCode,
     eventRuleType,
+    highlightedFieldsOverride: customHighlightedFields ?? [],
   });
 
   return data != null

x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_creation/helpers.test.ts

@@ -555,6 +555,7 @@ describe('helpers', () => {
         severity_mapping: [],
         tags: ['tag1', 'tag2'],
         threat: getThreatMock(),
+        custom_highlighted_fields: ['foo', 'bar'],
       };
 
       expect(result).toEqual(expected);
@@ -635,6 +636,7 @@ describe('helpers', () => {
         severity_mapping: [],
         tags: ['tag1', 'tag2'],
         threat: getThreatMock(),
+        custom_highlighted_fields: ['foo', 'bar'],
       };
 
       expect(result).toEqual(expected);
@@ -659,6 +661,7 @@ describe('helpers', () => {
         severity_mapping: [],
         tags: ['tag1', 'tag2'],
         threat: getThreatMock(),
+        custom_highlighted_fields: ['foo', 'bar'],
       };
 
       expect(result).toEqual(expected);
@@ -702,6 +705,7 @@ describe('helpers', () => {
         severity_mapping: [],
         tags: ['tag1', 'tag2'],
         threat: getThreatMock(),
+        custom_highlighted_fields: ['foo', 'bar'],
       };
 
       expect(result).toEqual(expected);
@@ -754,6 +758,7 @@ describe('helpers', () => {
             ],
           },
         ],
+        custom_highlighted_fields: ['foo', 'bar'],
       };
 
       expect(result).toEqual(expected);
@@ -782,6 +787,95 @@ describe('helpers', () => {
         threat: getThreatMock(),
         timestamp_override: 'event.ingest',
         timestamp_override_fallback_disabled: true,
+        custom_highlighted_fields: ['foo', 'bar'],
+      };
+
+      expect(result).toEqual(expected);
+    });
+
+    test('returns formatted object if custom_highlighted_fields is empty array', () => {
+      const mockStepData: AboutStepRule = {
+        ...mockData,
+        customHighlightedFields: [],
+      };
+      const result = formatAboutStepData(mockStepData);
+      const expected: AboutStepRuleJson = {
+        author: ['Elastic'],
+        description: '24/7',
+        false_positives: ['test'],
+        license: 'Elastic License',
+        name: 'Query with rule-id',
+        note: '# this is some markdown documentation',
+        references: ['www.test.co'],
+        risk_score: 21,
+        risk_score_mapping: [],
+        severity: 'low',
+        severity_mapping: [],
+        tags: ['tag1', 'tag2'],
+        rule_name_override: undefined,
+        threat_indicator_path: undefined,
+        timestamp_override: undefined,
+        timestamp_override_fallback_disabled: undefined,
+        threat: getThreatMock(),
+        custom_highlighted_fields: [],
+      };
+
+      expect(result).toEqual(expected);
+    });
+
+    test('returns formatted object with custom_highlighted_fields', () => {
+      const mockStepData: AboutStepRule = {
+        ...mockData,
+        customHighlightedFields: ['foo', 'bar'],
+      };
+      const result = formatAboutStepData(mockStepData);
+      const expected: AboutStepRuleJson = {
+        author: ['Elastic'],
+        description: '24/7',
+        false_positives: ['test'],
+        license: 'Elastic License',
+        name: 'Query with rule-id',
+        note: '# this is some markdown documentation',
+        references: ['www.test.co'],
+        risk_score: 21,
+        risk_score_mapping: [],
+        severity: 'low',
+        severity_mapping: [],
+        tags: ['tag1', 'tag2'],
+        threat: getThreatMock(),
+        custom_highlighted_fields: ['foo', 'bar'],
+        threat_indicator_path: undefined,
+        timestamp_override: undefined,
+        timestamp_override_fallback_disabled: undefined,
+      };
+
+      expect(result).toEqual(expected);
+    });
+
+    test('returns formatted object if custom_highlighted_fields includes empty string', () => {
+      const mockStepData: AboutStepRule = {
+        ...mockData,
+        customHighlightedFields: [' '],
+      };
+      const result = formatAboutStepData(mockStepData);
+      const expected: AboutStepRuleJson = {
+        author: ['Elastic'],
+        description: '24/7',
+        false_positives: ['test'],
+        license: 'Elastic License',
+        name: 'Query with rule-id',
+        note: '# this is some markdown documentation',
+        references: ['www.test.co'],
+        risk_score: 21,
+        risk_score_mapping: [],
+        severity: 'low',
+        severity_mapping: [],
+        tags: ['tag1', 'tag2'],
+        threat: getThreatMock(),
+        custom_highlighted_fields: [],
+        threat_indicator_path: undefined,
+        timestamp_override: undefined,
+        timestamp_override_fallback_disabled: undefined,
       };
 
       expect(result).toEqual(expected);

x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_creation/helpers.ts

@@ -485,6 +485,7 @@ export const formatAboutStepData = (
   const {
     author,
     falsePositives,
+    customHighlightedFields,
     references,
     riskScore,
     severity,
@@ -524,6 +525,7 @@ export const formatAboutStepData = (
       : {}),
     false_positives: falsePositives.filter((item) => !isEmpty(item)),
     references: references.filter((item) => !isEmpty(item)),
+    custom_highlighted_fields: customHighlightedFields.filter((item) => !isEmpty(item.trim())),
     risk_score: riskScore.value,
     risk_score_mapping: riskScore.isMappingChecked
       ? riskScore.mapping.filter((m) => m.field != null && m.field !== '')