[DE] - Investigation fields followup

packages/kbn-securitysolution-utils/src/transform_data_to_ndjson/index.test.ts

@@ -20,7 +20,6 @@ const getRulesSchemaMock = (anchorDate: string = ANCHOR_DATE) => ({
   enabled: true,
   false_positives: ['false positive 1', 'false positive 2'],
   from: 'now-6m',
-  investigation_fields: ['custom.field1', 'custom.field2'],
   immutable: false,
   name: 'Query with a rule id',
   query: 'user.name: root or user.name: admin',

x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/common_attributes.ts

@@ -7,7 +7,7 @@
 
 import * as t from 'io-ts';
 import { listArray } from '@kbn/securitysolution-io-ts-list-types';
-import { NonEmptyString, version, UUID } from '@kbn/securitysolution-io-ts-types';
+import { NonEmptyString, version, UUID, NonEmptyArray } from '@kbn/securitysolution-io-ts-types';
 import { max_signals, threat } from '@kbn/securitysolution-io-ts-alerting-types';
 
 export type RuleObjectId = t.TypeOf<typeof RuleObjectId>;
@@ -55,14 +55,6 @@ export const RuleAuthorArray = t.array(t.string); // should be non-empty strings
 export type RuleFalsePositiveArray = t.TypeOf<typeof RuleFalsePositiveArray>;
 export const RuleFalsePositiveArray = t.array(t.string); // should be non-empty strings?
 
-/**
- * User defined fields to display in areas such as alert details and exceptions auto-populate
- * Field added in PR - https://github.com/elastic/kibana/pull/163235
- * @example const investigationFields: RuleCustomHighlightedFieldArray = ['host.os.name']
- */
-export type RuleCustomHighlightedFieldArray = t.TypeOf<typeof RuleCustomHighlightedFieldArray>;
-export const RuleCustomHighlightedFieldArray = t.array(NonEmptyString);
-
 export type RuleReferenceArray = t.TypeOf<typeof RuleReferenceArray>;
 export const RuleReferenceArray = t.array(t.string); // should be non-empty strings?
 
@@ -265,3 +257,32 @@ export const RelatedIntegration = t.exact(
  */
 export type RelatedIntegrationArray = t.TypeOf<typeof RelatedIntegrationArray>;
 export const RelatedIntegrationArray = t.array(RelatedIntegration);
+
+/**
+ * Schema for fields relating to investigation fields, these are user defined fields we use to highlight
+ * in various features in the UI such as alert details flyout and exceptions auto-population from alert.
+ * Added in PR #163235
+ * Right now we only have a single field but anticipate adding more related fields to store various
+ * configuration states such as `override` - where a user might say if they want only these fields to
+ * display, or if they want these fields + the fields we select. When expanding this field, it may look
+ * something like:
+ * export const investigationFields = t.intersection([
+ * t.exact(
+ *   t.type({
+ *     field_names: NonEmptyArray(NonEmptyString),
+ *   })
+ * ),
+ * t.exact(
+ *   t.partial({
+ *     overide: t.boolean,
+ *   })
+ * ),
+ * ]);
+ *
+ */
+export type InvestigationFields = t.TypeOf<typeof InvestigationFields>;
+export const InvestigationFields = t.exact(
+  t.type({
+    field_names: NonEmptyArray(NonEmptyString),
+  })
+);

x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_request_schema.test.ts

@@ -1290,10 +1290,38 @@ describe('rules schema', () => {
       expect(getPaths(left(message.errors))).toEqual(['invalid keys "data_view_id"']);
     });
 
-    test('You can optionally send in an array of investigation_fields', () => {
+    test('You can omit investigation_fields', () => {
+      // getCreateRulesSchemaMock doesn't include investigation_fields
+      const payload: RuleCreateProps = getCreateRulesSchemaMock();
+
+      const decoded = RuleCreateProps.decode(payload);
+      const checked = exactCheck(payload, decoded);
+      const message = pipe(checked, foldLeftRight);
+      expect(getPaths(left(message.errors))).toEqual([]);
+      expect(message.schema).toEqual(payload);
+    });
+
+    test('You cannot pass empty object for investigation_fields', () => {
+      const payload: Omit<RuleCreateProps, 'investigation_fields'> & {
+        investigation_fields: unknown;
+      } = {
+        ...getCreateRulesSchemaMock(),
+        investigation_fields: {},
+      };
+
+      const decoded = RuleCreateProps.decode(payload);
+      const checked = exactCheck(payload, decoded);
+      const message = pipe(checked, foldLeftRight);
+      expect(getPaths(left(message.errors))).toEqual([
+        'Invalid value "undefined" supplied to "investigation_fields,field_names"',
+      ]);
+      expect(message.schema).toEqual({});
+    });
+
+    test('You can send in investigation_fields', () => {
       const payload: RuleCreateProps = {
         ...getCreateRulesSchemaMock(),
-        investigation_fields: ['field1', 'field2'],
+        investigation_fields: { field_names: ['field1', 'field2'] },
       };
 
       const decoded = RuleCreateProps.decode(payload);
@@ -1303,19 +1331,49 @@ describe('rules schema', () => {
       expect(message.schema).toEqual(payload);
     });
 
-    test('You cannot send in an array of investigation_fields that are numbers', () => {
+    test('You cannot send in an empty array of investigation_fields.field_names', () => {
+      const payload = {
+        ...getCreateRulesSchemaMock(),
+        investigation_fields: { field_names: [] },
+      };
+
+      const decoded = RuleCreateProps.decode(payload);
+      const checked = exactCheck(payload, decoded);
+      const message = pipe(checked, foldLeftRight);
+      expect(getPaths(left(message.errors))).toEqual([
+        'Invalid value "[]" supplied to "investigation_fields,field_names"',
+      ]);
+      expect(message.schema).toEqual({});
+    });
+
+    test('You cannot send in an array of investigation_fields.field_names that are numbers', () => {
+      const payload = {
+        ...getCreateRulesSchemaMock(),
+        investigation_fields: { field_names: [0, 1, 2] },
+      };
+
+      const decoded = RuleCreateProps.decode(payload);
+      const checked = exactCheck(payload, decoded);
+      const message = pipe(checked, foldLeftRight);
+      expect(getPaths(left(message.errors))).toEqual([
+        'Invalid value "0" supplied to "investigation_fields,field_names"',
+        'Invalid value "1" supplied to "investigation_fields,field_names"',
+        'Invalid value "2" supplied to "investigation_fields,field_names"',
+      ]);
+      expect(message.schema).toEqual({});
+    });
+
+    test('You cannot send in investigation_fields without specifying fields', () => {
       const payload = {
         ...getCreateRulesSchemaMock(),
-        investigation_fields: [0, 1, 2],
+        investigation_fields: { foo: true },
       };
 
       const decoded = RuleCreateProps.decode(payload);
       const checked = exactCheck(payload, decoded);
       const message = pipe(checked, foldLeftRight);
       expect(getPaths(left(message.errors))).toEqual([
-        'Invalid value "0" supplied to "investigation_fields"',
-        'Invalid value "1" supplied to "investigation_fields"',
-        'Invalid value "2" supplied to "investigation_fields"',
+        'Invalid value "undefined" supplied to "investigation_fields,field_names"',
       ]);
       expect(message.schema).toEqual({});
     });

x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_response_schema.test.ts

@@ -234,47 +234,73 @@ describe('Rule response schema', () => {
   });
 
   describe('investigation_fields', () => {
-    test('it should validate rule with empty array for "investigation_fields"', () => {
+    test('it should validate rule with "investigation_fields"', () => {
       const payload = getRulesSchemaMock();
-      payload.investigation_fields = [];
+      payload.investigation_fields = { field_names: ['foo', 'bar'] };
 
       const decoded = RuleResponse.decode(payload);
       const checked = exactCheck(payload, decoded);
       const message = pipe(checked, foldLeftRight);
-      const expected = { ...getRulesSchemaMock(), investigation_fields: [] };
+      const expected = {
+        ...getRulesSchemaMock(),
+        investigation_fields: { field_names: ['foo', 'bar'] },
+      };
 
       expect(getPaths(left(message.errors))).toEqual([]);
       expect(message.schema).toEqual(expected);
     });
 
-    test('it should validate rule with "investigation_fields"', () => {
-      const payload = getRulesSchemaMock();
-      payload.investigation_fields = ['foo', 'bar'];
+    test('it should validate undefined for "investigation_fields"', () => {
+      const payload: RuleResponse = {
+        ...getRulesSchemaMock(),
+        investigation_fields: undefined,
+      };
 
       const decoded = RuleResponse.decode(payload);
       const checked = exactCheck(payload, decoded);
       const message = pipe(checked, foldLeftRight);
-      const expected = { ...getRulesSchemaMock(), investigation_fields: ['foo', 'bar'] };
+      const expected = { ...getRulesSchemaMock(), investigation_fields: undefined };
 
       expect(getPaths(left(message.errors))).toEqual([]);
       expect(message.schema).toEqual(expected);
     });
 
-    test('it should validate undefined for "investigation_fields"', () => {
+    test('it should validate "investigation_fields" not in schema', () => {
       const payload: RuleResponse = {
         ...getRulesSchemaMock(),
         investigation_fields: undefined,
       };
 
+      delete payload.investigation_fields;
+
       const decoded = RuleResponse.decode(payload);
       const checked = exactCheck(payload, decoded);
       const message = pipe(checked, foldLeftRight);
-      const expected = { ...getRulesSchemaMock(), investigation_fields: undefined };
+      const expected = getRulesSchemaMock();
 
       expect(getPaths(left(message.errors))).toEqual([]);
       expect(message.schema).toEqual(expected);
     });
 
+    test('it should NOT validate an empty array for "investigation_fields.field_names"', () => {
+      const payload: RuleResponse = {
+        ...getRulesSchemaMock(),
+        investigation_fields: {
+          field_names: [],
+        },
+      };
+
+      const decoded = RuleResponse.decode(payload);
+      const checked = exactCheck(payload, decoded);
+      const message = pipe(checked, foldLeftRight);
+
+      expect(getPaths(left(message.errors))).toEqual([
+        'Invalid value "[]" supplied to "investigation_fields,field_names"',
+        'Invalid value "{"field_names":[]}" supplied to "investigation_fields"',
+      ]);
+      expect(message.schema).toEqual({});
+    });
+
     test('it should NOT validate a string for "investigation_fields"', () => {
       const payload: Omit<RuleResponse, 'investigation_fields'> & {
         investigation_fields: string;

x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.ts

@@ -53,7 +53,7 @@ import {
   RelatedIntegrationArray,
   RequiredFieldArray,
   RuleAuthorArray,
-  RuleCustomHighlightedFieldArray,
+  InvestigationFields,
   RuleDescription,
   RuleFalsePositiveArray,
   RuleFilterArray,
@@ -117,7 +117,7 @@ export const baseSchema = buildRuleSchemas({
     output_index: AlertsIndex,
     namespace: AlertsIndexNamespace,
     meta: RuleMetadata,
-    investigation_fields: RuleCustomHighlightedFieldArray,
+    investigation_fields: InvestigationFields,
     // Throttle
     throttle: RuleActionThrottle,
   },

x-pack/plugins/security_solution/public/common/components/event_details/event_details.tsx

@@ -288,7 +288,7 @@ const EventDetailsComponent: React.FC<Props> = ({
                     isReadOnly,
                   }}
                   goToTable={goToTableTab}
-                  investigationFields={maybeRule?.investigation_fields ?? []}
+                  investigationFields={maybeRule?.investigation_fields?.field_names ?? []}
                 />
                 <EuiSpacer size="xl" />
                 <Insights

x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_creation/helpers.test.ts

@@ -555,7 +555,7 @@ describe('helpers', () => {
         severity_mapping: [],
         tags: ['tag1', 'tag2'],
         threat: getThreatMock(),
-        investigation_fields: ['foo', 'bar'],
+        investigation_fields: { field_names: ['foo', 'bar'] },
       };
 
       expect(result).toEqual(expected);
@@ -636,7 +636,7 @@ describe('helpers', () => {
         severity_mapping: [],
         tags: ['tag1', 'tag2'],
         threat: getThreatMock(),
-        investigation_fields: ['foo', 'bar'],
+        investigation_fields: { field_names: ['foo', 'bar'] },
       };
 
       expect(result).toEqual(expected);
@@ -661,7 +661,7 @@ describe('helpers', () => {
         severity_mapping: [],
         tags: ['tag1', 'tag2'],
         threat: getThreatMock(),
-        investigation_fields: ['foo', 'bar'],
+        investigation_fields: { field_names: ['foo', 'bar'] },
       };
 
       expect(result).toEqual(expected);
@@ -705,7 +705,7 @@ describe('helpers', () => {
         severity_mapping: [],
         tags: ['tag1', 'tag2'],
         threat: getThreatMock(),
-        investigation_fields: ['foo', 'bar'],
+        investigation_fields: { field_names: ['foo', 'bar'] },
       };
 
       expect(result).toEqual(expected);
@@ -758,7 +758,7 @@ describe('helpers', () => {
             ],
           },
         ],
-        investigation_fields: ['foo', 'bar'],
+        investigation_fields: { field_names: ['foo', 'bar'] },
       };
 
       expect(result).toEqual(expected);
@@ -787,13 +787,13 @@ describe('helpers', () => {
         threat: getThreatMock(),
         timestamp_override: 'event.ingest',
         timestamp_override_fallback_disabled: true,
-        investigation_fields: ['foo', 'bar'],
+        investigation_fields: { field_names: ['foo', 'bar'] },
       };
 
       expect(result).toEqual(expected);
     });
 
-    test('returns formatted object if investigation_fields is empty array', () => {
+    test('returns formatted object if investigationFields is empty array', () => {
       const mockStepData: AboutStepRule = {
         ...mockData,
         investigationFields: [],
@@ -817,7 +817,7 @@ describe('helpers', () => {
         timestamp_override: undefined,
         timestamp_override_fallback_disabled: undefined,
         threat: getThreatMock(),
-        investigation_fields: [],
+        investigation_fields: undefined,
       };
 
       expect(result).toEqual(expected);
@@ -843,7 +843,7 @@ describe('helpers', () => {
         severity_mapping: [],
         tags: ['tag1', 'tag2'],
         threat: getThreatMock(),
-        investigation_fields: ['foo', 'bar'],
+        investigation_fields: { field_names: ['foo', 'bar'] },
         threat_indicator_path: undefined,
         timestamp_override: undefined,
         timestamp_override_fallback_disabled: undefined,
@@ -855,7 +855,7 @@ describe('helpers', () => {
     test('returns formatted object if investigation_fields includes empty string', () => {
       const mockStepData: AboutStepRule = {
         ...mockData,
-        investigationFields: [' '],
+        investigationFields: ['  '],
       };
       const result = formatAboutStepData(mockStepData);
       const expected: AboutStepRuleJson = {
@@ -872,7 +872,7 @@ describe('helpers', () => {
         severity_mapping: [],
         tags: ['tag1', 'tag2'],
         threat: getThreatMock(),
-        investigation_fields: [],
+        investigation_fields: undefined,
         threat_indicator_path: undefined,
         timestamp_override: undefined,
         timestamp_override_fallback_disabled: undefined,

x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_creation/helpers.ts

@@ -502,6 +502,7 @@ export const formatAboutStepData = (
 
   const detectionExceptionLists =
     exceptionsList != null ? exceptionsList.filter((list) => list.type !== 'endpoint') : [];
+  const isinvestigationFieldsEmpty = investigationFields.every((item) => isEmpty(item.trim()));
 
   const resp = {
     author: author.filter((item) => !isEmpty(item)),
@@ -525,7 +526,9 @@ export const formatAboutStepData = (
       : {}),
     false_positives: falsePositives.filter((item) => !isEmpty(item)),
     references: references.filter((item) => !isEmpty(item)),
-    investigation_fields: investigationFields.filter((item) => !isEmpty(item.trim())),
+    investigation_fields: isinvestigationFieldsEmpty
+      ? undefined
+      : { field_names: investigationFields },
     risk_score: riskScore.value,
     risk_score_mapping: riskScore.isMappingChecked
       ? riskScore.mapping.filter((m) => m.field != null && m.field !== '')

x-pack/plugins/security_solution/public/detection_engine/rule_exceptions/components/add_exception_flyout/index.test.tsx

@@ -671,7 +671,7 @@ describe('When the add exception modal is opened', () => {
                 rules={[
                   {
                     ...getRulesSchemaMock(),
-                    investigation_fields: ['foo.bar'],
+                    investigation_fields: { field_names: ['foo.bar'] },
                     exceptions_list: [],
                   } as Rule,
                 ]}