[Security Solution] Migrate to fields API

packages/kbn-securitysolution-t-grid/src/mock/mock_event_details.ts

@@ -292,37 +292,17 @@ export const eventDetailsFormattedFields = [
   },
   {
     category: 'threat',
-    field: 'threat.enrichments.lazer.great.field',
-    values: ['grrrrr', 'grrrrr_2'],
-    originalValue: ['grrrrr', 'grrrrr_2'],
-    isObjectArray: false,
-  },
-  {
-    category: 'threat',
-    field: 'threat.enrichments.lazer.great.field.wowoe.fooooo',
-    values: ['grrrrr'],
-    originalValue: ['grrrrr'],
-    isObjectArray: false,
-  },
-  {
-    category: 'threat',
-    field: 'threat.enrichments.lazer.great.field.astring',
-    values: ['cool'],
-    originalValue: ['cool'],
-    isObjectArray: false,
-  },
-  {
-    category: 'threat',
-    field: 'threat.enrichments.lazer.great.field.aNumber',
-    values: ['1'],
-    originalValue: ['1'],
-    isObjectArray: false,
-  },
-  {
-    category: 'threat',
-    field: 'threat.enrichments.lazer.great.field.neat',
-    values: ['true'],
-    originalValue: ['true'],
-    isObjectArray: false,
+    field: 'threat.enrichments.lazer',
+    values: [
+      '{"great.field":["grrrrr"]}',
+      '{"great.field":["grrrrr_2"]}',
+      '{"great.field":[{"wowoe":[{"fooooo":["grrrrr"]}],"astring":"cool","aNumber":1,"neat":true}]}',
+    ],
+    originalValue: [
+      '{"great.field":["grrrrr"]}',
+      '{"great.field":["grrrrr_2"]}',
+      '{"great.field":[{"wowoe":[{"fooooo":["grrrrr"]}],"astring":"cool","aNumber":1,"neat":true}]}',
+    ],
+    isObjectArray: true,
   },
 ];

x-pack/plugins/security_solution/common/utils/field_formatters.test.ts

@@ -5,8 +5,8 @@
  * 2.0.
  */
 
-import type { EventHit, EventSource } from '../search_strategy';
-import { getDataFromFieldsHits, getDataFromSourceHits, getDataSafety } from './field_formatters';
+import type { EventHit } from '../search_strategy';
+import { getDataFromFieldsHits, getDataSafety } from './field_formatters';
 import { eventDetailsFormattedFields, eventHit } from '@kbn/securitysolution-t-grid';
 
 describe('Events Details Helpers', () => {
@@ -73,122 +73,20 @@ describe('Events Details Helpers', () => {
       const whackResultFields = [
         {
           category: 'crazy',
-          field: 'crazy.pants.matched.field',
-          values: ['matched_field'],
-          originalValue: ['matched_field'],
-          isObjectArray: false,
-        },
-        {
-          category: 'crazy',
-          field: 'crazy.pants.first_seen',
-          values: ['2021-02-22T17:29:25.195Z'],
-          originalValue: ['2021-02-22T17:29:25.195Z'],
-          isObjectArray: false,
-        },
-        {
-          category: 'crazy',
-          field: 'crazy.pants.provider',
-          values: ['yourself'],
-          originalValue: ['yourself'],
-          isObjectArray: false,
-        },
-        {
-          category: 'crazy',
-          field: 'crazy.pants.type',
-          values: ['custom'],
-          originalValue: ['custom'],
-          isObjectArray: false,
-        },
-        {
-          category: 'crazy',
-          field: 'crazy.pants.matched.atomic',
-          values: ['matched_atomic'],
-          originalValue: ['matched_atomic'],
-          isObjectArray: false,
-        },
-        {
-          category: 'crazy',
-          field: 'crazy.pants.lazer.great.field',
-          values: ['grrrrr', 'grrrrr_2'],
-          originalValue: ['grrrrr', 'grrrrr_2'],
-          isObjectArray: false,
-        },
-        {
-          category: 'crazy',
-          field: 'crazy.pants.lazer.lazer.lazer.cool',
-          values: ['true', 'false'],
-          originalValue: ['true', 'false'],
-          isObjectArray: false,
-        },
-        {
-          category: 'crazy',
-          field: 'crazy.pants.lazer.lazer.lazer.lazer.lazer.lazer.lazer.whoa',
-          values: ['false'],
-          originalValue: ['false'],
-          isObjectArray: false,
+          field: 'crazy.pants',
+          values: [
+            '{"matched.field":["matched_field"],"first_seen":["2021-02-22T17:29:25.195Z"],"provider":["yourself"],"type":["custom"],"matched.atomic":["matched_atomic"],"lazer":[{"great.field":["grrrrr"],"lazer":[{"lazer":[{"cool":true,"lazer":[{"lazer":[{"lazer":[{"lazer":[{"whoa":false}]}]}]}]}]},{"lazer":[{"cool":false}]}]},{"great.field":["grrrrr_2"]}]}',
+          ],
+          originalValue: [
+            '{"matched.field":["matched_field"],"first_seen":["2021-02-22T17:29:25.195Z"],"provider":["yourself"],"type":["custom"],"matched.atomic":["matched_atomic"],"lazer":[{"great.field":["grrrrr"],"lazer":[{"lazer":[{"cool":true,"lazer":[{"lazer":[{"lazer":[{"lazer":[{"whoa":false}]}]}]}]}]},{"lazer":[{"cool":false}]}]},{"great.field":["grrrrr_2"]}]}',
+          ],
+          isObjectArray: true,
         },
       ];
       const result = getDataFromFieldsHits(whackFields);
       expect(result).toEqual(whackResultFields);
     });
   });
-  it('#getDataFromSourceHits', () => {
-    const _source: EventSource = {
-      '@timestamp': '2021-02-24T00:41:06.527Z',
-      'kibana.alert.workflow_status': 'open',
-      'signal.rule.name': 'Rawr',
-      'threat.indicator': [
-        {
-          provider: 'yourself',
-          type: 'custom',
-          first_seen: ['2021-02-22T17:29:25.195Z'],
-          matched: { atomic: 'atom', field: 'field', type: 'type' },
-        },
-        {
-          provider: 'other_you',
-          type: 'custom',
-          first_seen: '2021-02-22T17:29:25.195Z',
-          matched: { atomic: 'atom', field: 'field', type: 'type' },
-        },
-      ],
-    };
-    expect(getDataFromSourceHits(_source)).toEqual([
-      {
-        category: 'base',
-        field: '@timestamp',
-        values: ['2021-02-24T00:41:06.527Z'],
-        originalValue: ['2021-02-24T00:41:06.527Z'],
-        isObjectArray: false,
-      },
-      {
-        category: 'kibana',
-        field: 'kibana.alert.workflow_status',
-        values: ['open'],
-        originalValue: ['open'],
-        isObjectArray: false,
-      },
-      {
-        category: 'signal',
-        field: 'signal.rule.name',
-        values: ['Rawr'],
-        originalValue: ['Rawr'],
-        isObjectArray: false,
-      },
-      {
-        category: 'threat',
-        field: 'threat.indicator',
-        values: [
-          '{"provider":"yourself","type":"custom","first_seen":["2021-02-22T17:29:25.195Z"],"matched":{"atomic":"atom","field":"field","type":"type"}}',
-          '{"provider":"other_you","type":"custom","first_seen":"2021-02-22T17:29:25.195Z","matched":{"atomic":"atom","field":"field","type":"type"}}',
-        ],
-        originalValue: [
-          '{"provider":"yourself","type":"custom","first_seen":["2021-02-22T17:29:25.195Z"],"matched":{"atomic":"atom","field":"field","type":"type"}}',
-          '{"provider":"other_you","type":"custom","first_seen":"2021-02-22T17:29:25.195Z","matched":{"atomic":"atom","field":"field","type":"type"}}',
-        ],
-        isObjectArray: true,
-      },
-    ]);
-  });
   it('#getDataSafety', async () => {
     const result = await getDataSafety(getDataFromFieldsHits, fields);
     expect(result).toEqual(resultFields);

x-pack/plugins/security_solution/common/utils/field_formatters.ts

@@ -5,9 +5,12 @@
  * 2.0.
  */
 
-import { get, isEmpty, isNumber, isObject, isString } from 'lodash/fp';
+import { ecsFieldMap } from '@kbn/rule-registry-plugin/common/assets/field_maps/ecs_field_map';
+import { experimentalRuleFieldMap } from '@kbn/rule-registry-plugin/common/assets/field_maps/experimental_rule_field_map';
+import { technicalRuleFieldMap } from '@kbn/rule-registry-plugin/common/assets/field_maps/technical_rule_field_map';
+import { isEmpty } from 'lodash/fp';
 
-import type { EventHit, EventSource, TimelineEventsDetailsItem } from '../search_strategy';
+import type { EventHit, TimelineEventsDetailsItem } from '../search_strategy';
 import { toObjectArrayOfStrings, toStringArray } from './to_array';
 
 export const baseCategoryFields = ['@timestamp', 'labels', 'message', 'tags'];
@@ -38,40 +41,6 @@ export const formatGeoLocation = (item: unknown[]) => {
 export const isGeoField = (field: string) =>
   field.includes('geo.location') || field.includes('geoip.location');
 
-export const getDataFromSourceHits = (
-  sources: EventSource,
-  category?: string,
-  path?: string
-): TimelineEventsDetailsItem[] =>
-  Object.keys(sources).reduce<TimelineEventsDetailsItem[]>((accumulator, source) => {
-    const item: EventSource = get(source, sources);
-    if (Array.isArray(item) || isString(item) || isNumber(item)) {
-      const field = path ? `${path}.${source}` : source;
-      const fieldCategory = getFieldCategory(field);
-
-      const objArrStr = toObjectArrayOfStrings(item);
-      const strArr = objArrStr.map(({ str }) => str);
-      const isObjectArray = objArrStr.some((o) => o.isObjectArray);
-
-      return [
-        ...accumulator,
-        {
-          category: fieldCategory,
-          field,
-          values: strArr,
-          originalValue: strArr,
-          isObjectArray,
-        } as TimelineEventsDetailsItem,
-      ];
-    } else if (isObject(item)) {
-      return [
-        ...accumulator,
-        ...getDataFromSourceHits(item, category || source, path ? `${path}.${source}` : source),
-      ];
-    }
-    return accumulator;
-  }, []);
-
 export const getDataFromFieldsHits = (
   fields: EventHit['fields'],
   prependField?: string,
@@ -94,13 +63,19 @@ export const getDataFromFieldsHits = (
         },
       ];
     }
+
     const objArrStr = toObjectArrayOfStrings(item);
     const strArr = objArrStr.map(({ str }) => str);
     const isObjectArray = objArrStr.some((o) => o.isObjectArray);
     const dotField = prependField ? `${prependField}.${field}` : field;
 
-    // return simple field value (non-object, non-array)
-    if (!isObjectArray) {
+    // return simple field value (non-esc object, non-array)
+    if (
+      !isObjectArray ||
+      Object.keys({ ...ecsFieldMap, ...technicalRuleFieldMap, ...experimentalRuleFieldMap }).find(
+        (ecsField) => ecsField === field
+      ) === undefined
+    ) {
       return [
         ...accumulator,
         {

x-pack/plugins/security_solution/cypress/ccs_integration/detection_alerts/alerts_details.spec.ts

@@ -36,7 +36,7 @@ describe('Alert details with unmapped fields', () => {
 
     cy.get(JSON_TEXT).then((x) => {
       const parsed = JSON.parse(x.text());
-      expect(parsed._source.unmapped).to.equal(expectedUnmappedValue);
+      expect(parsed.fields.unmapped[0]).to.equal(expectedUnmappedValue);
     });
   });
 });

x-pack/plugins/security_solution/cypress/integration/detection_alerts/alerts_details.spec.ts

@@ -50,7 +50,7 @@ describe('Alert details with unmapped fields', () => {
 
     cy.get(JSON_TEXT).then((x) => {
       const parsed = JSON.parse(x.text());
-      expect(parsed._source.unmapped).to.equal(expectedUnmappedValue);
+      expect(parsed.fields.unmapped[0]).to.equal(expectedUnmappedValue);
     });
   });
 
@@ -61,7 +61,7 @@ describe('Alert details with unmapped fields', () => {
     };
 
     openTable();
-    cy.get(ALERT_FLYOUT).find(pageSelector(5)).click({ force: true });
+    cy.get(ALERT_FLYOUT).find(pageSelector(4)).click({ force: true });
     cy.get(ALERT_FLYOUT)
       .find(TABLE_ROWS)
       .within(() => {

x-pack/plugins/security_solution/cypress/integration/detection_alerts/cti_enrichments.spec.ts

@@ -67,32 +67,26 @@ describe('CTI Enrichment', () => {
   it('Displays persisted enrichments on the JSON view', () => {
     const expectedEnrichment = [
       {
-        feed: {
-          name: 'AbuseCH malware',
-        },
-        indicator: {
-          first_seen: '2021-03-10T08:02:14.000Z',
-          file: {
-            size: 80280,
-            pe: {},
-            type: 'elf',
-            hash: {
-              sha256: 'a04ac6d98ad989312783d4fe3456c53730b212c79a426fb215708b6c6daa3de3',
-              tlsh: '6D7312E017B517CC1371A8353BED205E9128223972AE35302E97528DF957703BAB2DBE',
-              ssdeep:
-                '1536:87vbq1lGAXSEYQjbChaAU2yU23M51DjZgSQAvcYkFtZTjzBht5:8D+CAXFYQChaAUk5ljnQssL',
-              md5: '9b6c3518a91d23ed77504b5416bfb5b3',
-            },
-          },
-          type: 'file',
-        },
-        matched: {
-          atomic: 'a04ac6d98ad989312783d4fe3456c53730b212c79a426fb215708b6c6daa3de3',
-          field: 'myhash.mysha256',
-          id: '84cf452c1e0375c3d4412cb550bd1783358468a3b3b777da4829d72c7d6fb74f',
-          index: 'logs-ti_abusech.malware',
-          type: 'indicator_match_rule',
-        },
+        'indicator.file.hash.md5': ['9b6c3518a91d23ed77504b5416bfb5b3'],
+        'matched.index': ['logs-ti_abusech.malware'],
+        'indicator.file.type': ['elf'],
+        'indicator.file.hash.tlsh': [
+          '6D7312E017B517CC1371A8353BED205E9128223972AE35302E97528DF957703BAB2DBE',
+        ],
+        'feed.name': ['AbuseCH malware'],
+        'indicator.file.hash.ssdeep': [
+          '1536:87vbq1lGAXSEYQjbChaAU2yU23M51DjZgSQAvcYkFtZTjzBht5:8D+CAXFYQChaAUk5ljnQssL',
+        ],
+        'indicator.file.hash.sha256': [
+          'a04ac6d98ad989312783d4fe3456c53730b212c79a426fb215708b6c6daa3de3',
+        ],
+        'indicator.first_seen': ['2021-03-10T08:02:14.000Z'],
+        'matched.field': ['myhash.mysha256'],
+        'indicator.type': ['file'],
+        'matched.type': ['indicator_match_rule'],
+        'matched.id': ['84cf452c1e0375c3d4412cb550bd1783358468a3b3b777da4829d72c7d6fb74f'],
+        'matched.atomic': ['a04ac6d98ad989312783d4fe3456c53730b212c79a426fb215708b6c6daa3de3'],
+        'indicator.file.size': [80280],
       },
     ];
 
@@ -101,7 +95,7 @@ describe('CTI Enrichment', () => {
 
     cy.get(JSON_TEXT).then((x) => {
       const parsed = JSON.parse(x.text());
-      expect(parsed._source.threat.enrichments).to.deep.equal(expectedEnrichment);
+      expect(parsed.fields['threat.enrichments']).to.deep.equal(expectedEnrichment);
     });
   });
 

x-pack/plugins/security_solution/public/cases/pages/index.tsx

@@ -27,13 +27,10 @@ import { DetailsPanel } from '../../timelines/components/side_panel';
 import { useFetchAlertData } from './use_fetch_alert_data';
 
 const TimelineDetailsPanel = () => {
-  const { browserFields, docValueFields, runtimeMappings } = useSourcererDataView(
-    SourcererScopeName.detections
-  );
+  const { browserFields, runtimeMappings } = useSourcererDataView(SourcererScopeName.detections);
   return (
     <DetailsPanel
       browserFields={browserFields}
-      docValueFields={docValueFields}
       entityType="events"
       isFlyoutView
       runtimeMappings={runtimeMappings}