Update CTI ECS 1.11 fields (elastic#113404) (elastic#115088)

* Update threatintel to threat * Remove CTI mappings * Update CTI_DATASET_KEY_MAP * Update default threat index * Change mapping to dataset * Fix tests * Fix tests * Fix test Co-authored-by: Kibana Machine <[email protected]> # Conflicts: # x-pack/plugins/security_solution/server/plugin.ts
kibanamachine · Oct 14, 2021 · c052859 · c052859
1 parent 8daa574
commit c052859
Show file tree

Hide file tree

Showing 19 changed files with 99 additions and 259 deletions.
diff --git a/x-pack/plugins/security_solution/common/constants.ts b/x-pack/plugins/security_solution/common/constants.ts
@@ -61,10 +61,10 @@ export const DEFAULT_SPACE_ID = 'default';
 
 // Document path where threat indicator fields are expected. Fields are used
 // to enrich signals, and are copied to threat.enrichments.
-export const DEFAULT_INDICATOR_SOURCE_PATH = 'threatintel.indicator';
+export const DEFAULT_INDICATOR_SOURCE_PATH = 'threat.indicator';
 export const ENRICHMENT_DESTINATION_PATH = 'threat.enrichments';
 export const DEFAULT_THREAT_INDEX_KEY = 'securitySolution:defaultThreatIndex';
-export const DEFAULT_THREAT_INDEX_VALUE = ['filebeat-*'];
+export const DEFAULT_THREAT_INDEX_VALUE = ['logs-ti_*'];
 export const DEFAULT_THREAT_MATCH_QUERY = '@timestamp >= "now-30d"';
 
 export enum SecurityPageName {

diff --git a/x-pack/plugins/security_solution/common/cti/constants.ts b/x-pack/plugins/security_solution/common/cti/constants.ts
@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import { ENRICHMENT_DESTINATION_PATH } from '../constants';
+import { ENRICHMENT_DESTINATION_PATH, DEFAULT_INDICATOR_SOURCE_PATH } from '../constants';
 
 export const MATCHED_ATOMIC = 'matched.atomic';
 export const MATCHED_FIELD = 'matched.field';
@@ -43,27 +43,27 @@ export enum ENRICHMENT_TYPES {
 }
 
 export const EVENT_ENRICHMENT_INDICATOR_FIELD_MAP = {
-  'file.hash.md5': 'threatintel.indicator.file.hash.md5',
-  'file.hash.sha1': 'threatintel.indicator.file.hash.sha1',
-  'file.hash.sha256': 'threatintel.indicator.file.hash.sha256',
-  'file.pe.imphash': 'threatintel.indicator.file.pe.imphash',
-  'file.elf.telfhash': 'threatintel.indicator.file.elf.telfhash',
-  'file.hash.ssdeep': 'threatintel.indicator.file.hash.ssdeep',
-  'source.ip': 'threatintel.indicator.ip',
-  'destination.ip': 'threatintel.indicator.ip',
-  'url.full': 'threatintel.indicator.url.full',
-  'registry.path': 'threatintel.indicator.registry.path',
+  'file.hash.md5': `${DEFAULT_INDICATOR_SOURCE_PATH}.file.hash.md5`,
+  'file.hash.sha1': `${DEFAULT_INDICATOR_SOURCE_PATH}.file.hash.sha1`,
+  'file.hash.sha256': `${DEFAULT_INDICATOR_SOURCE_PATH}.file.hash.sha256`,
+  'file.pe.imphash': `${DEFAULT_INDICATOR_SOURCE_PATH}.file.pe.imphash`,
+  'file.elf.telfhash': `${DEFAULT_INDICATOR_SOURCE_PATH}.file.elf.telfhash`,
+  'file.hash.ssdeep': `${DEFAULT_INDICATOR_SOURCE_PATH}.file.hash.ssdeep`,
+  'source.ip': `${DEFAULT_INDICATOR_SOURCE_PATH}.ip`,
+  'destination.ip': `${DEFAULT_INDICATOR_SOURCE_PATH}.ip`,
+  'url.full': `${DEFAULT_INDICATOR_SOURCE_PATH}.url.full`,
+  'registry.path': `${DEFAULT_INDICATOR_SOURCE_PATH}.registry.path`,
 };
 
 export const DEFAULT_EVENT_ENRICHMENT_FROM = 'now-30d';
 export const DEFAULT_EVENT_ENRICHMENT_TO = 'now';
 
 export const CTI_DATASET_KEY_MAP: { [key: string]: string } = {
-  'Abuse URL': 'threatintel.abuseurl',
-  'Abuse Malware': 'threatintel.abusemalware',
-  'AlienVault OTX': 'threatintel.otx',
-  Anomali: 'threatintel.anomali',
-  'Malware Bazaar': 'threatintel.malwarebazaar',
-  MISP: 'threatintel.misp',
-  'Recorded Future': 'threatintel.recordedfuture',
+  'Abuse URL': 'ti_abusech.url',
+  'Abuse Malware': 'ti_abusech.malware',
+  'Malware Bazaar': 'ti_abusech.malwarebazaar',
+  'AlienVault OTX': 'ti_otx.threat',
+  'Anomali Limo': 'ti_anomali.limo',
+  'Anomali ThreatStream': 'ti_anomali.threatstream',
+  MISP: 'ti_misp.threat',
 };
diff --git a/x-pack/plugins/security_solution/common/search_strategy/security_solution/cti/index.mock.ts b/x-pack/plugins/security_solution/common/search_strategy/security_solution/cti/index.mock.ts
@@ -52,29 +52,29 @@ export const buildEventEnrichmentRawResponseMock = (): IEsSearchResponse => ({
           _score: 6.0637846,
           fields: {
             'event.category': ['threat'],
-            'threatintel.indicator.file.type': ['html'],
+            'threat.indicator.file.type': ['html'],
             'related.hash': [
               '5529de7b60601aeb36f57824ed0e1ae8',
               '15b012e6f626d0f88c2926d2bf4ca394d7b8ee07cc06d2ec05ea76bed3e8a05e',
               '768:NXSFGJ/ooP6FawrB7Bo1MWnF/jRmhJImp:1SFXIqBo1Mwj2p',
             ],
-            'threatintel.indicator.first_seen': ['2021-05-28T18:33:29.000Z'],
-            'threatintel.indicator.file.hash.tlsh': [
+            'threat.indicator.first_seen': ['2021-05-28T18:33:29.000Z'],
+            'threat.indicator.file.hash.tlsh': [
               'FFB20B82F6617061C32784E2712F7A46B179B04FD1EA54A0F28CD8E9CFE4CAA1617F1C',
             ],
             'service.type': ['threatintel'],
-            'threatintel.indicator.file.hash.ssdeep': [
+            'threat.indicator.file.hash.ssdeep': [
               '768:NXSFGJ/ooP6FawrB7Bo1MWnF/jRmhJImp:1SFXIqBo1Mwj2p',
             ],
             'agent.type': ['filebeat'],
             'event.module': ['threatintel'],
-            'threatintel.indicator.type': ['file'],
+            'threat.indicator.type': ['file'],
             'agent.name': ['rylastic.local'],
-            'threatintel.indicator.file.hash.sha256': [
+            'threat.indicator.file.hash.sha256': [
               '15b012e6f626d0f88c2926d2bf4ca394d7b8ee07cc06d2ec05ea76bed3e8a05e',
             ],
             'event.kind': ['enrichment'],
-            'threatintel.indicator.file.hash.md5': ['5529de7b60601aeb36f57824ed0e1ae8'],
+            'threat.indicator.file.hash.md5': ['5529de7b60601aeb36f57824ed0e1ae8'],
             'fileset.name': ['abusemalware'],
             'input.type': ['httpjson'],
             'agent.hostname': ['rylastic.local'],
@@ -89,9 +89,9 @@ export const buildEventEnrichmentRawResponseMock = (): IEsSearchResponse => ({
             'event.type': ['indicator'],
             'event.created': ['2021-05-28T18:33:52.993Z'],
             'agent.ephemeral_id': ['d6b14f65-5bf3-430d-8315-7b5613685979'],
-            'threatintel.indicator.file.size': [24738],
+            'threat.indicator.file.size': [24738],
             'agent.version': ['8.0.0'],
-            'event.dataset': ['threatintel.abusemalware'],
+            'event.dataset': ['ti_abusech.malware'],
           },
           matched_queries: ['file.hash.md5'],
         },
@@ -113,7 +113,7 @@ export const buildEventEnrichmentMock = (
   'ecs.version': ['1.6.0'],
   'event.category': ['threat'],
   'event.created': ['2021-05-28T18:33:52.993Z'],
-  'event.dataset': ['threatintel.abusemalware'],
+  'event.dataset': ['ti_abusech.malware'],
   'event.ingested': ['2021-05-28T18:33:55.086Z'],
   'event.kind': ['enrichment'],
   'event.module': ['threatintel'],
@@ -135,20 +135,18 @@ export const buildEventEnrichmentMock = (
   ],
   'service.type': ['threatintel'],
   tags: ['threatintel-abusemalware', 'forwarded'],
-  'threatintel.indicator.file.hash.md5': ['5529de7b60601aeb36f57824ed0e1ae8'],
-  'threatintel.indicator.file.hash.sha256': [
+  'threat.indicator.file.hash.md5': ['5529de7b60601aeb36f57824ed0e1ae8'],
+  'threat.indicator.file.hash.sha256': [
     '15b012e6f626d0f88c2926d2bf4ca394d7b8ee07cc06d2ec05ea76bed3e8a05e',
   ],
-  'threatintel.indicator.file.hash.ssdeep': [
-    '768:NXSFGJ/ooP6FawrB7Bo1MWnF/jRmhJImp:1SFXIqBo1Mwj2p',
-  ],
-  'threatintel.indicator.file.hash.tlsh': [
+  'threat.indicator.file.hash.ssdeep': ['768:NXSFGJ/ooP6FawrB7Bo1MWnF/jRmhJImp:1SFXIqBo1Mwj2p'],
+  'threat.indicator.file.hash.tlsh': [
     'FFB20B82F6617061C32784E2712F7A46B179B04FD1EA54A0F28CD8E9CFE4CAA1617F1C',
   ],
-  'threatintel.indicator.file.size': [24738],
-  'threatintel.indicator.file.type': ['html'],
-  'threatintel.indicator.first_seen': ['2021-05-28T18:33:29.000Z'],
-  'threatintel.indicator.type': ['file'],
+  'threat.indicator.file.size': [24738],
+  'threat.indicator.file.type': ['html'],
+  'threat.indicator.first_seen': ['2021-05-28T18:33:29.000Z'],
+  'threat.indicator.type': ['file'],
   ...overrides,
 });
 

diff --git a/...ck/plugins/security_solution/cypress/integration/detection_alerts/cti_enrichments.spec.ts b/...ck/plugins/security_solution/cypress/integration/detection_alerts/cti_enrichments.spec.ts
@@ -79,7 +79,7 @@ describe('CTI Enrichment', () => {
       { line: 4, text: '  "threat": {' },
       {
         line: 3,
-        text: '    "enrichments": "{\\"indicator\\":{\\"first_seen\\":\\"2021-03-10T08:02:14.000Z\\",\\"file\\":{\\"size\\":80280,\\"pe\\":{},\\"type\\":\\"elf\\",\\"hash\\":{\\"sha256\\":\\"a04ac6d98ad989312783d4fe3456c53730b212c79a426fb215708b6c6daa3de3\\",\\"tlsh\\":\\"6D7312E017B517CC1371A8353BED205E9128223972AE35302E97528DF957703BAB2DBE\\",\\"ssdeep\\":\\"1536:87vbq1lGAXSEYQjbChaAU2yU23M51DjZgSQAvcYkFtZTjzBht5:8D+CAXFYQChaAUk5ljnQssL\\",\\"md5\\":\\"9b6c3518a91d23ed77504b5416bfb5b3\\"}},\\"type\\":\\"file\\"},\\"matched\\":{\\"atomic\\":\\"a04ac6d98ad989312783d4fe3456c53730b212c79a426fb215708b6c6daa3de3\\",\\"field\\":\\"myhash.mysha256\\",\\"id\\":\\"84cf452c1e0375c3d4412cb550bd1783358468a3b3b777da4829d72c7d6fb74f\\",\\"index\\":\\"filebeat-7.12.0-2021.03.10-000001\\",\\"type\\":\\"indicator_match_rule\\"}}"',
+        text: '    "enrichments": "{\\"indicator\\":{\\"first_seen\\":\\"2021-03-10T08:02:14.000Z\\",\\"file\\":{\\"size\\":80280,\\"pe\\":{},\\"type\\":\\"elf\\",\\"hash\\":{\\"sha256\\":\\"a04ac6d98ad989312783d4fe3456c53730b212c79a426fb215708b6c6daa3de3\\",\\"tlsh\\":\\"6D7312E017B517CC1371A8353BED205E9128223972AE35302E97528DF957703BAB2DBE\\",\\"ssdeep\\":\\"1536:87vbq1lGAXSEYQjbChaAU2yU23M51DjZgSQAvcYkFtZTjzBht5:8D+CAXFYQChaAUk5ljnQssL\\",\\"md5\\":\\"9b6c3518a91d23ed77504b5416bfb5b3\\"}},\\"type\\":\\"file\\"},\\"matched\\":{\\"atomic\\":\\"a04ac6d98ad989312783d4fe3456c53730b212c79a426fb215708b6c6daa3de3\\",\\"field\\":\\"myhash.mysha256\\",\\"id\\":\\"84cf452c1e0375c3d4412cb550bd1783358468a3b3b777da4829d72c7d6fb74f\\",\\"index\\":\\"logs-ti_abusech.malware\\",\\"type\\":\\"indicator_match_rule\\"}}"',
       },
       { line: 2, text: '  }' },
     ];
@@ -127,7 +127,7 @@ describe('CTI Enrichment', () => {
         field: 'matched.id',
         value: '84cf452c1e0375c3d4412cb550bd1783358468a3b3b777da4829d72c7d6fb74f',
       },
-      { field: 'matched.index', value: 'filebeat-7.12.0-2021.03.10-000001' },
+      { field: 'matched.index', value: 'logs-ti_abusech.malware' },
       { field: 'matched.type', value: 'indicator_match_rule' },
     ];
 

diff --git a/x-pack/plugins/security_solution/cypress/objects/rule.ts b/x-pack/plugins/security_solution/cypress/objects/rule.ts
@@ -108,7 +108,7 @@ export const getIndexPatterns = (): string[] => [
   'winlogbeat-*',
 ];
 
-export const getThreatIndexPatterns = (): string[] => ['filebeat-*'];
+export const getThreatIndexPatterns = (): string[] => ['logs-ti_*'];
 
 const getMitre1 = (): Mitre => ({
   tactic: `${getMockThreatData().tactic.name} (${getMockThreatData().tactic.id})`,
@@ -380,7 +380,7 @@ export const getNewThreatIndicatorRule = (): ThreatIndicatorRule => ({
   lookBack: getLookBack(),
   indicatorIndexPattern: ['filebeat-*'],
   indicatorMappingField: 'myhash.mysha256',
-  indicatorIndexField: 'threatintel.indicator.file.hash.sha256',
+  indicatorIndexField: 'threat.indicator.file.hash.sha256',
   type: 'file',
   atomic: 'a04ac6d98ad989312783d4fe3456c53730b212c79a426fb215708b6c6daa3de3',
   timeline: getIndicatorMatchTimelineTemplate(),

diff --git a/..._solution/public/common/components/event_details/cti_details/threat_details_view.test.tsx b/..._solution/public/common/components/event_details/cti_details/threat_details_view.test.tsx
@@ -37,7 +37,7 @@ describe('ThreatDetailsView', () => {
   it('renders an anchor link for indicator.reference', () => {
     const enrichments = [
       buildEventEnrichmentMock({
-        'threatintel.indicator.reference': ['http://foo.baz'],
+        'threat.indicator.reference': ['http://foo.baz'],
       }),
     ];
     const wrapper = mount(
@@ -60,10 +60,10 @@ describe('ThreatDetailsView', () => {
     const existingEnrichment = buildEventEnrichmentMock({
       'indicator.first_seen': [mostRecentDate],
     });
-    delete existingEnrichment['threatintel.indicator.first_seen'];
+    delete existingEnrichment['threat.indicator.first_seen'];
     const newEnrichment = buildEventEnrichmentMock({
       'matched.id': ['other.id'],
-      'threatintel.indicator.first_seen': [olderDate],
+      'threat.indicator.first_seen': [olderDate],
     });
     const enrichments = [existingEnrichment, newEnrichment];
 

diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/field_maps/cti.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/field_maps/cti.ts
diff --git a/...lib/detection_engine/rule_types/indicator_match/create_indicator_match_alert_type.test.ts b/...lib/detection_engine/rule_types/indicator_match/create_indicator_match_alert_type.test.ts
@@ -42,7 +42,7 @@ describe('Indicator Match Alerts', () => {
           {
             field: 'file.hash.md5',
             type: 'mapping',
-            value: 'threatintel.indicator.file.hash.md5',
+            value: 'threat.indicator.file.hash.md5',
           },
         ],
       },
@@ -156,11 +156,11 @@ describe('Indicator Match Alerts', () => {
               ...sampleDocNoSortId(v4()),
               _source: {
                 ...sampleDocNoSortId(v4())._source,
-                'threatintel.indicator.file.hash.md5': 'a1b2c3',
+                'threat.indicator.file.hash.md5': 'a1b2c3',
               },
               fields: {
                 ...sampleDocNoSortId(v4()).fields,
-                'threatintel.indicator.file.hash.md5': ['a1b2c3'],
+                'threat.indicator.file.hash.md5': ['a1b2c3'],
               },
             },
           ],

diff --git a/...ty_solution/server/lib/detection_engine/rule_types/scripts/create_rule_indicator_match.sh b/...ty_solution/server/lib/detection_engine/rule_types/scripts/create_rule_indicator_match.sh
@@ -46,7 +46,7 @@ curl -X POST ${KIBANA_URL}${SPACE_URL}/api/alerts/alert \
             {
               "field":"file.hash.md5",
               "type":"mapping",
-              "value":"threatintel.indicator.file.hash.md5"
+              "value":"threat.indicator.file.hash.md5"
             }
           ]
         }