diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/ecs_mapping.json b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/ecs_mapping.json index 6126ee462ec20..70b62d569b9d3 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/ecs_mapping.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/ecs_mapping.json @@ -2457,6 +2457,144 @@ "ignore_above": 1024, "type": "keyword" }, + "indicator": { + "type": "nested", + "properties": { + "as": { + "properties": { + "number": { + "type": "long" + }, + "organization": { + "properties": { + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "confidence": { + "ignore_above": 1024, + "type": "keyword" + }, + "dataset": { + "ignore_above": 1024, + "type": "keyword" + }, + "description": { + "type": "wildcard" + }, + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "email": { + "properties": { + "address": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "first_seen": { + "type": "date" + }, + "geo": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "ip": { + "type": "ip" + }, + "last_seen": { + "type": "date" + }, + "marking": { + "properties": { + "tlp": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "matched": { + "properties": { + "atomic": { + "ignore_above": 1024, + "type": "keyword" + }, + "field": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "module": { + "ignore_above": 1024, + "type": "keyword" + }, + "port": { + "type": "long" + }, + "provider": { + "ignore_above": 1024, + "type": "keyword" + }, + "scanner_stats": { + "type": "long" + }, + "sightings": { + "type": "long" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, "tactic": { "properties": { "id": { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/get_signals_template.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/get_signals_template.ts index c9a4f168224d4..48036ec73511b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/get_signals_template.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/get_signals_template.ts @@ -8,7 +8,20 @@ import signalsMapping from './signals_mapping.json'; import ecsMapping from './ecs_mapping.json'; -export const SIGNALS_TEMPLATE_VERSION = 14; +/** + @constant + @type {number} + @description This value represents the template version assumed by app code. + If this number is greater than the user's signals index version, the + detections UI will attempt to update the signals template and roll over to + a new signals index. + + If making mappings changes in a patch release, this number should be incremented by 1. + If making mappings changes in a minor release, this number should be + incremented by 10 in order to add "room" for the aforementioned patch + release +*/ +export const SIGNALS_TEMPLATE_VERSION = 24; export const MIN_EQL_RULE_INDEX_VERSION = 2; export const getSignalsTemplate = (index: string) => { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts index b506a2463a311..55d128225c555 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts @@ -14,6 +14,7 @@ import { repeatedSearchResultsWithSortId, repeatedSearchResultsWithNoSortId, sampleDocSearchResultsNoSortIdNoHits, + sampleDocWithSortId, } from './__mocks__/es_results'; import { searchAfterAndBulkCreate } from './search_after_bulk_create'; import { buildRuleMessageFactory } from './rule_messages'; @@ -870,4 +871,93 @@ describe('searchAfterAndBulkCreate', () => { expect(createdSignalsCount).toEqual(4); expect(lastLookBackDate).toEqual(new Date('2020-04-20T21:27:45+0000')); }); + + it('invokes the enrichment callback with signal search results', async () => { + const sampleParams = sampleRuleAlertParams(30); + mockService.callCluster + .mockResolvedValueOnce(repeatedSearchResultsWithSortId(4, 1, someGuids.slice(0, 3))) + .mockResolvedValueOnce({ + took: 100, + errors: false, + items: [ + { + create: { + status: 201, + }, + }, + ], + }) + .mockResolvedValueOnce(repeatedSearchResultsWithSortId(4, 1, someGuids.slice(3, 6))) + .mockResolvedValueOnce({ + took: 100, + errors: false, + items: [ + { + create: { + status: 201, + }, + }, + ], + }) + .mockResolvedValueOnce(repeatedSearchResultsWithSortId(4, 1, someGuids.slice(6, 9))) + .mockResolvedValueOnce({ + took: 100, + errors: false, + items: [ + { + create: { + status: 201, + }, + }, + ], + }) + .mockResolvedValueOnce(sampleDocSearchResultsNoSortIdNoHits()); + + const mockEnrichment = jest.fn((a) => a); + const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({ + enrichment: mockEnrichment, + ruleParams: sampleParams, + gap: moment.duration(2, 'm'), + previousStartedAt: moment().subtract(10, 'm').toDate(), + listClient, + exceptionsList: [], + services: mockService, + logger: mockLogger, + eventsTelemetry: undefined, + id: sampleRuleGuid, + inputIndexPattern, + signalsIndex: DEFAULT_SIGNALS_INDEX, + name: 'rule-name', + actions: [], + createdAt: '2020-01-28T15:58:34.810Z', + updatedAt: '2020-01-28T15:59:14.004Z', + createdBy: 'elastic', + updatedBy: 'elastic', + interval: '5m', + enabled: true, + pageSize: 1, + filter: undefined, + refresh: false, + tags: ['some fake tag 1', 'some fake tag 2'], + throttle: 'no_actions', + buildRuleMessage, + }); + + expect(mockEnrichment).toHaveBeenCalledWith( + expect.objectContaining({ + hits: expect.objectContaining({ + hits: expect.arrayContaining([ + expect.objectContaining({ + ...sampleDocWithSortId(), + _id: expect.any(String), + }), + ]), + }), + }) + ); + expect(success).toEqual(true); + expect(mockService.callCluster).toHaveBeenCalledTimes(7); + expect(createdSignalsCount).toEqual(3); + expect(lastLookBackDate).toEqual(new Date('2020-04-20T21:27:45+0000')); + }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts index b821909ca907c..061aa4bba5a41 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts @@ -7,6 +7,7 @@ /* eslint-disable complexity */ +import { identity } from 'lodash'; import { singleSearchAfter } from './single_search_after'; import { singleBulkCreate } from './single_bulk_create'; import { filterEventsAgainstList } from './filters/filter_events_against_list'; @@ -49,6 +50,7 @@ export const searchAfterAndBulkCreate = async ({ tags, throttle, buildRuleMessage, + enrichment = identity, }: SearchAfterAndBulkCreateParams): Promise => { let toReturn = createSearchAfterReturnType(); @@ -106,7 +108,7 @@ export const searchAfterAndBulkCreate = async ({ services, logger, filter, - pageSize: tuple.maxSignals < pageSize ? Math.ceil(tuple.maxSignals) : pageSize, // maximum number of docs to receive per search result. + pageSize: Math.ceil(Math.min(tuple.maxSignals, pageSize)), timestampOverride: ruleParams.timestampOverride, excludeDocsWithTimestampOverride: true, }); @@ -117,14 +119,12 @@ export const searchAfterAndBulkCreate = async ({ backupSortId = lastSortId[0]; hasBackupSortId = true; } else { - // if no sort id on backup search and the initial search result was also empty logger.debug(buildRuleMessage('backupSortIds was empty on searchResultB')); hasBackupSortId = false; } mergedSearchResults = mergeSearchResults([mergedSearchResults, searchResultB]); - // merge the search result from the secondary search with the first toReturn = mergeReturns([ toReturn, createSearchAfterReturnTypeFromResponse({ @@ -139,7 +139,6 @@ export const searchAfterAndBulkCreate = async ({ } if (hasSortId) { - // only execute search if we have something to sort on or if it is the first search const { searchResult, searchDuration, searchErrors } = await singleSearchAfter({ buildRuleMessage, searchAfterSortId: sortId, @@ -149,7 +148,7 @@ export const searchAfterAndBulkCreate = async ({ services, logger, filter, - pageSize: tuple.maxSignals < pageSize ? Math.ceil(tuple.maxSignals) : pageSize, // maximum number of docs to receive per search result. + pageSize: Math.ceil(Math.min(tuple.maxSignals, pageSize)), timestampOverride: ruleParams.timestampOverride, excludeDocsWithTimestampOverride: false, }); @@ -166,10 +165,6 @@ export const searchAfterAndBulkCreate = async ({ }), ]); - // we are guaranteed to have searchResult hits at this point - // because we check before if the totalHits or - // searchResult.hits.hits.length is 0 - // call this function setSortIdOrExit() const lastSortId = searchResult.hits.hits[searchResult.hits.hits.length - 1]?.sort; if (lastSortId != null && lastSortId.length !== 0) { sortId = lastSortId[0]; @@ -186,14 +181,6 @@ export const searchAfterAndBulkCreate = async ({ buildRuleMessage(`searchResult.hit.hits.length: ${mergedSearchResults.hits.hits.length}`) ); - // search results yielded zero hits so exit - // with search_after, these two values can be different when - // searching with the last sortId of a consecutive search_after - // yields zero hits, but there were hits using the previous - // sortIds. - // e.g. totalHits was 156, index 50 of 100 results, do another search-after - // this time with a new sortId, index 22 of the remaining 56, get another sortId - // search with that sortId, total is still 156 but the hits.hits array is empty. if (totalHits === 0 || mergedSearchResults.hits.hits.length === 0) { logger.debug( buildRuleMessage( @@ -228,6 +215,8 @@ export const searchAfterAndBulkCreate = async ({ tuple.maxSignals - signalsCreatedCount ); } + const enrichedEvents = await enrichment(filteredEvents); + const { bulkCreateDuration: bulkDuration, createdItemsCount: createdCount, @@ -236,7 +225,7 @@ export const searchAfterAndBulkCreate = async ({ errors: bulkErrors, } = await singleBulkCreate({ buildRuleMessage, - filteredEvents, + filteredEvents: enrichedEvents, ruleParams, services, logger, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_enrichment.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_enrichment.ts new file mode 100644 index 0000000000000..b14d148218938 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_enrichment.ts @@ -0,0 +1,55 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { SignalSearchResponse, SignalsEnrichment } from '../types'; +import { enrichSignalThreatMatches } from './enrich_signal_threat_matches'; +import { getThreatList } from './get_threat_list'; +import { BuildThreatEnrichmentOptions, GetMatchedThreats } from './types'; + +export const buildThreatEnrichment = ({ + buildRuleMessage, + exceptionItems, + listClient, + logger, + services, + threatFilters, + threatIndex, + threatLanguage, + threatQuery, +}: BuildThreatEnrichmentOptions): SignalsEnrichment => { + const getMatchedThreats: GetMatchedThreats = async (ids) => { + const matchedThreatsFilter = { + query: { + bool: { + filter: { + ids: { values: ids }, + }, + }, + }, + }; + const threatResponse = await getThreatList({ + callCluster: services.callCluster, + exceptionItems, + threatFilters: [...threatFilters, matchedThreatsFilter], + query: threatQuery, + language: threatLanguage, + index: threatIndex, + listClient, + searchAfter: undefined, + sortField: undefined, + sortOrder: undefined, + logger, + buildRuleMessage, + perPage: undefined, + }); + + return threatResponse.hits.hits; + }; + + return (signals: SignalSearchResponse): Promise => + enrichSignalThreatMatches(signals, getMatchedThreats); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.mock.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.mock.ts index 12865e4dd47a9..266903f568792 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.mock.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.mock.ts @@ -9,7 +9,7 @@ import { ThreatMapping } from '../../../../../common/detection_engine/schemas/ty import { Filter } from 'src/plugins/data/common'; import { SearchResponse } from 'elasticsearch'; -import { ThreatListItem } from './types'; +import { ThreatListDoc, ThreatListItem } from './types'; export const getThreatMappingMock = (): ThreatMapping => { return [ @@ -62,7 +62,7 @@ export const getThreatMappingMock = (): ThreatMapping => { ]; }; -export const getThreatListSearchResponseMock = (): SearchResponse => ({ +export const getThreatListSearchResponseMock = (): SearchResponse => ({ took: 0, timed_out: false, _shards: { @@ -74,33 +74,32 @@ export const getThreatListSearchResponseMock = (): SearchResponse ({ - '@timestamp': '2020-09-09T21:59:13Z', - host: { - name: 'host-1', - ip: '192.168.0.0.1', - }, - source: { - ip: '127.0.0.1', - port: 1, - }, - destination: { - ip: '127.0.0.1', - port: 1, +export const getThreatListItemMock = (overrides: Partial = {}): ThreatListItem => ({ + _id: '123', + _index: 'threat_index', + _type: '_doc', + _score: 0, + _source: { + '@timestamp': '2020-09-09T21:59:13Z', + host: { + name: 'host-1', + ip: '192.168.0.0.1', + }, + source: { + ip: '127.0.0.1', + port: 1, + }, + destination: { + ip: '127.0.0.1', + port: 1, + }, }, + fields: getThreatListItemFieldsMock(), + ...overrides, }); export const getThreatListItemFieldsMock = () => ({ @@ -188,13 +187,17 @@ export const getThreatMappingFilterShouldMock = (port = 1) => ({ filter: [ { bool: { - should: [{ match: { 'host.name': 'host-1' } }], + should: [ + { match: { 'host.name': { query: 'host-1', _name: expect.any(String) } } }, + ], minimum_should_match: 1, }, }, { bool: { - should: [{ match: { 'host.ip': '192.168.0.0.1' } }], + should: [ + { match: { 'host.ip': { query: '192.168.0.0.1', _name: expect.any(String) } } }, + ], minimum_should_match: 1, }, }, @@ -206,13 +209,19 @@ export const getThreatMappingFilterShouldMock = (port = 1) => ({ filter: [ { bool: { - should: [{ match: { 'destination.ip': '127.0.0.1' } }], + should: [ + { + match: { 'destination.ip': { query: '127.0.0.1', _name: expect.any(String) } }, + }, + ], minimum_should_match: 1, }, }, { bool: { - should: [{ match: { 'destination.port': port } }], + should: [ + { match: { 'destination.port': { query: port, _name: expect.any(String) } } }, + ], minimum_should_match: 1, }, }, @@ -224,7 +233,7 @@ export const getThreatMappingFilterShouldMock = (port = 1) => ({ filter: [ { bool: { - should: [{ match: { 'source.port': port } }], + should: [{ match: { 'source.port': { query: port, _name: expect.any(String) } } }], minimum_should_match: 1, }, }, @@ -236,7 +245,9 @@ export const getThreatMappingFilterShouldMock = (port = 1) => ({ filter: [ { bool: { - should: [{ match: { 'source.ip': '127.0.0.1' } }], + should: [ + { match: { 'source.ip': { query: '127.0.0.1', _name: expect.any(String) } } }, + ], minimum_should_match: 1, }, }, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.test.ts index 7a9c4b43b8f7a..1c0300ee0cc74 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.test.ts @@ -132,7 +132,7 @@ describe('build_threat_mapping_filter', () => { ], }, ], - threatListItem: { + threatListItem: getThreatListItemMock({ _source: { '@timestamp': '2020-09-09T21:59:13Z', host: { @@ -144,7 +144,7 @@ describe('build_threat_mapping_filter', () => { '@timestamp': ['2020-09-09T21:59:13Z'], 'host.name': ['host-1'], }, - }, + }), }); expect(item).toEqual([]); }); @@ -176,7 +176,7 @@ describe('build_threat_mapping_filter', () => { ], }, ], - threatListItem: { + threatListItem: getThreatListItemMock({ _source: { '@timestamp': '2020-09-09T21:59:13Z', host: { @@ -187,7 +187,7 @@ describe('build_threat_mapping_filter', () => { '@timestamp': ['2020-09-09T21:59:13Z'], 'host.name': ['host-1'], }, - }, + }), }); expect(item).toEqual([ { @@ -325,7 +325,10 @@ describe('build_threat_mapping_filter', () => { test('it should return an empty boolean clause given an empty object for a threat list item', () => { const threatMapping = getThreatMappingMock(); - const innerClause = createAndOrClauses({ threatMapping, threatListItem: { _source: {} } }); + const innerClause = createAndOrClauses({ + threatMapping, + threatListItem: getThreatListItemMock({ _source: {}, fields: {} }), + }); expect(innerClause).toEqual({ bool: { minimum_should_match: 1, should: [] } }); }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.ts index cab01a602b8a9..0a2789ec2f1d0 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.ts @@ -17,6 +17,7 @@ import { FilterThreatMappingOptions, SplitShouldClausesOptions, } from './types'; +import { encodeThreatMatchNamedQuery } from './utils'; export const MAX_CHUNK_SIZE = 1024; @@ -79,7 +80,14 @@ export const createInnerAndClauses = ({ should: [ { match: { - [threatMappingEntry.field]: value[0], + [threatMappingEntry.field]: { + query: value[0], + _name: encodeThreatMatchNamedQuery({ + id: threatListItem._id, + field: threatMappingEntry.field, + value: threatMappingEntry.value, + }), + }, }, }, ], diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signal.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signal.ts index a076ab46aae2a..ba428bc077125 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signal.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signal.ts @@ -14,6 +14,7 @@ import { SearchAfterAndBulkCreateReturnType } from '../types'; export const createThreatSignal = async ({ threatMapping, + threatEnrichment, query, inputIndex, type, @@ -77,6 +78,7 @@ export const createThreatSignal = async ({ `${threatFilter.query.bool.should.length} indicator items are being checked for existence of matches` ) ); + const result = await searchAfterAndBulkCreate({ gap, previousStartedAt, @@ -103,6 +105,7 @@ export const createThreatSignal = async ({ tags, throttle, buildRuleMessage, + enrichment: threatEnrichment, }); logger.debug( buildRuleMessage( diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts index 1e486e58aa073..7690eb5eb1d55 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts @@ -12,6 +12,7 @@ import { CreateThreatSignalsOptions } from './types'; import { createThreatSignal } from './create_threat_signal'; import { SearchAfterAndBulkCreateReturnType } from '../types'; import { combineConcurrentResults } from './utils'; +import { buildThreatEnrichment } from './build_threat_enrichment'; export const createThreatSignals = async ({ threatMapping, @@ -90,12 +91,25 @@ export const createThreatSignals = async ({ perPage, }); + const threatEnrichment = buildThreatEnrichment({ + buildRuleMessage, + exceptionItems, + listClient, + logger, + services, + threatFilters, + threatIndex, + threatLanguage, + threatQuery, + }); + while (threatList.hits.hits.length !== 0) { const chunks = chunk(itemsPerSearch, threatList.hits.hits); logger.debug(buildRuleMessage(`${chunks.length} concurrent indicator searches are starting.`)); const concurrentSearchesPerformed = chunks.map>( (slicedChunk) => createThreatSignal({ + threatEnrichment, threatMapping, query, inputIndex, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.mock.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.mock.ts new file mode 100644 index 0000000000000..a3ff932e97886 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.mock.ts @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { SignalSearchResponse, SignalSourceHit } from '../types'; +import { ThreatMatchNamedQuery } from './types'; + +export const getNamedQueryMock = ( + overrides: Partial = {} +): ThreatMatchNamedQuery => ({ + id: 'id', + field: 'field', + value: 'value', + ...overrides, +}); + +export const getSignalHitMock = (overrides: Partial = {}): SignalSourceHit => ({ + _id: '_id', + _index: '_index', + _source: { + '@timestamp': '2020-11-20T15:35:28.373Z', + }, + _type: '_type', + _score: 0, + ...overrides, +}); + +export const getSignalsResponseMock = (signals: SignalSourceHit[] = []): SignalSearchResponse => ({ + took: 1, + timed_out: false, + _shards: { total: 1, successful: 1, skipped: 0, failed: 0 }, + hits: { total: { value: signals.length, relation: 'eq' }, max_score: 0, hits: signals }, +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.test.ts new file mode 100644 index 0000000000000..3c0765b56ae20 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.test.ts @@ -0,0 +1,484 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { get } from 'lodash'; + +import { getThreatListItemMock } from './build_threat_mapping_filter.mock'; +import { + buildMatchedIndicator, + enrichSignalThreatMatches, + groupAndMergeSignalMatches, +} from './enrich_signal_threat_matches'; +import { + getNamedQueryMock, + getSignalHitMock, + getSignalsResponseMock, +} from './enrich_signal_threat_matches.mock'; +import { GetMatchedThreats, ThreatListItem, ThreatMatchNamedQuery } from './types'; +import { encodeThreatMatchNamedQuery } from './utils'; + +describe('groupAndMergeSignalMatches', () => { + it('returns an empty array if there are no signals', () => { + expect(groupAndMergeSignalMatches([])).toEqual([]); + }); + + it('returns the same list if there are no duplicates', () => { + const signals = [getSignalHitMock({ _id: '1' }), getSignalHitMock({ _id: '2' })]; + const expectedSignals = [...signals]; + expect(groupAndMergeSignalMatches(signals)).toEqual(expectedSignals); + }); + + it('deduplicates signals with the same ID', () => { + const signals = [getSignalHitMock({ _id: '1' }), getSignalHitMock({ _id: '1' })]; + const expectedSignals = [signals[0]]; + expect(groupAndMergeSignalMatches(signals)).toEqual(expectedSignals); + }); + + it('merges the matched_queries of duplicate signals', () => { + const signals = [ + getSignalHitMock({ _id: '1', matched_queries: ['query1'] }), + getSignalHitMock({ _id: '1', matched_queries: ['query3', 'query4'] }), + ]; + const [mergedSignal] = groupAndMergeSignalMatches(signals); + expect(mergedSignal.matched_queries).toEqual(['query1', 'query3', 'query4']); + }); + + it('does not deduplicate identical named queries on duplicate signals', () => { + const signals = [ + getSignalHitMock({ _id: '1', matched_queries: ['query1'] }), + getSignalHitMock({ _id: '1', matched_queries: ['query1', 'query2'] }), + ]; + const [mergedSignal] = groupAndMergeSignalMatches(signals); + expect(mergedSignal.matched_queries).toEqual(['query1', 'query1', 'query2']); + }); + + it('merges the matched_queries of multiple signals', () => { + const signals = [ + getSignalHitMock({ _id: '1', matched_queries: ['query1'] }), + getSignalHitMock({ _id: '1', matched_queries: ['query3', 'query4'] }), + getSignalHitMock({ _id: '2', matched_queries: ['query1', 'query2'] }), + getSignalHitMock({ _id: '2', matched_queries: ['query5', 'query6'] }), + ]; + const mergedSignals = groupAndMergeSignalMatches(signals); + expect(mergedSignals.map((signal) => signal.matched_queries)).toEqual([ + ['query1', 'query3', 'query4'], + ['query1', 'query2', 'query5', 'query6'], + ]); + }); +}); + +describe('buildMatchedIndicator', () => { + let threats: ThreatListItem[]; + let queries: ThreatMatchNamedQuery[]; + + beforeEach(() => { + threats = [ + getThreatListItemMock({ + _id: '123', + _source: { + threat: { indicator: { domain: 'domain_1', other: 'other_1', type: 'type_1' } }, + }, + }), + ]; + queries = [ + getNamedQueryMock({ id: '123', field: 'event.field', value: 'threat.indicator.domain' }), + ]; + }); + + it('returns an empty list if queries is empty', () => { + const indicators = buildMatchedIndicator({ + queries: [], + threats, + }); + + expect(indicators).toEqual([]); + }); + + it('returns the value of the matched indicator as matched.atomic', () => { + const [indicator] = buildMatchedIndicator({ + queries, + threats, + }); + + expect(get(indicator, 'matched.atomic')).toEqual('domain_1'); + }); + + it('returns the field of the matched indicator as matched.field', () => { + const [indicator] = buildMatchedIndicator({ + queries, + threats, + }); + + expect(get(indicator, 'matched.field')).toEqual('event.field'); + }); + + it('returns the type of the matched indicator as matched.type', () => { + const [indicator] = buildMatchedIndicator({ + queries, + threats, + }); + + expect(get(indicator, 'matched.type')).toEqual('type_1'); + }); + + it('returns indicators for each provided query', () => { + threats = [ + getThreatListItemMock({ + _id: '123', + _source: { + threat: { indicator: { domain: 'domain_1', other: 'other_1', type: 'type_1' } }, + }, + }), + getThreatListItemMock({ + _id: '456', + _source: { + threat: { indicator: { domain: 'domain_1', other: 'other_1', type: 'type_1' } }, + }, + }), + ]; + queries = [ + getNamedQueryMock({ id: '123', value: 'threat.indicator.domain' }), + getNamedQueryMock({ id: '456', value: 'threat.indicator.other' }), + getNamedQueryMock({ id: '456', value: 'threat.indicator.domain' }), + ]; + const indicators = buildMatchedIndicator({ + queries, + threats, + }); + + expect(indicators).toHaveLength(queries.length); + }); + + it('returns the indicator data specified at threat.indicator by default', () => { + const indicators = buildMatchedIndicator({ + queries, + threats, + }); + + expect(indicators).toEqual([ + { + domain: 'domain_1', + matched: { + atomic: 'domain_1', + field: 'event.field', + type: 'type_1', + }, + other: 'other_1', + type: 'type_1', + }, + ]); + }); + + it('returns the indicator data specified at the custom path', () => { + threats = [ + getThreatListItemMock({ + _id: '123', + _source: { + 'threat.indicator.domain': 'domain_1', + custom: { + indicator: { + path: { + indicator_field: 'indicator_field_1', + type: 'indicator_type', + }, + }, + }, + }, + }), + ]; + + const indicators = buildMatchedIndicator({ + indicatorPath: 'custom.indicator.path', + queries, + threats, + }); + + expect(indicators).toEqual([ + { + indicator_field: 'indicator_field_1', + matched: { + atomic: 'domain_1', + field: 'event.field', + type: 'indicator_type', + }, + type: 'indicator_type', + }, + ]); + }); + + it('returns only the match data if indicator field is absent', () => { + threats = [ + getThreatListItemMock({ + _id: '123', + _source: {}, + }), + ]; + + const indicators = buildMatchedIndicator({ + queries, + threats, + }); + + expect(indicators).toEqual([ + { + matched: { + atomic: undefined, + field: 'event.field', + type: undefined, + }, + }, + ]); + }); + + it('returns only the match data if indicator field is an empty array', () => { + threats = [ + getThreatListItemMock({ + _id: '123', + _source: { threat: { indicator: [] } }, + }), + ]; + + const indicators = buildMatchedIndicator({ + queries, + threats, + }); + + expect(indicators).toEqual([ + { + matched: { + atomic: undefined, + field: 'event.field', + type: undefined, + }, + }, + ]); + }); + + it('returns data sans atomic from first indicator if indicator field is an array of objects', () => { + threats = [ + getThreatListItemMock({ + _id: '123', + _source: { + threat: { + indicator: [ + { domain: 'foo', type: 'first' }, + { domain: 'bar', type: 'second' }, + ], + }, + }, + }), + ]; + + const indicators = buildMatchedIndicator({ + queries, + threats, + }); + + expect(indicators).toEqual([ + { + domain: 'foo', + matched: { + atomic: undefined, + field: 'event.field', + type: 'first', + }, + type: 'first', + }, + ]); + }); + + it('throws an error if indicator field is a not an object', () => { + threats = [ + getThreatListItemMock({ + _id: '123', + _source: { + threat: { + indicator: 'not an object', + }, + }, + }), + ]; + + expect(() => + buildMatchedIndicator({ + queries, + threats, + }) + ).toThrowError('Expected indicator field to be an object, but found: not an object'); + }); + + it('throws an error if indicator field is not an array of objects', () => { + threats = [ + getThreatListItemMock({ + _id: '123', + _source: { + threat: { + indicator: ['not an object'], + }, + }, + }), + ]; + + expect(() => + buildMatchedIndicator({ + queries, + threats, + }) + ).toThrowError('Expected indicator field to be an object, but found: not an object'); + }); +}); + +describe('enrichSignalThreatMatches', () => { + let getMatchedThreats: GetMatchedThreats; + let matchedQuery: string; + + beforeEach(() => { + getMatchedThreats = async () => [ + getThreatListItemMock({ + _id: '123', + _source: { + threat: { indicator: { domain: 'domain_1', other: 'other_1', type: 'type_1' } }, + }, + }), + ]; + matchedQuery = encodeThreatMatchNamedQuery( + getNamedQueryMock({ id: '123', field: 'event.field', value: 'threat.indicator.domain' }) + ); + }); + + it('performs no enrichment if there are no signals', async () => { + const signals = getSignalsResponseMock([]); + const enrichedSignals = await enrichSignalThreatMatches(signals, getMatchedThreats); + + expect(enrichedSignals.hits.hits).toEqual([]); + }); + + it('preserves existing threat.indicator objects on signals', async () => { + const signalHit = getSignalHitMock({ + _source: { '@timestamp': 'mocked', threat: { indicator: [{ existing: 'indicator' }] } }, + matched_queries: [matchedQuery], + }); + const signals = getSignalsResponseMock([signalHit]); + const enrichedSignals = await enrichSignalThreatMatches(signals, getMatchedThreats); + const [enrichedHit] = enrichedSignals.hits.hits; + const indicators = get(enrichedHit._source, 'threat.indicator'); + + expect(indicators).toEqual([ + { existing: 'indicator' }, + { + domain: 'domain_1', + matched: { atomic: 'domain_1', field: 'event.field', type: 'type_1' }, + other: 'other_1', + type: 'type_1', + }, + ]); + }); + + it('provides only match data if the matched threat cannot be found', async () => { + getMatchedThreats = async () => []; + const signalHit = getSignalHitMock({ + matched_queries: [matchedQuery], + }); + const signals = getSignalsResponseMock([signalHit]); + const enrichedSignals = await enrichSignalThreatMatches(signals, getMatchedThreats); + const [enrichedHit] = enrichedSignals.hits.hits; + const indicators = get(enrichedHit._source, 'threat.indicator'); + + expect(indicators).toEqual([ + { + matched: { atomic: undefined, field: 'event.field', type: undefined }, + }, + ]); + }); + + it('preserves an existing threat.indicator object on signals', async () => { + const signalHit = getSignalHitMock({ + _source: { '@timestamp': 'mocked', threat: { indicator: { existing: 'indicator' } } }, + matched_queries: [matchedQuery], + }); + const signals = getSignalsResponseMock([signalHit]); + const enrichedSignals = await enrichSignalThreatMatches(signals, getMatchedThreats); + const [enrichedHit] = enrichedSignals.hits.hits; + const indicators = get(enrichedHit._source, 'threat.indicator'); + + expect(indicators).toEqual([ + { existing: 'indicator' }, + { + domain: 'domain_1', + matched: { atomic: 'domain_1', field: 'event.field', type: 'type_1' }, + other: 'other_1', + type: 'type_1', + }, + ]); + }); + + it('throws an error if threat is neither an object nor undefined', async () => { + const signalHit = getSignalHitMock({ + _source: { '@timestamp': 'mocked', threat: 'whoops' }, + matched_queries: [matchedQuery], + }); + const signals = getSignalsResponseMock([signalHit]); + await expect(() => enrichSignalThreatMatches(signals, getMatchedThreats)).rejects.toThrowError( + 'Expected threat field to be an object, but found: whoops' + ); + }); + + it('merges duplicate matched signals into a single signal with multiple indicators', async () => { + getMatchedThreats = async () => [ + getThreatListItemMock({ + _id: '123', + _source: { + threat: { indicator: { domain: 'domain_1', other: 'other_1', type: 'type_1' } }, + }, + }), + getThreatListItemMock({ + _id: '456', + _source: { + threat: { indicator: { domain: 'domain_2', other: 'other_2', type: 'type_2' } }, + }, + }), + ]; + const signalHit = getSignalHitMock({ + _id: 'signal123', + matched_queries: [matchedQuery], + }); + const otherSignalHit = getSignalHitMock({ + _id: 'signal123', + matched_queries: [ + encodeThreatMatchNamedQuery( + getNamedQueryMock({ id: '456', field: 'event.other', value: 'threat.indicator.domain' }) + ), + ], + }); + const signals = getSignalsResponseMock([signalHit, otherSignalHit]); + const enrichedSignals = await enrichSignalThreatMatches(signals, getMatchedThreats); + expect(enrichedSignals.hits.total).toEqual(expect.objectContaining({ value: 1 })); + expect(enrichedSignals.hits.hits).toHaveLength(1); + + const [enrichedHit] = enrichedSignals.hits.hits; + const indicators = get(enrichedHit._source, 'threat.indicator'); + + expect(indicators).toEqual([ + { + domain: 'domain_1', + matched: { atomic: 'domain_1', field: 'event.field', type: 'type_1' }, + other: 'other_1', + type: 'type_1', + }, + { + domain: 'domain_2', + matched: { + atomic: 'domain_2', + field: 'event.other', + type: 'type_2', + }, + other: 'other_2', + type: 'type_2', + }, + ]); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.ts new file mode 100644 index 0000000000000..c298ef98ebcd5 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.ts @@ -0,0 +1,114 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { get, isObject } from 'lodash'; + +import type { SignalSearchResponse, SignalSourceHit } from '../types'; +import type { + GetMatchedThreats, + ThreatIndicator, + ThreatListItem, + ThreatMatchNamedQuery, +} from './types'; +import { extractNamedQueries } from './utils'; + +const DEFAULT_INDICATOR_PATH = 'threat.indicator'; +const getSignalId = (signal: SignalSourceHit): string => signal._id; + +export const groupAndMergeSignalMatches = (signalHits: SignalSourceHit[]): SignalSourceHit[] => { + const dedupedHitsMap = signalHits.reduce>((acc, signalHit) => { + const signalId = getSignalId(signalHit); + const existingSignalHit = acc[signalId]; + + if (existingSignalHit == null) { + acc[signalId] = signalHit; + } else { + const existingQueries = existingSignalHit?.matched_queries ?? []; + const newQueries = signalHit.matched_queries ?? []; + existingSignalHit.matched_queries = [...existingQueries, ...newQueries]; + + acc[signalId] = existingSignalHit; + } + + return acc; + }, {}); + const dedupedHits = Object.values(dedupedHitsMap); + return dedupedHits; +}; + +export const buildMatchedIndicator = ({ + queries, + threats, + indicatorPath = DEFAULT_INDICATOR_PATH, +}: { + queries: ThreatMatchNamedQuery[]; + threats: ThreatListItem[]; + indicatorPath?: string; +}): ThreatIndicator[] => + queries.map((query) => { + const matchedThreat = threats.find((threat) => threat._id === query.id); + const indicatorValue = get(matchedThreat?._source, indicatorPath) as unknown; + const indicator = [indicatorValue].flat()[0] ?? {}; + if (!isObject(indicator)) { + throw new Error(`Expected indicator field to be an object, but found: ${indicator}`); + } + const atomic = get(matchedThreat?._source, query.value) as unknown; + const type = get(indicator, 'type') as unknown; + + return { + ...indicator, + matched: { atomic, field: query.field, type }, + }; + }); + +export const enrichSignalThreatMatches = async ( + signals: SignalSearchResponse, + getMatchedThreats: GetMatchedThreats +): Promise => { + const signalHits = signals.hits.hits; + if (signalHits.length === 0) { + return signals; + } + + const uniqueHits = groupAndMergeSignalMatches(signalHits); + const signalMatches = uniqueHits.map((signalHit) => extractNamedQueries(signalHit)); + const matchedThreatIds = [...new Set(signalMatches.flat().map(({ id }) => id))]; + const matchedThreats = await getMatchedThreats(matchedThreatIds); + const matchedIndicators = signalMatches.map((queries) => + buildMatchedIndicator({ queries, threats: matchedThreats }) + ); + + const enrichedSignals: SignalSourceHit[] = uniqueHits.map((signalHit, i) => { + const threat = get(signalHit._source, 'threat') ?? {}; + if (!isObject(threat)) { + throw new Error(`Expected threat field to be an object, but found: ${threat}`); + } + const existingIndicatorValue = get(signalHit._source, 'threat.indicator') ?? []; + const existingIndicators = [existingIndicatorValue].flat(); // ensure indicators is an array + + return { + ...signalHit, + _source: { + ...signalHit._source, + threat: { + ...threat, + indicator: [...existingIndicators, ...matchedIndicators[i]], + }, + }, + }; + }); + /* eslint-disable require-atomic-updates */ + signals.hits.hits = enrichedSignals; + if (isObject(signals.hits.total)) { + signals.hits.total.value = enrichedSignals.length; + } else { + signals.hits.total = enrichedSignals.length; + } + /* eslint-enable require-atomic-updates */ + + return signals; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts index 26e42b795be3e..b80d3faf9b61c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts @@ -5,7 +5,9 @@ * 2.0. */ +import { SearchResponse } from 'elasticsearch'; import { Duration } from 'moment'; + import { ListClient } from '../../../../../../lists/server'; import { Type, @@ -31,7 +33,7 @@ import { ILegacyScopedClusterClient, Logger } from '../../../../../../../../src/ import { RuleAlertAction } from '../../../../../common/detection_engine/types'; import { TelemetryEventsSender } from '../../../telemetry/sender'; import { BuildRuleMessage } from '../rule_messages'; -import { SearchAfterAndBulkCreateReturnType } from '../types'; +import { SearchAfterAndBulkCreateReturnType, SignalsEnrichment } from '../types'; export type SortOrderOrUndefined = 'asc' | 'desc' | undefined; @@ -76,6 +78,7 @@ export interface CreateThreatSignalsOptions { export interface CreateThreatSignalOptions { threatMapping: ThreatMapping; + threatEnrichment: SignalsEnrichment; query: string; inputIndex: string[]; type: Type; @@ -177,14 +180,40 @@ export interface GetSortWithTieBreakerOptions { listItemIndex: string; } +export interface ThreatListDoc { + [key: string]: unknown; +} + /** * This is an ECS document being returned, but the user could return or use non-ecs based * documents potentially. */ -export interface ThreatListItem { +export type ThreatListItem = SearchResponse['hits']['hits'][number]; + +export interface ThreatIndicator { [key: string]: unknown; } export interface SortWithTieBreaker { [key: string]: string; } + +export interface ThreatMatchNamedQuery { + id: string; + field: string; + value: string; +} + +export type GetMatchedThreats = (ids: string[]) => Promise; + +export interface BuildThreatEnrichmentOptions { + buildRuleMessage: BuildRuleMessage; + exceptionItems: ExceptionListItemSchema[]; + listClient: ListClient; + logger: Logger; + services: AlertServices; + threatFilters: PartialFilter[]; + threatIndex: ThreatIndex; + threatLanguage: ThreatLanguageOrUndefined; + threatQuery: ThreatQuery; +} diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.test.ts index a738c8a864a1c..897143f9ae574 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.test.ts @@ -7,6 +7,7 @@ import { SearchAfterAndBulkCreateReturnType } from '../types'; import { sampleSignalHit } from '../__mocks__/es_results'; +import { ThreatMatchNamedQuery } from './types'; import { calculateAdditiveMax, @@ -14,6 +15,8 @@ import { calculateMaxLookBack, combineConcurrentResults, combineResults, + decodeThreatMatchNamedQuery, + encodeThreatMatchNamedQuery, } from './utils'; describe('utils', () => { @@ -580,4 +583,56 @@ describe('utils', () => { ); }); }); + + describe('threat match queries', () => { + describe('encodeThreatMatchNamedQuery()', () => { + it('generates a string that can be later decoded', () => { + const encoded = encodeThreatMatchNamedQuery({ + id: 'id', + field: 'field', + value: 'value', + }); + + expect(typeof encoded).toEqual('string'); + }); + }); + + describe('decodeThreatMatchNamedQuery()', () => { + it('can decode an encoded query', () => { + const query: ThreatMatchNamedQuery = { + id: 'my_id', + field: 'threat.indicator.domain', + value: 'host.name', + }; + + const encoded = encodeThreatMatchNamedQuery(query); + const decoded = decodeThreatMatchNamedQuery(encoded); + + expect(decoded).not.toBe(query); + expect(decoded).toEqual(query); + }); + + it('raises an error if the input is invalid', () => { + const badInput = 'nope'; + + expect(() => decodeThreatMatchNamedQuery(badInput)).toThrowError( + 'Decoded query is invalid. Decoded value: {"id":"nope"}' + ); + }); + + it('raises an error if the query is missing a value', () => { + const badQuery: ThreatMatchNamedQuery = { + id: 'my_id', + // @ts-expect-error field intentionally undefined + field: undefined, + value: 'host.name', + }; + const badInput = encodeThreatMatchNamedQuery(badQuery); + + expect(() => decodeThreatMatchNamedQuery(badInput)).toThrowError( + 'Decoded query is invalid. Decoded value: {"id":"my_id","field":"","value":"host.name"}' + ); + }); + }); + }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.ts index 87bcb657a53a5..72d9257798e1c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.ts @@ -5,7 +5,8 @@ * 2.0. */ -import { SearchAfterAndBulkCreateReturnType } from '../types'; +import { SearchAfterAndBulkCreateReturnType, SignalSourceHit } from '../types'; +import { ThreatMatchNamedQuery } from './types'; /** * Given two timers this will take the max of each and add them to each other and return that addition. @@ -113,3 +114,28 @@ export const combineConcurrentResults = ( return combineResults(currentResult, maxedNewResult); }; + +const separator = '___SEPARATOR___'; +export const encodeThreatMatchNamedQuery = ({ + id, + field, + value, +}: ThreatMatchNamedQuery): string => { + return [id, field, value].join(separator); +}; + +export const decodeThreatMatchNamedQuery = (encoded: string): ThreatMatchNamedQuery => { + const queryValues = encoded.split(separator); + const [id, field, value] = queryValues; + const query = { id, field, value }; + + if (queryValues.length !== 3 || !queryValues.every(Boolean)) { + const queryString = JSON.stringify(query); + throw new Error(`Decoded query is invalid. Decoded value: ${queryString}`); + } + + return query; +}; + +export const extractNamedQueries = (hit: SignalSourceHit): ThreatMatchNamedQuery[] => + hit.matched_queries?.map((match) => decodeThreatMatchNamedQuery(match)) ?? []; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts index 8031b81f70eb0..f7ac0425b2f2e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts @@ -228,6 +228,8 @@ export interface QueryFilter { }; } +export type SignalsEnrichment = (signals: SignalSearchResponse) => Promise; + export interface SearchAfterAndBulkCreateParams { gap: moment.Duration | null; previousStartedAt: Date | null | undefined; @@ -254,6 +256,7 @@ export interface SearchAfterAndBulkCreateParams { tags: string[]; throttle: string; buildRuleMessage: BuildRuleMessage; + enrichment?: SignalsEnrichment; } export interface SearchAfterAndBulkCreateReturnType { diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_threat_matching.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_threat_matching.ts index 9b29548cbe19e..9e1c290d16059 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_threat_matching.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_threat_matching.ts @@ -251,6 +251,382 @@ export default ({ getService }: FtrProviderContext) => { const signalsOpen = await getSignalsByIds(supertest, [ruleResponse.id]); expect(signalsOpen.hits.hits.length).equal(0); }); + + describe('indicator enrichment', () => { + beforeEach(async () => { + await esArchiver.load('filebeat/threat_intel'); + }); + + afterEach(async () => { + await esArchiver.unload('filebeat/threat_intel'); + }); + + it('enriches signals with the single indicator that matched', async () => { + const rule: CreateRulesSchema = { + description: 'Detecting root and admin users', + name: 'Query with a rule id', + severity: 'high', + index: ['auditbeat-*'], + type: 'threat_match', + risk_score: 55, + language: 'kuery', + rule_id: 'rule-1', + from: '1900-01-01T00:00:00.000Z', + query: '*:*', + threat_query: 'threat.indicator.domain: *', // narrow things down to indicators with a domain + threat_index: ['filebeat-*'], // Mimics indicators from the filebeat MISP module + threat_mapping: [ + { + entries: [ + { + value: 'threat.indicator.domain', + field: 'destination.ip', + type: 'mapping', + }, + ], + }, + ], + threat_filters: [], + }; + + const { id } = await createRule(supertest, rule); + await waitForRuleSuccessOrStatus(supertest, id); + await waitForSignalsToBePresent(supertest, 2, [id]); + const signalsOpen = await getSignalsByIds(supertest, [id]); + expect(signalsOpen.hits.hits.length).equal(2); + + const { hits } = signalsOpen.hits; + const threats = hits.map((hit) => hit._source.threat); + expect(threats).to.eql([ + { + indicator: [ + { + description: "domain should match the auditbeat hosts' data's source.ip", + domain: '159.89.119.67', + first_seen: '2021-01-26T11:09:04.000Z', + matched: { + atomic: '159.89.119.67', + field: 'destination.ip', + type: 'url', + }, + provider: 'geenensp', + type: 'url', + url: { + full: 'http://159.89.119.67:59600/bin.sh', + scheme: 'http', + }, + }, + ], + }, + { + indicator: [ + { + description: "domain should match the auditbeat hosts' data's source.ip", + domain: '159.89.119.67', + first_seen: '2021-01-26T11:09:04.000Z', + matched: { + atomic: '159.89.119.67', + field: 'destination.ip', + type: 'url', + }, + provider: 'geenensp', + type: 'url', + url: { + full: 'http://159.89.119.67:59600/bin.sh', + scheme: 'http', + }, + }, + ], + }, + ]); + }); + + it('enriches signals with multiple indicators if several matched', async () => { + const rule: CreateRulesSchema = { + description: 'Detecting root and admin users', + name: 'Query with a rule id', + severity: 'high', + index: ['auditbeat-*'], + type: 'threat_match', + risk_score: 55, + language: 'kuery', + rule_id: 'rule-1', + from: '1900-01-01T00:00:00.000Z', + query: 'source.port: 57324', // narrow our query to a single record that matches two indicators + threat_query: 'threat.indicator.ip: *', + threat_index: ['filebeat-*'], // Mimics indicators from the filebeat MISP module + threat_mapping: [ + { + entries: [ + { + value: 'threat.indicator.ip', + field: 'source.ip', + type: 'mapping', + }, + ], + }, + ], + threat_filters: [], + }; + + const { id } = await createRule(supertest, rule); + await waitForRuleSuccessOrStatus(supertest, id); + await waitForSignalsToBePresent(supertest, 1, [id]); + const signalsOpen = await getSignalsByIds(supertest, [id]); + expect(signalsOpen.hits.hits.length).equal(1); + + const { hits } = signalsOpen.hits; + const threats = hits.map((hit) => hit._source.threat); + expect(threats).to.eql([ + { + indicator: [ + { + description: 'this should match auditbeat/hosts on both port and ip', + first_seen: '2021-01-26T11:06:03.000Z', + ip: '45.115.45.3', + matched: { + atomic: '45.115.45.3', + field: 'source.ip', + type: 'url', + }, + port: 57324, + provider: 'geenensp', + type: 'url', + }, + { + description: 'this should match auditbeat/hosts on ip', + first_seen: '2021-01-26T11:06:03.000Z', + ip: '45.115.45.3', + matched: { + atomic: '45.115.45.3', + field: 'source.ip', + type: 'ip', + }, + provider: 'other_provider', + type: 'ip', + }, + ], + }, + ]); + }); + + it('adds a single indicator that matched multiple fields', async () => { + const rule: CreateRulesSchema = { + description: 'Detecting root and admin users', + name: 'Query with a rule id', + severity: 'high', + index: ['auditbeat-*'], + type: 'threat_match', + risk_score: 55, + language: 'kuery', + rule_id: 'rule-1', + from: '1900-01-01T00:00:00.000Z', + query: 'source.port: 57324', // narrow our query to a single record that matches two indicators + threat_query: 'threat.indicator.ip: *', + threat_index: ['filebeat-*'], // Mimics indicators from the filebeat MISP module + threat_mapping: [ + { + entries: [ + { + value: 'threat.indicator.port', + field: 'source.port', + type: 'mapping', + }, + ], + }, + { + entries: [ + { + value: 'threat.indicator.ip', + field: 'source.ip', + type: 'mapping', + }, + ], + }, + ], + threat_filters: [], + }; + + const { id } = await createRule(supertest, rule); + await waitForRuleSuccessOrStatus(supertest, id); + await waitForSignalsToBePresent(supertest, 1, [id]); + const signalsOpen = await getSignalsByIds(supertest, [id]); + expect(signalsOpen.hits.hits.length).equal(1); + + const { hits } = signalsOpen.hits; + const threats = hits.map((hit) => hit._source.threat); + + expect(threats).to.eql([ + { + indicator: [ + { + description: 'this should match auditbeat/hosts on both port and ip', + first_seen: '2021-01-26T11:06:03.000Z', + ip: '45.115.45.3', + matched: { + atomic: '45.115.45.3', + field: 'source.ip', + type: 'url', + }, + port: 57324, + provider: 'geenensp', + type: 'url', + }, + { + description: 'this should match auditbeat/hosts on ip', + first_seen: '2021-01-26T11:06:03.000Z', + ip: '45.115.45.3', + matched: { + atomic: '45.115.45.3', + field: 'source.ip', + type: 'ip', + }, + provider: 'other_provider', + type: 'ip', + }, + // We do not merge matched indicators during enrichment, so in + // certain circumstances a given indicator document could appear + // multiple times in an enriched alert (albeit with different + // threat.indicator.matched data). That's the case with the + // first and third indicators matched, here. + { + description: 'this should match auditbeat/hosts on both port and ip', + first_seen: '2021-01-26T11:06:03.000Z', + ip: '45.115.45.3', + matched: { + atomic: 57324, + field: 'source.port', + type: 'url', + }, + port: 57324, + provider: 'geenensp', + type: 'url', + }, + ], + }, + ]); + }); + + it('generates multiple signals with multiple matches', async () => { + const rule: CreateRulesSchema = { + description: 'Detecting root and admin users', + name: 'Query with a rule id', + severity: 'high', + index: ['auditbeat-*'], + type: 'threat_match', + risk_score: 55, + language: 'kuery', + rule_id: 'rule-1', + from: '1900-01-01T00:00:00.000Z', + query: '*:*', // narrow our query to a single record that matches two indicators + threat_query: '', + threat_index: ['filebeat-*'], // Mimics indicators from the filebeat MISP module + threat_mapping: [ + { + entries: [ + { + value: 'threat.indicator.port', + field: 'source.port', + type: 'mapping', + }, + { + value: 'threat.indicator.ip', + field: 'source.ip', + type: 'mapping', + }, + ], + }, + { + entries: [ + { + value: 'threat.indicator.domain', + field: 'destination.ip', + type: 'mapping', + }, + ], + }, + ], + threat_filters: [], + }; + + const { id } = await createRule(supertest, rule); + await waitForRuleSuccessOrStatus(supertest, id); + await waitForSignalsToBePresent(supertest, 2, [id]); + const signalsOpen = await getSignalsByIds(supertest, [id]); + expect(signalsOpen.hits.hits.length).equal(2); + + const { hits } = signalsOpen.hits; + const threats = hits.map((hit) => hit._source.threat); + expect(threats).to.eql([ + { + indicator: [ + { + description: "domain should match the auditbeat hosts' data's source.ip", + domain: '159.89.119.67', + first_seen: '2021-01-26T11:09:04.000Z', + matched: { + atomic: '159.89.119.67', + field: 'destination.ip', + type: 'url', + }, + provider: 'geenensp', + type: 'url', + url: { + full: 'http://159.89.119.67:59600/bin.sh', + scheme: 'http', + }, + }, + ], + }, + { + indicator: [ + { + description: "domain should match the auditbeat hosts' data's source.ip", + domain: '159.89.119.67', + first_seen: '2021-01-26T11:09:04.000Z', + matched: { + atomic: '159.89.119.67', + field: 'destination.ip', + type: 'url', + }, + provider: 'geenensp', + type: 'url', + url: { + full: 'http://159.89.119.67:59600/bin.sh', + scheme: 'http', + }, + }, + { + description: 'this should match auditbeat/hosts on both port and ip', + first_seen: '2021-01-26T11:06:03.000Z', + ip: '45.115.45.3', + matched: { + atomic: '45.115.45.3', + field: 'source.ip', + type: 'url', + }, + port: 57324, + provider: 'geenensp', + type: 'url', + }, + { + description: 'this should match auditbeat/hosts on both port and ip', + first_seen: '2021-01-26T11:06:03.000Z', + ip: '45.115.45.3', + matched: { + atomic: 57324, + field: 'source.port', + type: 'url', + }, + port: 57324, + provider: 'geenensp', + type: 'url', + }, + ], + }, + ]); + }); + }); }); }); }; diff --git a/x-pack/test/functional/es_archives/filebeat/threat_intel/data.json b/x-pack/test/functional/es_archives/filebeat/threat_intel/data.json new file mode 100644 index 0000000000000..0cbc7f37bd519 --- /dev/null +++ b/x-pack/test/functional/es_archives/filebeat/threat_intel/data.json @@ -0,0 +1,276 @@ +{ + "type": "doc", + "value": { + "id": "978783", + "index": "filebeat-8.0.0-2021.01.26-000001", + "source": { + "@timestamp": "2021-01-26T11:09:05.529Z", + "agent": { + "ephemeral_id": "b7b56c3e-1f27-4c69-96f4-aa9ca47888d0", + "id": "69acb5f0-1e79-4cfe-a4dc-e0dbf229ff51", + "name": "MacBook-Pro-de-Gloria.local", + "type": "filebeat", + "version": "8.0.0" + }, + "ecs": { + "version": "1.6.0" + }, + "event": { + "category": "threat", + "created": "2021-01-26T11:09:05.529Z", + "dataset": "threatintel.abuseurl", + "ingested": "2021-01-26T11:09:06.595350Z", + "kind": "enrichment", + "module": "threatintel", + "reference": "https://urlhaus.abuse.ch/url/978783/", + "type": "indicator" + }, + "fileset": { + "name": "abuseurl" + }, + "input": { + "type": "httpjson" + }, + "service": { + "type": "threatintel" + }, + "tags": [ + "threatintel-abuseurls", + "forwarded" + ], + "threat": { + "indicator": { + "description": "domain should match the auditbeat hosts' data's source.ip", + "domain": "159.89.119.67", + "first_seen": "2021-01-26T11:09:04.000Z", + "provider": "geenensp", + "type": "url", + "url": { + "full": "http://159.89.119.67:59600/bin.sh", + "scheme": "http" + } + } + }, + "threatintel": { + "abuseurl": { + "blacklists": { + "spamhaus_dbl": "not listed", + "surbl": "not listed" + }, + "larted": false, + "tags": null, + "threat": "malware_download", + "url_status": "online" + } + } + } + } +} + +{ + "type": "doc", + "value": { + "id": "978784", + "index": "filebeat-8.0.0-2021.01.26-000001", + "source": { + "@timestamp": "2021-01-26T11:09:05.529Z", + "agent": { + "ephemeral_id": "b7b56c3e-1f27-4c69-96f4-aa9ca47888d0", + "id": "69acb5f0-1e79-4cfe-a4dc-e0dbf229ff51", + "name": "MacBook-Pro-de-Gloria.local", + "type": "filebeat", + "version": "8.0.0" + }, + "ecs": { + "version": "1.6.0" + }, + "event": { + "category": "threat", + "created": "2021-01-26T11:09:05.529Z", + "dataset": "threatintel.abuseurl", + "ingested": "2021-01-26T11:09:06.616763Z", + "kind": "enrichment", + "module": "threatintel", + "reference": "https://urlhaus.abuse.ch/url/978782/", + "type": "indicator" + }, + "fileset": { + "name": "abuseurl" + }, + "input": { + "type": "httpjson" + }, + "service": { + "type": "threatintel" + }, + "tags": [ + "threatintel-abuseurls", + "forwarded" + ], + "threat": { + "indicator": { + "description": "this should not match the auditbeat hosts data", + "ip": "125.46.136.106", + "first_seen": "2021-01-26T11:06:03.000Z", + "provider": "geenensp", + "type": "ip" + } + }, + "threatintel": { + "abuseurl": { + "blacklists": { + "spamhaus_dbl": "not listed", + "surbl": "not listed" + }, + "larted": true, + "tags": [ + "32-bit", + "elf", + "mips" + ], + "threat": "malware_download", + "url_status": "online" + } + } + } + } +} + +{ + "type": "doc", + "value": { + "id": "978785", + "index": "filebeat-8.0.0-2021.01.26-000001", + "source": { + "@timestamp": "2021-01-26T11:09:05.529Z", + "agent": { + "ephemeral_id": "b7b56c3e-1f27-4c69-96f4-aa9ca47888d0", + "id": "69acb5f0-1e79-4cfe-a4dc-e0dbf229ff51", + "name": "MacBook-Pro-de-Gloria.local", + "type": "filebeat", + "version": "8.0.0" + }, + "ecs": { + "version": "1.6.0" + }, + "event": { + "category": "threat", + "created": "2021-01-26T11:09:05.529Z", + "dataset": "threatintel.abuseurl", + "ingested": "2021-01-26T11:09:06.616763Z", + "kind": "enrichment", + "module": "threatintel", + "reference": "https://urlhaus.abuse.ch/url/978782/", + "type": "indicator" + }, + "fileset": { + "name": "abuseurl" + }, + "input": { + "type": "httpjson" + }, + "service": { + "type": "threatintel" + }, + "tags": [ + "threatintel-abuseurls", + "forwarded" + ], + "threat": { + "indicator": { + "description": "this should match auditbeat/hosts on both port and ip", + "ip": "45.115.45.3", + "port": 57324, + "first_seen": "2021-01-26T11:06:03.000Z", + "provider": "geenensp", + "type": "url" + } + }, + "threatintel": { + "abuseurl": { + "blacklists": { + "spamhaus_dbl": "not listed", + "surbl": "not listed" + }, + "larted": true, + "tags": [ + "32-bit", + "elf", + "mips" + ], + "threat": "malware_download", + "url_status": "online" + } + } + } + } +} + +{ + "type": "doc", + "value": { + "id": "978787", + "index": "filebeat-8.0.0-2021.01.26-000001", + "source": { + "@timestamp": "2021-01-26T11:09:05.529Z", + "agent": { + "ephemeral_id": "b7b56c3e-1f27-4c69-96f4-aa9ca47888d0", + "id": "69acb5f0-1e79-4cfe-a4dc-e0dbf229ff51", + "name": "MacBook-Pro-de-Gloria.local", + "type": "filebeat", + "version": "8.0.0" + }, + "ecs": { + "version": "1.6.0" + }, + "event": { + "category": "threat", + "created": "2021-01-26T11:09:05.529Z", + "dataset": "threatintel.abuseurl", + "ingested": "2021-01-26T11:09:06.616763Z", + "kind": "enrichment", + "module": "threatintel", + "reference": "https://urlhaus.abuse.ch/url/978782/", + "type": "indicator" + }, + "fileset": { + "name": "abuseurl" + }, + "input": { + "type": "httpjson" + }, + "service": { + "type": "threatintel" + }, + "tags": [ + "threatintel-abuseurls", + "forwarded" + ], + "threat": { + "indicator": { + "description": "this should match auditbeat/hosts on ip", + "ip": "45.115.45.3", + "first_seen": "2021-01-26T11:06:03.000Z", + "provider": "other_provider", + "type": "ip" + } + }, + "threatintel": { + "abuseurl": { + "blacklists": { + "spamhaus_dbl": "not listed", + "surbl": "not listed" + }, + "larted": true, + "tags": [ + "32-bit", + "elf", + "mips" + ], + "threat": "malware_download", + "url_status": "online" + } + } + } + } +} diff --git a/x-pack/test/functional/es_archives/filebeat/threat_intel/mappings.json b/x-pack/test/functional/es_archives/filebeat/threat_intel/mappings.json new file mode 100644 index 0000000000000..26d8e29eaecf7 --- /dev/null +++ b/x-pack/test/functional/es_archives/filebeat/threat_intel/mappings.json @@ -0,0 +1,243 @@ +{ + "type": "index", + "value": { + "aliases": {}, + "index": "filebeat-8.0.0-2021.01.26-000001", + "mappings": { + "_meta": { + "beat": "filebeat", + "version": "7.0.0" + }, + "properties": { + "@timestamp": { + "type": "date" + }, + "@version": { + "ignore_above": 1024, + "type": "keyword" + }, + "threat": { + "properties": { + "framework": { + "ignore_above": 1024, + "type": "keyword" + }, + "indicator": { + "properties": { + "as": { + "properties": { + "number": { + "type": "long" + }, + "organization": { + "properties": { + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "confidence": { + "ignore_above": 1024, + "type": "keyword" + }, + "dataset": { + "ignore_above": 1024, + "type": "keyword" + }, + "description": { + "type": "wildcard" + }, + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "email": { + "properties": { + "address": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "first_seen": { + "type": "date" + }, + "geo": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "ip": { + "type": "ip" + }, + "last_seen": { + "type": "date" + }, + "marking": { + "properties": { + "tlp": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "matched": { + "properties": { + "atomic": { + "ignore_above": 1024, + "type": "keyword" + }, + "field": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "module": { + "ignore_above": 1024, + "type": "keyword" + }, + "port": { + "type": "long" + }, + "provider": { + "ignore_above": 1024, + "type": "keyword" + }, + "scanner_stats": { + "type": "long" + }, + "sightings": { + "type": "long" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "tactic": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "reference": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "technique": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "reference": { + "ignore_above": 1024, + "type": "keyword" + }, + "subtechnique": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "reference": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + } + } + } + } + }, + "settings": { + "index": { + "lifecycle": { + "name": "filebeat-8.0.0", + "rollover_alias": "filebeat-filebeat-8.0.0" + }, + "mapping": { + "total_fields": { + "limit": "10000" + } + }, + "number_of_replicas": "0", + "number_of_shards": "1", + "refresh_interval": "5s" + } + } + } +}