diff --git a/x-pack/legacy/plugins/siem/scripts/convert_saved_search_to_signals.js b/x-pack/legacy/plugins/siem/scripts/convert_saved_search_to_signals.js index a1889a400a183..597d93a44210b 100644 --- a/x-pack/legacy/plugins/siem/scripts/convert_saved_search_to_signals.js +++ b/x-pack/legacy/plugins/siem/scripts/convert_saved_search_to_signals.js @@ -35,6 +35,8 @@ const FROM = 'now-6m'; const TO = 'now'; const IMMUTABLE = true; const INDEX = ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*']; +const OUTPUT_INDEX = process.env.SIGNALS_INDEX || '.siem-signals'; +const RISK_SCORE = 50; const walk = dir => { const list = fs.readdirSync(dir); @@ -119,6 +121,7 @@ async function main() { if (query != null && query.trim() !== '') { const outputMessage = { rule_id: fileToWrite, + risk_score: RISK_SCORE, description: description || title, immutable: IMMUTABLE, index: INDEX, @@ -131,6 +134,7 @@ async function main() { query, language, filters: filter, + output_index: OUTPUT_INDEX, }; fs.writeFileSync( diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/__mocks__/es_results.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/__mocks__/es_results.ts index 5630149617d1b..5004c2af0838a 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/__mocks__/es_results.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/__mocks__/es_results.ts @@ -4,19 +4,17 @@ * you may not use this file except in compliance with the Elastic License. */ -import { SignalSourceHit, SignalSearchResponse, SignalAlertParams } from '../types'; +import { SignalSourceHit, SignalSearchResponse, AlertTypeParams } from '../types'; export const sampleSignalAlertParams = ( maxSignals: number | undefined, riskScore?: number | undefined -): SignalAlertParams => ({ +): AlertTypeParams => ({ ruleId: 'rule-1', description: 'Detecting root and admin users', falsePositives: [], immutable: false, index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], - interval: '5m', - name: 'Detect Root/Admin Users', type: 'query', from: 'now-6m', tags: ['some fake tag'], @@ -28,7 +26,6 @@ export const sampleSignalAlertParams = ( references: ['http://google.com'], riskScore: riskScore ? riskScore : 50, maxSignals: maxSignals ? maxSignals : 10000, - enabled: true, filter: undefined, filters: undefined, savedId: undefined, diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/build_events_reindex.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/build_events_reindex.ts deleted file mode 100644 index 0e8d95e4f7ac1..0000000000000 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/build_events_reindex.ts +++ /dev/null @@ -1,146 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -// TODO: Re-index is just a temporary solution in order to speed up development -// of any front end pieces. This should be replaced with a combination of the file -// build_events_query.ts and any scrolling/scaling solutions from that particular -// file. - -interface BuildEventsReIndexParams { - description: string; - index: string[]; - from: string; - to: string; - signalsIndex: string; - maxDocs: number; - filter: unknown; - severity: string; - name: string; - timeDetected: string; - ruleRevision: number; - id: string; - ruleId: string | undefined | null; - type: string; - references: string[]; -} - -export const buildEventsReIndex = ({ - description, - index, - from, - to, - signalsIndex, - maxDocs, - filter, - severity, - name, - timeDetected, - ruleRevision, - id, - ruleId, - type, - references, -}: BuildEventsReIndexParams) => { - const indexPatterns = index.map(element => `"${element}"`).join(','); - const refs = references.map(element => `"${element}"`).join(','); - const filterWithTime = [ - filter, - { - bool: { - filter: [ - { - bool: { - should: [ - { - range: { - '@timestamp': { - gte: from, - }, - }, - }, - ], - minimum_should_match: 1, - }, - }, - { - bool: { - should: [ - { - range: { - '@timestamp': { - lte: to, - }, - }, - }, - ], - minimum_should_match: 1, - }, - }, - ], - }, - }, - ]; - return { - body: { - source: { - index, - sort: [ - { - '@timestamp': 'desc', - }, - { - _doc: 'desc', - }, - ], - query: { - bool: { - filter: [ - ...filterWithTime, - { - match_all: {}, - }, - ], - }, - }, - }, - dest: { - index: signalsIndex, - }, - script: { - source: ` - String[] indexPatterns = new String[] {${indexPatterns}}; - String[] references = new String[] {${refs}}; - - def parent = [ - "id": ctx._id, - "type": "event", - "index": ctx._index, - "depth": 1 - ]; - - def signal = [ - "id": "${id}", - "rule_revision": "${ruleRevision}", - "rule_id": "${ruleId}", - "rule_type": "${type}", - "parent": parent, - "name": "${name}", - "severity": "${severity}", - "description": "${description}", - "original_time": ctx._source['@timestamp'], - "index_patterns": indexPatterns, - "references": references - ]; - - ctx._source.signal = signal; - ctx._source['@timestamp'] = "${timeDetected}"; - `, - lang: 'painless', - }, - max_docs: maxDocs, - }, - }; -}; diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/signals_alert_type.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/signals_alert_type.ts index 6cdf10707ce2b..a6fa6fb3bdecf 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/signals_alert_type.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/signals_alert_type.ts @@ -7,8 +7,6 @@ import { schema } from '@kbn/config-schema'; import { Logger } from 'src/core/server'; import { SIGNALS_ID } from '../../../../common/constants'; -// TODO: Remove this for the build_events_query call eventually -import { buildEventsReIndex } from './build_events_reindex'; import { buildEventsSearchQuery } from './build_events_query'; import { searchAfterAndBulkIndex } from './utils'; @@ -46,7 +44,6 @@ export const signalsAlertType = ({ logger }: { logger: Logger }): SignalAlertTyp }, async executor({ alertId, services, params }) { const { - description, filter, from, ruleId, @@ -56,10 +53,6 @@ export const signalsAlertType = ({ logger }: { logger: Logger }): SignalAlertTyp outputIndex, savedId, query, - maxSignals, - // riskScore, TODO: Add and copy this data and any other data over to the rule - references, - severity, to, type, size, @@ -67,7 +60,13 @@ export const signalsAlertType = ({ logger }: { logger: Logger }): SignalAlertTyp // TODO: Remove this hard extraction of name once this is fixed: https://github.com/elastic/kibana/issues/50522 const savedObject = await services.savedObjectsClient.get('alert', alertId); - const name = savedObject.attributes.name; + const name: string = savedObject.attributes.name; + + const createdBy: string = savedObject.attributes.createdBy; + const updatedBy: string = savedObject.attributes.updatedBy; + const interval: string = savedObject.attributes.interval; + const enabled: boolean = savedObject.attributes.enabled; + const searchAfterSize = size ? size : 1000; const esFilter = await getFilter({ @@ -92,55 +91,34 @@ export const signalsAlertType = ({ logger }: { logger: Logger }): SignalAlertTyp try { logger.debug(`Starting signal rule "id: ${alertId}", "ruleId: ${ruleId}"`); - if (process.env.USE_REINDEX_API === 'true') { - const reIndex = buildEventsReIndex({ - index, - from, - to, - signalsIndex: outputIndex, - severity, - description, - name, - timeDetected: new Date().toISOString(), - filter: esFilter, - maxDocs: maxSignals, - ruleRevision: 1, - id: alertId, - ruleId, - type, - references, - }); - const result = await services.callCluster('reindex', reIndex); - if (result.total > 0) { - logger.info( - `Total signals found from signal rule "id: ${alertId}", "ruleId: ${ruleId}" (reindex algorithm): ${result.total}` - ); - } - } else { - logger.debug( - `[+] Initial search call of signal rule "id: ${alertId}", "ruleId: ${ruleId}"` + logger.debug( + `[+] Initial search call of signal rule "id: ${alertId}", "ruleId: ${ruleId}"` + ); + const noReIndexResult = await services.callCluster('search', noReIndex); + if (noReIndexResult.hits.total.value !== 0) { + logger.info( + `Total signals found from signal rule "id: ${alertId}", "ruleId: ${ruleId}": ${noReIndexResult.hits.total.value}` ); - const noReIndexResult = await services.callCluster('search', noReIndex); - if (noReIndexResult.hits.total.value !== 0) { - logger.info( - `Total signals found from signal rule "id: ${alertId}", "ruleId: ${ruleId}": ${noReIndexResult.hits.total.value}` - ); - } + } - const bulkIndexResult = await searchAfterAndBulkIndex( - noReIndexResult, - params, - services, - logger, - alertId, - outputIndex - ); + const bulkIndexResult = await searchAfterAndBulkIndex({ + someResult: noReIndexResult, + signalParams: params, + services, + logger, + id: alertId, + signalsIndex: outputIndex, + name, + createdBy, + updatedBy, + interval, + enabled, + }); - if (bulkIndexResult) { - logger.debug(`Finished signal rule "id: ${alertId}", "ruleId: ${ruleId}"`); - } else { - logger.error(`Error processing signal rule "id: ${alertId}", "ruleId: ${ruleId}"`); - } + if (bulkIndexResult) { + logger.debug(`Finished signal rule "id: ${alertId}", "ruleId: ${ruleId}"`); + } else { + logger.error(`Error processing signal rule "id: ${alertId}", "ruleId: ${ruleId}"`); } } catch (err) { // TODO: Error handling and writing of errors into a signal that has error diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/types.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/types.ts index cdfea884bd464..64dda913f2c9d 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/types.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/types.ts @@ -65,6 +65,10 @@ export type OutputSignalAlertRest = SignalAlertParamsRest & { updated_by: string | undefined | null; }; +export type OutputSignalES = OutputSignalAlertRest & { + status: 'open' | 'closed'; +}; + export type UpdateSignalAlertParamsRest = Partial & { id: string | undefined; rule_id: SignalAlertParams['ruleId'] | undefined; diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.test.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.test.ts index bcd639eddfae9..bc147fa1dae07 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.test.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.test.ts @@ -47,28 +47,54 @@ describe('utils', () => { describe('buildBulkBody', () => { test('if bulk body builds well-defined body', () => { const sampleParams = sampleSignalAlertParams(undefined); - const fakeSignalSourceHit = buildBulkBody(sampleDocNoSortId, sampleParams, sampleSignalId); + const fakeSignalSourceHit = buildBulkBody({ + doc: sampleDocNoSortId, + signalParams: sampleParams, + id: sampleSignalId, + name: 'rule-name', + createdBy: 'elastic', + updatedBy: 'elastic', + interval: '5m', + enabled: true, + }); + // Timestamp will potentially always be different so remove it for the test + delete fakeSignalSourceHit['@timestamp']; expect(fakeSignalSourceHit).toEqual({ someKey: 'someValue', - '@timestamp': 'someTimeStamp', signal: { - id: sampleSignalId, - '@timestamp': fakeSignalSourceHit.signal['@timestamp'], // timestamp generated in the body - rule_revision: 1, - rule_id: sampleParams.ruleId, - rule_type: sampleParams.type, parent: { - id: sampleDocNoSortId._id, + id: 'someFakeId', type: 'event', - index: sampleDocNoSortId._index, + index: 'myFakeSignalIndex', depth: 1, }, - name: sampleParams.name, - severity: sampleParams.severity, - description: sampleParams.description, - original_time: sampleDocNoSortId._source['@timestamp'], - index_patterns: sampleParams.index, - references: sampleParams.references, + original_time: 'someTimeStamp', + rule: { + id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + rule_id: 'rule-1', + false_positives: [], + max_signals: 10000, + risk_score: 50, + output_index: '.siem-signals', + description: 'Detecting root and admin users', + from: 'now-6m', + immutable: false, + index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], + interval: '5m', + language: 'kuery', + name: 'rule-name', + query: 'user.name: root or user.name: admin', + references: ['http://google.com'], + severity: 'high', + tags: ['some fake tag'], + type: 'query', + size: 1000, + status: 'open', + to: 'now', + enabled: true, + created_by: 'elastic', + updated_by: 'elastic', + }, }, }); }); @@ -86,28 +112,38 @@ describe('utils', () => { }, ], }); - const successfulSingleBulkIndex = await singleBulkIndex( - sampleSearchResult, - sampleParams, - mockService, - mockLogger, - sampleSignalId, - DEFAULT_SIGNALS_INDEX - ); + const successfulSingleBulkIndex = await singleBulkIndex({ + someResult: sampleSearchResult, + signalParams: sampleParams, + services: mockService, + logger: mockLogger, + id: sampleSignalId, + signalsIndex: DEFAULT_SIGNALS_INDEX, + name: 'rule-name', + createdBy: 'elastic', + updatedBy: 'elastic', + interval: '5m', + enabled: true, + }); expect(successfulSingleBulkIndex).toEqual(true); }); test('create unsuccessful bulk index due to empty search results', async () => { const sampleParams = sampleSignalAlertParams(undefined); const sampleSearchResult = sampleEmptyDocSearchResults; mockService.callCluster.mockReturnValue(false); - const successfulSingleBulkIndex = await singleBulkIndex( - sampleSearchResult, - sampleParams, - mockService, - mockLogger, - sampleSignalId, - DEFAULT_SIGNALS_INDEX - ); + const successfulSingleBulkIndex = await singleBulkIndex({ + someResult: sampleSearchResult, + signalParams: sampleParams, + services: mockService, + logger: mockLogger, + id: sampleSignalId, + signalsIndex: DEFAULT_SIGNALS_INDEX, + name: 'rule-name', + createdBy: 'elastic', + updatedBy: 'elastic', + interval: '5m', + enabled: true, + }); expect(successfulSingleBulkIndex).toEqual(true); }); test('create unsuccessful bulk index due to bulk index errors', async () => { @@ -118,14 +154,19 @@ describe('utils', () => { took: 100, errors: true, }); - const successfulSingleBulkIndex = await singleBulkIndex( - sampleSearchResult, - sampleParams, - mockService, - mockLogger, - sampleSignalId, - DEFAULT_SIGNALS_INDEX - ); + const successfulSingleBulkIndex = await singleBulkIndex({ + someResult: sampleSearchResult, + signalParams: sampleParams, + services: mockService, + logger: mockLogger, + id: sampleSignalId, + signalsIndex: DEFAULT_SIGNALS_INDEX, + name: 'rule-name', + createdBy: 'elastic', + updatedBy: 'elastic', + interval: '5m', + enabled: true, + }); expect(mockLogger.error).toHaveBeenCalled(); expect(successfulSingleBulkIndex).toEqual(false); }); @@ -136,19 +177,24 @@ describe('utils', () => { const sampleParams = sampleSignalAlertParams(undefined); mockService.callCluster.mockReturnValue(sampleDocSearchResultsNoSortId); await expect( - singleSearchAfter(searchAfterSortId, sampleParams, mockService, mockLogger) + singleSearchAfter({ + searchAfterSortId, + signalParams: sampleParams, + services: mockService, + logger: mockLogger, + }) ).rejects.toThrow('Attempted to search after with empty sort id'); }); test('if singleSearchAfter works with a given sort id', async () => { const searchAfterSortId = '1234567891111'; const sampleParams = sampleSignalAlertParams(undefined); mockService.callCluster.mockReturnValue(sampleDocSearchResultsWithSortId); - const searchAfterResult = await singleSearchAfter( + const searchAfterResult = await singleSearchAfter({ searchAfterSortId, - sampleParams, - mockService, - mockLogger - ); + signalParams: sampleParams, + services: mockService, + logger: mockLogger, + }); expect(searchAfterResult).toEqual(sampleDocSearchResultsWithSortId); }); test('if singleSearchAfter throws error', async () => { @@ -158,21 +204,31 @@ describe('utils', () => { throw Error('Fake Error'); }); await expect( - singleSearchAfter(searchAfterSortId, sampleParams, mockService, mockLogger) + singleSearchAfter({ + searchAfterSortId, + signalParams: sampleParams, + services: mockService, + logger: mockLogger, + }) ).rejects.toThrow('Fake Error'); }); }); describe('searchAfterAndBulkIndex', () => { test('if successful with empty search results', async () => { const sampleParams = sampleSignalAlertParams(undefined); - const result = await searchAfterAndBulkIndex( - sampleEmptyDocSearchResults, - sampleParams, - mockService, - mockLogger, - sampleSignalId, - DEFAULT_SIGNALS_INDEX - ); + const result = await searchAfterAndBulkIndex({ + someResult: sampleEmptyDocSearchResults, + signalParams: sampleParams, + services: mockService, + logger: mockLogger, + id: sampleSignalId, + signalsIndex: DEFAULT_SIGNALS_INDEX, + name: 'rule-name', + createdBy: 'elastic', + updatedBy: 'elastic', + interval: '5m', + enabled: true, + }); expect(mockService.callCluster).toHaveBeenCalledTimes(0); expect(result).toEqual(true); }); @@ -208,14 +264,19 @@ describe('utils', () => { }, ], }); - const result = await searchAfterAndBulkIndex( - repeatedSearchResultsWithSortId(4), - sampleParams, - mockService, - mockLogger, - sampleSignalId, - DEFAULT_SIGNALS_INDEX - ); + const result = await searchAfterAndBulkIndex({ + someResult: repeatedSearchResultsWithSortId(4), + signalParams: sampleParams, + services: mockService, + logger: mockLogger, + id: sampleSignalId, + signalsIndex: DEFAULT_SIGNALS_INDEX, + name: 'rule-name', + createdBy: 'elastic', + updatedBy: 'elastic', + interval: '5m', + enabled: true, + }); expect(mockService.callCluster).toHaveBeenCalledTimes(5); expect(result).toEqual(true); }); @@ -225,14 +286,19 @@ describe('utils', () => { took: 100, errors: true, // will cause singleBulkIndex to return false }); - const result = await searchAfterAndBulkIndex( - repeatedSearchResultsWithSortId(4), - sampleParams, - mockService, - mockLogger, - sampleSignalId, - DEFAULT_SIGNALS_INDEX - ); + const result = await searchAfterAndBulkIndex({ + someResult: repeatedSearchResultsWithSortId(4), + signalParams: sampleParams, + services: mockService, + logger: mockLogger, + id: sampleSignalId, + signalsIndex: DEFAULT_SIGNALS_INDEX, + name: 'rule-name', + createdBy: 'elastic', + updatedBy: 'elastic', + interval: '5m', + enabled: true, + }); expect(mockLogger.error).toHaveBeenCalled(); expect(result).toEqual(false); }); @@ -248,14 +314,19 @@ describe('utils', () => { }, ], }); - const result = await searchAfterAndBulkIndex( - sampleDocSearchResultsNoSortId, - sampleParams, - mockService, - mockLogger, - sampleSignalId, - DEFAULT_SIGNALS_INDEX - ); + const result = await searchAfterAndBulkIndex({ + someResult: sampleDocSearchResultsNoSortId, + signalParams: sampleParams, + services: mockService, + logger: mockLogger, + id: sampleSignalId, + signalsIndex: DEFAULT_SIGNALS_INDEX, + name: 'rule-name', + createdBy: 'elastic', + updatedBy: 'elastic', + interval: '5m', + enabled: true, + }); expect(mockLogger.error).toHaveBeenCalled(); expect(result).toEqual(false); }); @@ -270,14 +341,19 @@ describe('utils', () => { }, ], }); - const result = await searchAfterAndBulkIndex( - sampleDocSearchResultsNoSortIdNoHits, - sampleParams, - mockService, - mockLogger, - sampleSignalId, - DEFAULT_SIGNALS_INDEX - ); + const result = await searchAfterAndBulkIndex({ + someResult: sampleDocSearchResultsNoSortIdNoHits, + signalParams: sampleParams, + services: mockService, + logger: mockLogger, + id: sampleSignalId, + signalsIndex: DEFAULT_SIGNALS_INDEX, + name: 'rule-name', + createdBy: 'elastic', + updatedBy: 'elastic', + interval: '5m', + enabled: true, + }); expect(result).toEqual(true); }); test('if successful iteration of while loop with maxDocs and search after returns results with no sort ids', async () => { @@ -293,14 +369,19 @@ describe('utils', () => { ], }) .mockReturnValueOnce(sampleDocSearchResultsNoSortId); - const result = await searchAfterAndBulkIndex( - repeatedSearchResultsWithSortId(4), - sampleParams, - mockService, - mockLogger, - sampleSignalId, - DEFAULT_SIGNALS_INDEX - ); + const result = await searchAfterAndBulkIndex({ + someResult: repeatedSearchResultsWithSortId(4), + signalParams: sampleParams, + services: mockService, + logger: mockLogger, + id: sampleSignalId, + signalsIndex: DEFAULT_SIGNALS_INDEX, + name: 'rule-name', + createdBy: 'elastic', + updatedBy: 'elastic', + interval: '5m', + enabled: true, + }); expect(result).toEqual(true); }); test('if successful iteration of while loop with maxDocs and search after returns empty results with no sort ids', async () => { @@ -316,14 +397,19 @@ describe('utils', () => { ], }) .mockReturnValueOnce(sampleEmptyDocSearchResults); - const result = await searchAfterAndBulkIndex( - repeatedSearchResultsWithSortId(4), - sampleParams, - mockService, - mockLogger, - sampleSignalId, - DEFAULT_SIGNALS_INDEX - ); + const result = await searchAfterAndBulkIndex({ + someResult: repeatedSearchResultsWithSortId(4), + signalParams: sampleParams, + services: mockService, + logger: mockLogger, + id: sampleSignalId, + signalsIndex: DEFAULT_SIGNALS_INDEX, + name: 'rule-name', + createdBy: 'elastic', + updatedBy: 'elastic', + interval: '5m', + enabled: true, + }); expect(result).toEqual(true); }); test('if logs error when iteration is unsuccessful when bulk index results in a failure', async () => { @@ -340,14 +426,19 @@ describe('utils', () => { ], }) .mockReturnValueOnce(sampleDocSearchResultsWithSortId); // get some more docs - const result = await searchAfterAndBulkIndex( - repeatedSearchResultsWithSortId(4), - sampleParams, - mockService, - mockLogger, - sampleSignalId, - DEFAULT_SIGNALS_INDEX - ); + const result = await searchAfterAndBulkIndex({ + someResult: repeatedSearchResultsWithSortId(4), + signalParams: sampleParams, + services: mockService, + logger: mockLogger, + id: sampleSignalId, + signalsIndex: DEFAULT_SIGNALS_INDEX, + name: 'rule-name', + createdBy: 'elastic', + updatedBy: 'elastic', + interval: '5m', + enabled: true, + }); expect(mockLogger.error).toHaveBeenCalled(); expect(result).toEqual(true); }); @@ -364,14 +455,19 @@ describe('utils', () => { ], }) .mockRejectedValueOnce(Error('Fake Error')); - const result = await searchAfterAndBulkIndex( - repeatedSearchResultsWithSortId(4), - sampleParams, - mockService, - mockLogger, - sampleSignalId, - DEFAULT_SIGNALS_INDEX - ); + const result = await searchAfterAndBulkIndex({ + someResult: repeatedSearchResultsWithSortId(4), + signalParams: sampleParams, + services: mockService, + logger: mockLogger, + id: sampleSignalId, + signalsIndex: DEFAULT_SIGNALS_INDEX, + name: 'rule-name', + createdBy: 'elastic', + updatedBy: 'elastic', + interval: '5m', + enabled: true, + }); expect(result).toEqual(false); }); }); diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.ts index cb11e07049e3b..aa122d0c99535 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.ts @@ -4,65 +4,164 @@ * you may not use this file except in compliance with the Elastic License. */ import { performance } from 'perf_hooks'; -import { SignalHit } from '../../types'; +import { pickBy } from 'lodash/fp'; +import { SignalHit, Signal } from '../../types'; import { Logger } from '../../../../../../../../src/core/server'; import { AlertServices } from '../../../../../alerting/server/types'; -import { SignalSourceHit, SignalSearchResponse, SignalAlertParams, BulkResponse } from './types'; +import { + SignalSourceHit, + SignalSearchResponse, + BulkResponse, + AlertTypeParams, + OutputSignalES, +} from './types'; import { buildEventsSearchQuery } from './build_events_query'; -// format search_after result for signals index. -export const buildBulkBody = ( - doc: SignalSourceHit, - signalParams: SignalAlertParams, - id: string -): SignalHit => { +interface BuildRuleParams { + signalParams: AlertTypeParams; + name: string; + id: string; + enabled: boolean; + createdBy: string; + updatedBy: string; + interval: string; +} + +export const buildRule = ({ + signalParams, + name, + id, + enabled, + createdBy, + updatedBy, + interval, +}: BuildRuleParams): Partial => { + return pickBy((value: unknown) => value != null, { + id, + status: 'open', + rule_id: signalParams.ruleId, + false_positives: signalParams.falsePositives, + saved_id: signalParams.savedId, + max_signals: signalParams.maxSignals, + risk_score: signalParams.riskScore, + output_index: signalParams.outputIndex, + description: signalParams.description, + filter: signalParams.filter, + from: signalParams.from, + immutable: signalParams.immutable, + index: signalParams.index, + interval, + language: signalParams.language, + name, + query: signalParams.query, + references: signalParams.references, + severity: signalParams.severity, + tags: signalParams.tags, + type: signalParams.type, + size: signalParams.size, + to: signalParams.to, + enabled, + filters: signalParams.filters, + created_by: createdBy, + updated_by: updatedBy, + }); +}; + +export const buildSignal = (doc: SignalSourceHit, rule: Partial): Signal => { return { - ...doc._source, - signal: { - '@timestamp': new Date().toISOString(), - id, - rule_revision: 1, - rule_id: signalParams.ruleId, - rule_type: signalParams.type, - parent: { - id: doc._id, - type: 'event', - index: doc._index, - depth: 1, - }, - name: signalParams.name, - severity: signalParams.severity, - description: signalParams.description, - original_time: doc._source['@timestamp'], - index_patterns: signalParams.index, - references: signalParams.references, + parent: { + id: doc._id, + type: 'event', + index: doc._index, + depth: 1, }, + original_time: doc._source['@timestamp'], + rule, + }; +}; + +interface BuildBulkBodyParams { + doc: SignalSourceHit; + signalParams: AlertTypeParams; + id: string; + name: string; + createdBy: string; + updatedBy: string; + interval: string; + enabled: boolean; +} + +// format search_after result for signals index. +export const buildBulkBody = ({ + doc, + signalParams, + id, + name, + createdBy, + updatedBy, + interval, + enabled, +}: BuildBulkBodyParams): SignalHit => { + const rule = buildRule({ + signalParams, + id, + name, + enabled, + createdBy, + updatedBy, + interval, + }); + const signal = buildSignal(doc, rule); + const signalHit: SignalHit = { + ...doc._source, + '@timestamp': new Date().toISOString(), + signal, }; + return signalHit; }; +interface SingleBulkIndexParams { + someResult: SignalSearchResponse; + signalParams: AlertTypeParams; + services: AlertServices; + logger: Logger; + id: string; + signalsIndex: string; + name: string; + createdBy: string; + updatedBy: string; + interval: string; + enabled: boolean; +} + // Bulk Index documents. -export const singleBulkIndex = async ( - sr: SignalSearchResponse, - params: SignalAlertParams, - service: AlertServices, - logger: Logger, - id: string, - signalsIndex: string -): Promise => { - if (sr.hits.hits.length === 0) { +export const singleBulkIndex = async ({ + someResult, + signalParams, + services, + logger, + id, + signalsIndex, + name, + createdBy, + updatedBy, + interval, + enabled, +}: SingleBulkIndexParams): Promise => { + if (someResult.hits.hits.length === 0) { return true; } - const bulkBody = sr.hits.hits.flatMap(doc => [ + const bulkBody = someResult.hits.hits.flatMap(doc => [ { index: { _index: signalsIndex, _id: doc._id, }, }, - buildBulkBody(doc, params, id), + buildBulkBody({ doc, signalParams, id, name, createdBy, updatedBy, interval, enabled }), ]); const time1 = performance.now(); - const firstResult: BulkResponse = await service.callCluster('bulk', { + const firstResult: BulkResponse = await services.callCluster('bulk', { index: signalsIndex, refresh: false, body: bulkBody, @@ -77,26 +176,33 @@ export const singleBulkIndex = async ( return true; }; +interface SingleSearchAfterParams { + searchAfterSortId: string | undefined; + signalParams: AlertTypeParams; + services: AlertServices; + logger: Logger; +} + // utilize search_after for paging results into bulk. -export const singleSearchAfter = async ( - searchAfterSortId: string | undefined, - params: SignalAlertParams, - service: AlertServices, - logger: Logger -): Promise => { +export const singleSearchAfter = async ({ + searchAfterSortId, + signalParams, + services, + logger, +}: SingleSearchAfterParams): Promise => { if (searchAfterSortId == null) { throw Error('Attempted to search after with empty sort id'); } try { const searchAfterQuery = buildEventsSearchQuery({ - index: params.index, - from: params.from, - to: params.to, - filter: params.filter, - size: params.size ? params.size : 1000, + index: signalParams.index, + from: signalParams.from, + to: signalParams.to, + filter: signalParams.filter, + size: signalParams.size ? signalParams.size : 1000, searchAfterSortId, }); - const nextSearchAfterResult: SignalSearchResponse = await service.callCluster( + const nextSearchAfterResult: SignalSearchResponse = await services.callCluster( 'search', searchAfterQuery ); @@ -107,28 +213,52 @@ export const singleSearchAfter = async ( } }; +interface SearchAfterAndBulkIndexParams { + someResult: SignalSearchResponse; + signalParams: AlertTypeParams; + services: AlertServices; + logger: Logger; + id: string; + signalsIndex: string; + name: string; + createdBy: string; + updatedBy: string; + interval: string; + enabled: boolean; +} + // search_after through documents and re-index using bulk endpoint. -export const searchAfterAndBulkIndex = async ( - someResult: SignalSearchResponse, - params: SignalAlertParams, - service: AlertServices, - logger: Logger, - id: string, - signalsIndex: string -): Promise => { +export const searchAfterAndBulkIndex = async ({ + someResult, + signalParams, + services, + logger, + id, + signalsIndex, + name, + createdBy, + updatedBy, + interval, + enabled, +}: SearchAfterAndBulkIndexParams): Promise => { if (someResult.hits.hits.length === 0) { return true; } logger.debug('[+] starting bulk insertion'); - const firstBulkIndexSuccess = await singleBulkIndex( + const firstBulkIndexSuccess = await singleBulkIndex({ someResult, - params, - service, + signalParams, + services, logger, id, - signalsIndex - ); + signalsIndex, + name, + createdBy, + updatedBy, + interval, + enabled, + }); if (!firstBulkIndexSuccess) { logger.error('First bulk index was unsuccessful'); return false; @@ -140,7 +270,7 @@ export const searchAfterAndBulkIndex = async ( // query for. If maxSignals is present we will only query // up to max signals - otherwise use the value // from track_total_hits. - const maxTotalHitsSize = params.maxSignals ? params.maxSignals : totalHits; + const maxTotalHitsSize = signalParams.maxSignals ? signalParams.maxSignals : totalHits; // number of docs in the current search result let hitsSize = someResult.hits.hits.length; @@ -159,12 +289,12 @@ export const searchAfterAndBulkIndex = async ( while (hitsSize < maxTotalHitsSize && hitsSize !== 0) { try { logger.debug(`sortIds: ${sortIds}`); - const searchAfterResult: SignalSearchResponse = await singleSearchAfter( - sortId, - params, - service, - logger - ); + const searchAfterResult: SignalSearchResponse = await singleSearchAfter({ + searchAfterSortId: sortId, + signalParams, + services, + logger, + }); if (searchAfterResult.hits.hits.length === 0) { return true; } @@ -177,14 +307,19 @@ export const searchAfterAndBulkIndex = async ( } sortId = sortIds[0]; logger.debug('next bulk index'); - const bulkSuccess = await singleBulkIndex( - searchAfterResult, - params, - service, + const bulkSuccess = await singleBulkIndex({ + someResult: searchAfterResult, + signalParams, + services, logger, id, - signalsIndex - ); + signalsIndex, + name, + createdBy, + updatedBy, + interval, + enabled, + }); logger.debug('finished next bulk index'); if (!bulkSuccess) { logger.error('[-] bulk index failed but continuing'); diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals_mapping.json b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals_mapping.json index dd80e786a3121..a95c9625a0003 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals_mapping.json +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals_mapping.json @@ -23,38 +23,90 @@ } } }, - "id": { - "type": "keyword" + "rule": { + "properties": { + "id": { + "type": "keyword" + }, + "rule_id": { + "type": "keyword" + }, + "false_positives": { + "type": "keyword" + }, + "saved_id": { + "type": "keyword" + }, + "max_signals": { + "type": "keyword" + }, + "risk_score": { + "type": "keyword" + }, + "output_index": { + "type": "keyword" + }, + "description": { + "type": "keyword" + }, + "filter": { + "type": "object" + }, + "from": { + "type": "keyword" + }, + "immutable": { + "type": "keyword" + }, + "index": { + "type": "keyword" + }, + "interval": { + "type": "keyword" + }, + "language": { + "type": "keyword" + }, + "name": { + "type": "keyword" + }, + "query": { + "type": "keyword" + }, + "references": { + "type": "keyword" + }, + "severity": { + "type": "keyword" + }, + "tags": { + "type": "keyword" + }, + "type": { + "type": "keyword" + }, + "size": { + "type": "keyword" + }, + "to": { + "type": "keyword" + }, + "enabled": { + "type": "keyword" + }, + "filters": { + "type": "object" + }, + "created_by": { + "type": "keyword" + }, + "updated_by": { + "type": "keyword" + } + } }, "original_time": { "type": "date" - }, - "rule_revision": { - "type": "long" - }, - "rule_id": { - "type": "keyword" - }, - "rule_type": { - "type": "keyword" - }, - "rule_query": { - "type": "keyword" - }, - "index_patterns": { - "type": "keyword" - }, - "name": { - "type": "keyword" - }, - "description": { - "type": "text" - }, - "severity": { - "type": "keyword" - }, - "references": { - "type": "text" } } }, diff --git a/x-pack/legacy/plugins/siem/server/lib/types.ts b/x-pack/legacy/plugins/siem/server/lib/types.ts index 2769199ad1fb5..a5429ebf76517 100644 --- a/x-pack/legacy/plugins/siem/server/lib/types.ts +++ b/x-pack/legacy/plugins/siem/server/lib/types.ts @@ -22,6 +22,7 @@ import { Note } from './note/saved_object'; import { PinnedEvent } from './pinned_event/saved_object'; import { Timeline } from './timeline/saved_object'; import { TLS } from './tls'; +import { SignalAlertParamsRest } from './detection_engine/alerts/types'; export * from './hosts'; @@ -62,26 +63,20 @@ export interface SiemContext { req: FrameworkRequest; } -export interface SignalHit { - signal: { - '@timestamp': string; +export interface Signal { + rule: Partial; + parent: { id: string; - rule_revision: number; - rule_id: string | undefined | null; - rule_type: string; - parent: { - id: string; - type: string; - index: string; - depth: number; - }; - name: string; - severity: string; - description: string; - original_time: string; - index_patterns: string[]; - references: string[]; + type: string; + index: string; + depth: number; }; + original_time: string; +} + +export interface SignalHit { + '@timestamp': string; + signal: Partial; } export interface TotalValue {