diff --git a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.mock.ts b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.mock.ts new file mode 100644 index 0000000000000..86ef14e491f4e --- /dev/null +++ b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.mock.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { alertsMock } from '../../../alerting/server/mocks'; +import { PersistenceServices } from './persistence_types'; + +export const createPersistenceServicesMock = (): jest.Mocked => { + return { + alertWithPersistence: jest.fn(), + }; +}; + +export const createPersistenceExecutorOptionsMock = () => { + return { + ...alertsMock.createAlertServices(), + ...createPersistenceServicesMock(), + }; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_wrapper.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_wrapper.ts index 38300dff14558..f25bb16e90004 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_wrapper.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_wrapper.ts @@ -8,7 +8,6 @@ import { isEmpty } from 'lodash'; import { parseScheduleDates } from '@kbn/securitysolution-io-ts-utils'; -import { ListArray } from '@kbn/securitysolution-io-ts-list-types'; import agent from 'elastic-apm-node'; import { createPersistenceRuleTypeWrapper } from '../../../../../rule_registry/server'; @@ -19,11 +18,7 @@ import { getRuleRangeTuples, hasReadIndexPrivileges, hasTimestampFields, - isEqlParams, - isQueryParams, - isSavedQueryParams, - isThreatParams, - isThresholdParams, + isMachineLearningParams, } from '../signals/utils'; import { DEFAULT_MAX_SIGNALS, DEFAULT_SEARCH_AFTER_PAGE_SIZE } from '../../../../common/constants'; import { CreateSecurityRuleTypeWrapper } from './types'; @@ -133,22 +128,13 @@ export const createSecurityRuleTypeWrapper: CreateSecurityRuleTypeWrapper = ...params, name, id: alertId, - } as unknown as NotificationRuleTypeParams; + }; // check if rule has permissions to access given index pattern // move this collection of lines into a function in utils // so that we can use it in create rules route, bulk, etc. try { - // Typescript 4.1.3 can't figure out that `!isMachineLearningParams(params)` also excludes the only rule type - // of rule params that doesn't include `params.index`, but Typescript 4.3.5 does compute the stricter type correctly. - // When we update Typescript to >= 4.3.5, we can replace this logic with `!isMachineLearningParams(params)` again. - if ( - isEqlParams(params) || - isThresholdParams(params) || - isQueryParams(params) || - isSavedQueryParams(params) || - isThreatParams(params) - ) { + if (!isMachineLearningParams(params)) { const index = params.index; const hasTimestampOverride = !!timestampOverride; @@ -170,7 +156,7 @@ export const createSecurityRuleTypeWrapper: CreateSecurityRuleTypeWrapper = { index, fields: hasTimestampOverride - ? ['@timestamp', timestampOverride as string] + ? ['@timestamp', timestampOverride] : ['@timestamp'], include_unmapped: true, }, @@ -178,9 +164,7 @@ export const createSecurityRuleTypeWrapper: CreateSecurityRuleTypeWrapper = ) ); wroteWarningStatus = await hasTimestampFields({ - timestampField: hasTimestampOverride - ? (timestampOverride as string) - : '@timestamp', + timestampField: hasTimestampOverride ? timestampOverride : '@timestamp', timestampFieldCapsResponse: timestampFieldCaps, inputIndices, ruleExecutionLogger, @@ -202,8 +186,8 @@ export const createSecurityRuleTypeWrapper: CreateSecurityRuleTypeWrapper = const { tuples, remainingGap } = getRuleRangeTuples({ logger, previousStartedAt, - from: from as string, - to: to as string, + from, + to, interval, maxSignals: maxSignals ?? DEFAULT_MAX_SIGNALS, buildRuleMessage, @@ -236,7 +220,7 @@ export const createSecurityRuleTypeWrapper: CreateSecurityRuleTypeWrapper = const exceptionItems = await getExceptions({ client: exceptionsClient, - lists: (params.exceptionsList as ListArray) ?? [], + lists: params.exceptionsList, }); const bulkCreate = bulkCreateFactory( diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert_group_from_sequence.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert_group_from_sequence.test.ts index 969f7caab6456..b1b68829665fc 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert_group_from_sequence.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert_group_from_sequence.test.ts @@ -10,7 +10,11 @@ import { Logger } from 'kibana/server'; import { ALERT_RULE_CONSUMER } from '@kbn/rule-data-utils'; import { sampleDocNoSortId, sampleRuleGuid } from '../../../signals/__mocks__/es_results'; -import { buildAlertGroupFromSequence } from './build_alert_group_from_sequence'; +import { + buildAlertGroupFromSequence, + objectArrayIntersection, + objectPairIntersection, +} from './build_alert_group_from_sequence'; import { SERVER_APP_ID } from '../../../../../../common/constants'; import { getCompleteRuleMock, getQueryRuleParams } from '../../../schemas/rule_schemas.mock'; import { QueryRuleParams } from '../../../schemas/rule_schemas'; @@ -134,4 +138,342 @@ describe('buildAlert', () => { expect(groupId).toEqual(groupIds[0]); } }); + + describe('recursive intersection between objects', () => { + test('should treat numbers and strings as unequal', () => { + const a = { + field1: 1, + field2: 1, + }; + const b = { + field1: 1, + field2: '1', + }; + const intersection = objectPairIntersection(a, b); + const expected = { + field1: 1, + }; + expect(intersection).toEqual(expected); + }); + + test('should strip unequal numbers and strings', () => { + const a = { + field1: 1, + field2: 1, + field3: 'abcd', + field4: 'abcd', + }; + const b = { + field1: 1, + field2: 100, + field3: 'abcd', + field4: 'wxyz', + }; + const intersection = objectPairIntersection(a, b); + const expected = { + field1: 1, + field3: 'abcd', + }; + expect(intersection).toEqual(expected); + }); + + test('should handle null values', () => { + const a = { + field1: 1, + field2: '1', + field3: null, + }; + const b = { + field1: null, + field2: null, + field3: null, + }; + const intersection = objectPairIntersection(a, b); + const expected = { + field3: null, + }; + expect(intersection).toEqual(expected); + }); + + test('should handle explicit undefined values and return undefined if left with only undefined fields', () => { + const a = { + field1: 1, + field2: '1', + field3: undefined, + }; + const b = { + field1: undefined, + field2: undefined, + field3: undefined, + }; + const intersection = objectPairIntersection(a, b); + const expected = undefined; + expect(intersection).toEqual(expected); + }); + + test('should strip arrays out regardless of whether they are equal', () => { + const a = { + array_field1: [1, 2], + array_field2: [1, 2], + }; + const b = { + array_field1: [1, 2], + array_field2: [3, 4], + }; + const intersection = objectPairIntersection(a, b); + const expected = undefined; + expect(intersection).toEqual(expected); + }); + + test('should strip fields that are not in both objects', () => { + const a = { + field1: 1, + }; + const b = { + field2: 1, + }; + const intersection = objectPairIntersection(a, b); + const expected = undefined; + expect(intersection).toEqual(expected); + }); + + test('should work on objects within objects', () => { + const a = { + container_field: { + field1: 1, + field2: 1, + field3: 10, + field5: 1, + field6: null, + array_field: [1, 2], + nested_container_field: { + field1: 1, + field2: 1, + }, + nested_container_field2: { + field1: undefined, + }, + }, + container_field_without_intersection: { + sub_field1: 1, + }, + }; + const b = { + container_field: { + field1: 1, + field2: 2, + field4: 10, + field5: '1', + field6: null, + array_field: [1, 2], + nested_container_field: { + field1: 1, + field2: 2, + }, + nested_container_field2: { + field1: undefined, + }, + }, + container_field_without_intersection: { + sub_field2: 1, + }, + }; + const intersection = objectPairIntersection(a, b); + const expected = { + container_field: { + field1: 1, + field6: null, + nested_container_field: { + field1: 1, + }, + }, + }; + expect(intersection).toEqual(expected); + }); + + test('should work on objects with a variety of fields', () => { + const a = { + field1: 1, + field2: 1, + field3: 10, + field5: 1, + field6: null, + array_field: [1, 2], + container_field: { + sub_field1: 1, + sub_field2: 1, + sub_field3: 10, + }, + container_field_without_intersection: { + sub_field1: 1, + }, + }; + const b = { + field1: 1, + field2: 2, + field4: 10, + field5: '1', + field6: null, + array_field: [1, 2], + container_field: { + sub_field1: 1, + sub_field2: 2, + sub_field4: 10, + }, + container_field_without_intersection: { + sub_field2: 1, + }, + }; + const intersection = objectPairIntersection(a, b); + const expected = { + field1: 1, + field6: null, + container_field: { + sub_field1: 1, + }, + }; + expect(intersection).toEqual(expected); + }); + }); + + describe('objectArrayIntersection', () => { + test('should return undefined if the array is empty', () => { + const intersection = objectArrayIntersection([]); + const expected = undefined; + expect(intersection).toEqual(expected); + }); + test('should return the initial object if there is only 1', () => { + const a = { + field1: 1, + field2: 1, + field3: 10, + field5: 1, + field6: null, + array_field: [1, 2], + container_field: { + sub_field1: 1, + sub_field2: 1, + sub_field3: 10, + }, + container_field_without_intersection: { + sub_field1: 1, + }, + }; + const intersection = objectArrayIntersection([a]); + const expected = { + field1: 1, + field2: 1, + field3: 10, + field5: 1, + field6: null, + array_field: [1, 2], + container_field: { + sub_field1: 1, + sub_field2: 1, + sub_field3: 10, + }, + container_field_without_intersection: { + sub_field1: 1, + }, + }; + expect(intersection).toEqual(expected); + }); + test('should work with exactly 2 objects', () => { + const a = { + field1: 1, + field2: 1, + field3: 10, + field5: 1, + field6: null, + array_field: [1, 2], + container_field: { + sub_field1: 1, + sub_field2: 1, + sub_field3: 10, + }, + container_field_without_intersection: { + sub_field1: 1, + }, + }; + const b = { + field1: 1, + field2: 2, + field4: 10, + field5: '1', + field6: null, + array_field: [1, 2], + container_field: { + sub_field1: 1, + sub_field2: 2, + sub_field4: 10, + }, + container_field_without_intersection: { + sub_field2: 1, + }, + }; + const intersection = objectArrayIntersection([a, b]); + const expected = { + field1: 1, + field6: null, + container_field: { + sub_field1: 1, + }, + }; + expect(intersection).toEqual(expected); + }); + + test('should work with 3 or more objects', () => { + const a = { + field1: 1, + field2: 1, + field3: 10, + field5: 1, + field6: null, + array_field: [1, 2], + container_field: { + sub_field1: 1, + sub_field2: 1, + sub_field3: 10, + }, + container_field_without_intersection: { + sub_field1: 1, + }, + }; + const b = { + field1: 1, + field2: 2, + field4: 10, + field5: '1', + field6: null, + array_field: [1, 2], + container_field: { + sub_field1: 1, + sub_field2: 2, + sub_field4: 10, + }, + container_field_without_intersection: { + sub_field2: 1, + }, + }; + const c = { + field1: 1, + field2: 2, + field4: 10, + field5: '1', + array_field: [1, 2], + container_field: { + sub_field2: 2, + sub_field4: 10, + }, + container_field_without_intersection: { + sub_field2: 1, + }, + }; + const intersection = objectArrayIntersection([a, b, c]); + const expected = { + field1: 1, + }; + expect(intersection).toEqual(expected); + }); + }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert_group_from_sequence.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert_group_from_sequence.ts index 180494f9209dd..26e0289732bfb 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert_group_from_sequence.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert_group_from_sequence.ts @@ -16,7 +16,6 @@ import { buildAlert, buildAncestors, generateAlertId } from './build_alert'; import { buildBulkBody } from './build_bulk_body'; import { EqlSequence } from '../../../../../../common/detection_engine/types'; import { generateBuildingBlockIds } from './generate_building_block_ids'; -import { objectArrayIntersection } from '../../../signals/build_bulk_body'; import { BuildReasonMessage } from '../../../signals/reason_formatters'; import { CompleteRule, RuleParams } from '../../../schemas/rule_schemas'; import { @@ -118,3 +117,54 @@ export const buildAlertRoot = ( [ALERT_GROUP_ID]: generateAlertId(doc), }; }; + +export const objectArrayIntersection = (objects: object[]) => { + if (objects.length === 0) { + return undefined; + } else if (objects.length === 1) { + return objects[0]; + } else { + return objects + .slice(1) + .reduce( + (acc: object | undefined, obj): object | undefined => objectPairIntersection(acc, obj), + objects[0] + ); + } +}; + +export const objectPairIntersection = (a: object | undefined, b: object | undefined) => { + if (a === undefined || b === undefined) { + return undefined; + } + const intersection: Record = {}; + Object.entries(a).forEach(([key, aVal]) => { + if (key in b) { + const bVal = (b as Record)[key]; + if ( + typeof aVal === 'object' && + !(aVal instanceof Array) && + aVal !== null && + typeof bVal === 'object' && + !(bVal instanceof Array) && + bVal !== null + ) { + intersection[key] = objectPairIntersection(aVal, bVal); + } else if (aVal === bVal) { + intersection[key] = aVal; + } + } + }); + // Count up the number of entries that are NOT undefined in the intersection + // If there are no keys OR all entries are undefined, return undefined + if ( + Object.values(intersection).reduce( + (acc: number, value) => (value !== undefined ? acc + 1 : acc), + 0 + ) === 0 + ) { + return undefined; + } else { + return intersection; + } +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts deleted file mode 100644 index 21bfced47df42..0000000000000 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts +++ /dev/null @@ -1,251 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { TIMESTAMP } from '@kbn/rule-data-utils'; -import { getMergeStrategy } from './source_fields_merging/strategies'; -import { - SignalSourceHit, - SignalHit, - Signal, - BaseSignalHit, - SignalSource, - WrappedSignalHit, -} from './types'; -import { buildRuleWithoutOverrides, buildRuleWithOverrides } from './build_rule'; -import { additionalSignalFields, buildSignal } from './build_signal'; -import { buildEventTypeSignal } from './build_event_type_signal'; -import { EqlSequence } from '../../../../common/detection_engine/types'; -import { generateSignalId, wrapBuildingBlocks, wrapSignal } from './utils'; -import type { ConfigType } from '../../../config'; -import { BuildReasonMessage } from './reason_formatters'; -import { CompleteRule, RuleParams } from '../schemas/rule_schemas'; - -/** - * Formats the search_after result for insertion into the signals index. We first create a - * "best effort" merged "fields" with the "_source" object, then build the signal object, - * then the event object, and finally we strip away any additional temporary data that was added - * such as the "threshold_result". - * @param completeRule The rule object to build overrides - * @param doc The SignalSourceHit with "_source", "fields", and additional data such as "threshold_result" - * @returns The body that can be added to a bulk call for inserting the signal. - */ -export const buildBulkBody = ( - completeRule: CompleteRule, - doc: SignalSourceHit, - mergeStrategy: ConfigType['alertMergeStrategy'], - ignoreFields: ConfigType['alertIgnoreFields'], - buildReasonMessage: BuildReasonMessage -): SignalHit => { - const mergedDoc = getMergeStrategy(mergeStrategy)({ doc, ignoreFields }); - const rule = buildRuleWithOverrides(completeRule, mergedDoc._source ?? {}); - const timestamp = new Date().toISOString(); - const reason = buildReasonMessage({ - name: completeRule.ruleConfig.name, - severity: completeRule.ruleParams.severity, - mergedDoc, - }); - const signal: Signal = { - ...buildSignal([mergedDoc], rule, reason), - ...additionalSignalFields(mergedDoc), - }; - const event = buildEventTypeSignal(mergedDoc); - // Filter out any kibana.* fields from the generated signal - kibana.* fields are aliases - // in siem-signals so we can't write to them, but for signals-on-signals they'll be returned - // in the fields API response and merged into the mergedDoc source - const { - threshold_result: thresholdResult, - kibana, - ...filteredSource - } = mergedDoc._source || { - threshold_result: null, - }; - const signalHit: SignalHit = { - ...filteredSource, - [TIMESTAMP]: timestamp, - event, - signal, - }; - return signalHit; -}; - -/** - * Takes N raw documents from ES that form a sequence and builds them into N+1 signals ready to be indexed - - * one signal for each event in the sequence, and a "shell" signal that ties them all together. All N+1 signals - * share the same signal.group.id to make it easy to query them. - * @param sequence The raw ES documents that make up the sequence - * @param completeRule rule object representing the rule that found the sequence - * @param outputIndex Index to write the resulting signals to - */ -export const buildSignalGroupFromSequence = ( - sequence: EqlSequence, - completeRule: CompleteRule, - outputIndex: string, - mergeStrategy: ConfigType['alertMergeStrategy'], - ignoreFields: ConfigType['alertIgnoreFields'], - buildReasonMessage: BuildReasonMessage -): WrappedSignalHit[] => { - const wrappedBuildingBlocks = wrapBuildingBlocks( - sequence.events.map((event) => { - const signal = buildSignalFromEvent( - event, - completeRule, - false, - mergeStrategy, - ignoreFields, - buildReasonMessage - ); - signal.signal.rule.building_block_type = 'default'; - return signal; - }), - outputIndex - ); - - if ( - wrappedBuildingBlocks.some((block) => - block._source.signal?.ancestors.some((ancestor) => ancestor.rule === completeRule.alertId) - ) - ) { - return []; - } - - // Now that we have an array of building blocks for the events in the sequence, - // we can build the signal that links the building blocks together - // and also insert the group id (which is also the "shell" signal _id) in each building block - const sequenceSignal = wrapSignal( - buildSignalFromSequence(wrappedBuildingBlocks, completeRule, buildReasonMessage), - outputIndex - ); - wrappedBuildingBlocks.forEach((block, idx) => { - // TODO: fix type of blocks so we don't have to check existence of _source.signal - if (block._source.signal) { - block._source.signal.group = { - id: sequenceSignal._id, - index: idx, - }; - } - }); - return [...wrappedBuildingBlocks, sequenceSignal]; -}; - -export const buildSignalFromSequence = ( - events: WrappedSignalHit[], - completeRule: CompleteRule, - buildReasonMessage: BuildReasonMessage -): SignalHit => { - const rule = buildRuleWithoutOverrides(completeRule); - const timestamp = new Date().toISOString(); - const mergedEvents = objectArrayIntersection(events.map((event) => event._source)); - const reason = buildReasonMessage({ - name: completeRule.ruleConfig.name, - severity: completeRule.ruleParams.severity, - mergedDoc: mergedEvents as SignalSourceHit, - }); - const signal: Signal = buildSignal(events, rule, reason); - return { - ...mergedEvents, - [TIMESTAMP]: timestamp, - event: { - kind: 'signal', - }, - signal: { - ...signal, - group: { - // This is the same function that is used later to generate the _id for the sequence signal document, - // so _id should equal signal.group.id for the "shell" document - id: generateSignalId(signal), - }, - }, - }; -}; - -export const buildSignalFromEvent = ( - event: BaseSignalHit, - completeRule: CompleteRule, - applyOverrides: boolean, - mergeStrategy: ConfigType['alertMergeStrategy'], - ignoreFields: ConfigType['alertIgnoreFields'], - buildReasonMessage: BuildReasonMessage -): SignalHit => { - const mergedEvent = getMergeStrategy(mergeStrategy)({ doc: event, ignoreFields }); - const rule = applyOverrides - ? buildRuleWithOverrides(completeRule, mergedEvent._source ?? {}) - : buildRuleWithoutOverrides(completeRule); - const timestamp = new Date().toISOString(); - const reason = buildReasonMessage({ - name: completeRule.ruleConfig.name, - severity: completeRule.ruleParams.severity, - mergedDoc: mergedEvent, - }); - const signal: Signal = { - ...buildSignal([mergedEvent], rule, reason), - ...additionalSignalFields(mergedEvent), - }; - const eventFields = buildEventTypeSignal(mergedEvent); - // Filter out any kibana.* fields from the generated signal - kibana.* fields are aliases - // in siem-signals so we can't write to them, but for signals-on-signals they'll be returned - // in the fields API response and merged into the mergedDoc source - const { kibana, ...filteredSource } = mergedEvent._source || {}; - // TODO: better naming for SignalHit - it's really a new signal to be inserted - const signalHit: SignalHit = { - ...filteredSource, - [TIMESTAMP]: timestamp, - event: eventFields, - signal, - }; - return signalHit; -}; - -export const objectArrayIntersection = (objects: object[]) => { - if (objects.length === 0) { - return undefined; - } else if (objects.length === 1) { - return objects[0]; - } else { - return objects - .slice(1) - .reduce( - (acc: object | undefined, obj): object | undefined => objectPairIntersection(acc, obj), - objects[0] - ); - } -}; - -export const objectPairIntersection = (a: object | undefined, b: object | undefined) => { - if (a === undefined || b === undefined) { - return undefined; - } - const intersection: Record = {}; - Object.entries(a).forEach(([key, aVal]) => { - if (key in b) { - const bVal = (b as Record)[key]; - if ( - typeof aVal === 'object' && - !(aVal instanceof Array) && - aVal !== null && - typeof bVal === 'object' && - !(bVal instanceof Array) && - bVal !== null - ) { - intersection[key] = objectPairIntersection(aVal, bVal); - } else if (aVal === bVal) { - intersection[key] = aVal; - } - } - }); - // Count up the number of entries that are NOT undefined in the intersection - // If there are no keys OR all entries are undefined, return undefined - if ( - Object.values(intersection).reduce( - (acc: number, value) => (value !== undefined ? acc + 1 : acc), - 0 - ) === 0 - ) { - return undefined; - } else { - return intersection; - } -}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_event_type_signal.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_event_type_signal.test.ts deleted file mode 100644 index cc3456e7ab968..0000000000000 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_event_type_signal.test.ts +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { sampleDocNoSortId } from './__mocks__/es_results'; -import { buildEventTypeSignal, isEventTypeSignal } from './build_event_type_signal'; - -describe('buildEventTypeSignal', () => { - beforeEach(() => { - jest.clearAllMocks(); - }); - - test('it returns the event appended of kind signal if it does not exist', () => { - const doc = sampleDocNoSortId(); - delete doc._source.event; - const eventType = buildEventTypeSignal(doc); - const expected: object = { kind: 'signal' }; - expect(eventType).toEqual(expected); - }); - - test('it returns the event appended of kind signal if it is an empty object', () => { - const doc = sampleDocNoSortId(); - doc._source.event = {}; - const eventType = buildEventTypeSignal(doc); - const expected: object = { kind: 'signal' }; - expect(eventType).toEqual(expected); - }); - - test('it returns the event with kind signal and other properties if they exist', () => { - const doc = sampleDocNoSortId(); - doc._source.event = { - action: 'socket_opened', - module: 'system', - dataset: 'socket', - }; - const eventType = buildEventTypeSignal(doc); - const expected: object = { - action: 'socket_opened', - module: 'system', - dataset: 'socket', - kind: 'signal', - }; - expect(eventType).toEqual(expected); - }); - - test('It validates a sample doc with no signal type as "false"', () => { - const doc = sampleDocNoSortId(); - expect(isEventTypeSignal(doc)).toEqual(false); - }); - - test('It validates a sample doc with a signal type as "true"', () => { - const doc = { - ...sampleDocNoSortId(), - _source: { - ...sampleDocNoSortId()._source, - signal: { - rule: { id: 'id-123' }, - }, - }, - }; - expect(isEventTypeSignal(doc)).toEqual(true); - }); - - test('It validates a numeric signal string as "false"', () => { - const doc = { - ...sampleDocNoSortId(), - _source: { - ...sampleDocNoSortId()._source, - signal: 'something', - }, - }; - expect(isEventTypeSignal(doc)).toEqual(false); - }); - - test('It validates an empty object as "false"', () => { - const doc = { - ...sampleDocNoSortId(), - _source: { - ...sampleDocNoSortId()._source, - signal: {}, - }, - }; - expect(isEventTypeSignal(doc)).toEqual(false); - }); - - test('It validates an empty rule object as "false"', () => { - const doc = { - ...sampleDocNoSortId(), - _source: { - ...sampleDocNoSortId()._source, - signal: { - rule: {}, - }, - }, - }; - expect(isEventTypeSignal(doc)).toEqual(false); - }); -}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_event_type_signal.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_event_type_signal.ts deleted file mode 100644 index 0dd2acfb88ffe..0000000000000 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_event_type_signal.ts +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -import { BaseSignalHit, SimpleHit } from './types'; -import { getField } from './utils'; - -export const buildEventTypeSignal = (doc: BaseSignalHit): object => { - if (doc._source != null && doc._source.event instanceof Object) { - return { ...doc._source.event, kind: 'signal' }; - } else { - return { kind: 'signal' }; - } -}; - -/** - * Given a document this will return true if that document is a signal - * document. We can't guarantee the code will call this function with a document - * before adding the _source.event.kind = "signal" from "buildEventTypeSignal" - * so we do basic testing to ensure that if the object has the fields of: - * "signal.rule.id" then it will be one of our signals rather than a customer - * overwritten signal. - * @param doc The document which might be a signal or it might be a regular log - */ -export const isEventTypeSignal = (doc: SimpleHit): boolean => { - const ruleId = getField(doc, 'signal.rule.id'); - return ruleId != null && typeof ruleId === 'string'; -}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.test.ts deleted file mode 100644 index 9ae51688ee676..0000000000000 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.test.ts +++ /dev/null @@ -1,189 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { buildRuleWithOverrides, buildRuleWithoutOverrides } from './build_rule'; -import { sampleDocNoSortId, expectedRule, sampleDocSeverity } from './__mocks__/es_results'; -import { RulesSchema } from '../../../../common/detection_engine/schemas/response/rules_schema'; -import { INTERNAL_RULE_ID_KEY, INTERNAL_IMMUTABLE_KEY } from '../../../../common/constants'; -import { - getCompleteRuleMock, - getQueryRuleParams, - getThreatRuleParams, -} from '../schemas/rule_schemas.mock'; -import { - CompleteRule, - QueryRuleParams, - RuleParams, - ThreatRuleParams, -} from '../schemas/rule_schemas'; - -describe('buildRuleWithoutOverrides', () => { - let params: RuleParams; - let completeRule: CompleteRule; - - beforeEach(() => { - params = getQueryRuleParams(); - completeRule = getCompleteRuleMock(params); - }); - - test('builds a rule using rule alert', () => { - const rule = buildRuleWithoutOverrides(completeRule); - expect(rule).toEqual(expectedRule()); - }); - - test('builds a rule and removes internal tags', () => { - completeRule.ruleConfig.tags = [ - 'some fake tag 1', - 'some fake tag 2', - `${INTERNAL_RULE_ID_KEY}:rule-1`, - `${INTERNAL_IMMUTABLE_KEY}:true`, - ]; - const rule = buildRuleWithoutOverrides(completeRule); - expect(rule.tags).toEqual(['some fake tag 1', 'some fake tag 2']); - }); - - test('it builds a rule as expected with filters present', () => { - const ruleFilters = [ - { - query: 'host.name: Rebecca', - }, - { - query: 'host.name: Evan', - }, - { - query: 'host.name: Braden', - }, - ]; - completeRule.ruleParams.filters = ruleFilters; - const rule = buildRuleWithoutOverrides(completeRule); - expect(rule.filters).toEqual(ruleFilters); - }); - - test('it creates a indicator/threat_mapping/threat_matching rule', () => { - const ruleParams: ThreatRuleParams = { - ...getThreatRuleParams(), - threatMapping: [ - { - entries: [ - { - field: 'host.name', - value: 'host.name', - type: 'mapping', - }, - ], - }, - ], - threatFilters: [ - { - query: { - bool: { - must: [ - { - query_string: { - query: 'host.name: linux', - analyze_wildcard: true, - time_zone: 'Zulu', - }, - }, - ], - }, - }, - }, - ], - threatIndicatorPath: 'some.path', - threatQuery: 'threat_query', - threatIndex: ['threat_index'], - threatLanguage: 'kuery', - }; - const threatMatchCompleteRule = getCompleteRuleMock(ruleParams); - const threatMatchRule = buildRuleWithoutOverrides(threatMatchCompleteRule); - const expected: Partial = { - threat_mapping: ruleParams.threatMapping, - threat_filters: ruleParams.threatFilters, - threat_indicator_path: ruleParams.threatIndicatorPath, - threat_query: ruleParams.threatQuery, - threat_index: ruleParams.threatIndex, - threat_language: ruleParams.threatLanguage, - }; - expect(threatMatchRule).toEqual(expect.objectContaining(expected)); - }); -}); - -describe('buildRuleWithOverrides', () => { - let params: RuleParams; - let completeRule: CompleteRule; - - beforeEach(() => { - params = getQueryRuleParams(); - completeRule = getCompleteRuleMock(params); - }); - - test('it applies rule name override in buildRule', () => { - completeRule.ruleParams.ruleNameOverride = 'someKey'; - const rule = buildRuleWithOverrides(completeRule, sampleDocNoSortId()._source!); - const expected = { - ...expectedRule(), - name: 'someValue', - rule_name_override: 'someKey', - meta: { - ruleNameOverridden: true, - someMeta: 'someField', - }, - }; - expect(rule).toEqual(expected); - }); - - test('it applies risk score override in buildRule', () => { - const newRiskScore = 79; - completeRule.ruleParams.riskScoreMapping = [ - { - field: 'new_risk_score', - // value and risk_score aren't used for anything but are required in the schema - value: '', - operator: 'equals', - risk_score: undefined, - }, - ]; - const doc = sampleDocNoSortId(); - doc._source.new_risk_score = newRiskScore; - const rule = buildRuleWithOverrides(completeRule, doc._source!); - const expected = { - ...expectedRule(), - risk_score: newRiskScore, - risk_score_mapping: completeRule.ruleParams.riskScoreMapping, - meta: { - riskScoreOverridden: true, - someMeta: 'someField', - }, - }; - expect(rule).toEqual(expected); - }); - - test('it applies severity override in buildRule', () => { - const eventSeverity = '42'; - completeRule.ruleParams.severityMapping = [ - { - field: 'event.severity', - value: eventSeverity, - operator: 'equals', - severity: 'critical', - }, - ]; - const doc = sampleDocSeverity(Number(eventSeverity)); - const rule = buildRuleWithOverrides(completeRule, doc._source!); - const expected = { - ...expectedRule(), - severity: 'critical', - severity_mapping: completeRule.ruleParams.severityMapping, - meta: { - severityOverrideField: 'event.severity', - someMeta: 'someField', - }, - }; - expect(rule).toEqual(expected); - }); -}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.ts deleted file mode 100644 index ab40ce330370c..0000000000000 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.ts +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { RulesSchema } from '../../../../common/detection_engine/schemas/response/rules_schema'; -import { buildRiskScoreFromMapping } from './mappings/build_risk_score_from_mapping'; -import { SignalSource } from './types'; -import { buildSeverityFromMapping } from './mappings/build_severity_from_mapping'; -import { buildRuleNameFromMapping } from './mappings/build_rule_name_from_mapping'; -import { CompleteRule, RuleParams } from '../schemas/rule_schemas'; -import { commonParamsCamelToSnake, typeSpecificCamelToSnake } from '../schemas/rule_converters'; -import { transformTags } from '../routes/rules/utils'; -import { transformAlertToRuleAction } from '../../../../common/detection_engine/transform_actions'; - -export const buildRuleWithoutOverrides = (completeRule: CompleteRule): RulesSchema => { - const ruleParams = completeRule.ruleParams; - const { - actions, - schedule, - name, - tags, - enabled, - createdBy, - updatedBy, - throttle, - createdAt, - updatedAt, - } = completeRule.ruleConfig; - return { - actions: actions.map(transformAlertToRuleAction), - created_at: createdAt.toISOString(), - created_by: createdBy ?? '', - enabled, - id: completeRule.alertId, - interval: schedule.interval, - name, - tags: transformTags(tags), - throttle: throttle ?? undefined, - updated_at: updatedAt.toISOString(), - updated_by: updatedBy ?? '', - ...commonParamsCamelToSnake(ruleParams), - ...typeSpecificCamelToSnake(ruleParams), - }; -}; - -export const buildRuleWithOverrides = ( - completeRule: CompleteRule, - eventSource: SignalSource -): RulesSchema => { - const ruleWithoutOverrides = buildRuleWithoutOverrides(completeRule); - return applyRuleOverrides(ruleWithoutOverrides, eventSource, completeRule.ruleParams); -}; - -export const applyRuleOverrides = ( - rule: RulesSchema, - eventSource: SignalSource, - ruleParams: RuleParams -): RulesSchema => { - const { riskScore, riskScoreMeta } = buildRiskScoreFromMapping({ - eventSource, - riskScore: ruleParams.riskScore, - riskScoreMapping: ruleParams.riskScoreMapping, - }); - - const { severity, severityMeta } = buildSeverityFromMapping({ - eventSource, - severity: ruleParams.severity, - severityMapping: ruleParams.severityMapping, - }); - - const { ruleName, ruleNameMeta } = buildRuleNameFromMapping({ - eventSource, - ruleName: rule.name, - ruleNameMapping: ruleParams.ruleNameOverride, - }); - - const meta = { ...ruleParams.meta, ...riskScoreMeta, ...severityMeta, ...ruleNameMeta }; - return { - ...rule, - risk_score: riskScore, - risk_score_mapping: ruleParams.riskScoreMapping ?? [], - severity, - severity_mapping: ruleParams.severityMapping ?? [], - name: ruleName, - rule_name_override: ruleParams.ruleNameOverride, - meta: Object.keys(meta).length > 0 ? meta : undefined, - }; -}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.test.ts deleted file mode 100644 index e06e8a5cdcf76..0000000000000 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.test.ts +++ /dev/null @@ -1,376 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { sampleDocNoSortId } from './__mocks__/es_results'; -import { - buildSignal, - buildParent, - buildAncestors, - additionalSignalFields, - removeClashes, -} from './build_signal'; -import { Signal, Ancestor, BaseSignalHit } from './types'; -import { - getRulesSchemaMock, - ANCHOR_DATE, -} from '../../../../common/detection_engine/schemas/response/rules_schema.mocks'; -import { getListArrayMock } from '../../../../common/detection_engine/schemas/types/lists.mock'; -import { SIGNALS_TEMPLATE_VERSION } from '../routes/index/get_signals_template'; - -describe('buildSignal', () => { - beforeEach(() => { - jest.clearAllMocks(); - }); - - test('it builds a signal as expected without original_event if event does not exist', () => { - const doc = sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'); - delete doc._source.event; - const rule = getRulesSchemaMock(); - const reason = 'signal reasonable reason'; - - const signal = { - ...buildSignal([doc], rule, reason), - ...additionalSignalFields(doc), - }; - const expected: Signal = { - _meta: { - version: SIGNALS_TEMPLATE_VERSION, - }, - parent: { - id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', - type: 'event', - index: 'myFakeSignalIndex', - depth: 0, - }, - parents: [ - { - id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', - type: 'event', - index: 'myFakeSignalIndex', - depth: 0, - }, - ], - ancestors: [ - { - id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', - type: 'event', - index: 'myFakeSignalIndex', - depth: 0, - }, - ], - original_time: '2020-04-20T21:27:45.000Z', - reason: 'signal reasonable reason', - status: 'open', - rule: { - author: [], - id: '7a7065d7-6e8b-4aae-8d20-c93613dec9f9', - created_at: new Date(ANCHOR_DATE).toISOString(), - updated_at: new Date(ANCHOR_DATE).toISOString(), - created_by: 'elastic', - description: 'some description', - enabled: true, - false_positives: ['false positive 1', 'false positive 2'], - from: 'now-6m', - immutable: false, - name: 'Query with a rule id', - query: 'user.name: root or user.name: admin', - references: ['test 1', 'test 2'], - severity: 'high', - severity_mapping: [], - updated_by: 'elastic_kibana', - tags: ['some fake tag 1', 'some fake tag 2'], - to: 'now', - type: 'query', - threat: [], - version: 1, - output_index: '.siem-signals-default', - max_signals: 100, - risk_score: 55, - risk_score_mapping: [], - language: 'kuery', - rule_id: 'query-rule-id', - interval: '5m', - exceptions_list: getListArrayMock(), - }, - depth: 1, - }; - expect(signal).toEqual(expected); - }); - - test('it builds a signal as expected with original_event if is present', () => { - const doc = sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'); - doc._source.event = { - action: 'socket_opened', - dataset: 'socket', - kind: 'event', - module: 'system', - }; - const rule = getRulesSchemaMock(); - const reason = 'signal reasonable reason'; - const signal = { - ...buildSignal([doc], rule, reason), - ...additionalSignalFields(doc), - }; - const expected: Signal = { - _meta: { - version: SIGNALS_TEMPLATE_VERSION, - }, - parent: { - id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', - type: 'event', - index: 'myFakeSignalIndex', - depth: 0, - }, - parents: [ - { - id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', - type: 'event', - index: 'myFakeSignalIndex', - depth: 0, - }, - ], - ancestors: [ - { - id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', - type: 'event', - index: 'myFakeSignalIndex', - depth: 0, - }, - ], - original_time: '2020-04-20T21:27:45.000Z', - reason: 'signal reasonable reason', - original_event: { - action: 'socket_opened', - dataset: 'socket', - kind: 'event', - module: 'system', - }, - status: 'open', - rule: { - author: [], - id: '7a7065d7-6e8b-4aae-8d20-c93613dec9f9', - created_at: new Date(ANCHOR_DATE).toISOString(), - updated_at: new Date(ANCHOR_DATE).toISOString(), - created_by: 'elastic', - description: 'some description', - enabled: true, - false_positives: ['false positive 1', 'false positive 2'], - from: 'now-6m', - immutable: false, - name: 'Query with a rule id', - query: 'user.name: root or user.name: admin', - references: ['test 1', 'test 2'], - severity: 'high', - severity_mapping: [], - updated_by: 'elastic_kibana', - tags: ['some fake tag 1', 'some fake tag 2'], - to: 'now', - type: 'query', - threat: [], - version: 1, - output_index: '.siem-signals-default', - max_signals: 100, - risk_score: 55, - risk_score_mapping: [], - language: 'kuery', - rule_id: 'query-rule-id', - interval: '5m', - exceptions_list: getListArrayMock(), - }, - depth: 1, - }; - expect(signal).toEqual(expected); - }); - - test('it builds a ancestor correctly if the parent does not exist', () => { - const doc = sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'); - doc._source.event = { - action: 'socket_opened', - dataset: 'socket', - kind: 'event', - module: 'system', - }; - const signal = buildParent(doc); - const expected: Ancestor = { - id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', - type: 'event', - index: 'myFakeSignalIndex', - depth: 0, - }; - expect(signal).toEqual(expected); - }); - - test('it builds a ancestor correctly if the parent does exist', () => { - const doc = sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'); - doc._source.event = { - action: 'socket_opened', - dataset: 'socket', - kind: 'event', - module: 'system', - }; - doc._source.signal = { - parents: [ - { - id: '730ddf9e-5a00-4f85-9ddf-5878ca511a87', - type: 'event', - index: 'myFakeSignalIndex', - depth: 0, - }, - ], - ancestors: [ - { - id: '730ddf9e-5a00-4f85-9ddf-5878ca511a87', - type: 'event', - index: 'myFakeSignalIndex', - depth: 0, - }, - ], - depth: 1, - rule: { - id: '98c0bf9e-4d38-46f4-9a6a-8a820426256b', - }, - }; - const signal = buildParent(doc); - const expected: Ancestor = { - rule: '98c0bf9e-4d38-46f4-9a6a-8a820426256b', - id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', - type: 'signal', - index: 'myFakeSignalIndex', - depth: 1, - }; - expect(signal).toEqual(expected); - }); - - test('it builds a signal ancestor correctly if the parent does not exist', () => { - const doc = sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'); - doc._source.event = { - action: 'socket_opened', - dataset: 'socket', - kind: 'event', - module: 'system', - }; - const signal = buildAncestors(doc); - const expected: Ancestor[] = [ - { - id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', - type: 'event', - index: 'myFakeSignalIndex', - depth: 0, - }, - ]; - expect(signal).toEqual(expected); - }); - - test('it builds a signal ancestor correctly if the parent does exist', () => { - const doc = sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'); - doc._source.event = { - action: 'socket_opened', - dataset: 'socket', - kind: 'event', - module: 'system', - }; - doc._source.signal = { - parents: [ - { - id: '730ddf9e-5a00-4f85-9ddf-5878ca511a87', - type: 'event', - index: 'myFakeSignalIndex', - depth: 0, - }, - ], - ancestors: [ - { - id: '730ddf9e-5a00-4f85-9ddf-5878ca511a87', - type: 'event', - index: 'myFakeSignalIndex', - depth: 0, - }, - ], - rule: { - id: '98c0bf9e-4d38-46f4-9a6a-8a820426256b', - }, - depth: 1, - }; - const signal = buildAncestors(doc); - const expected: Ancestor[] = [ - { - id: '730ddf9e-5a00-4f85-9ddf-5878ca511a87', - type: 'event', - index: 'myFakeSignalIndex', - depth: 0, - }, - { - rule: '98c0bf9e-4d38-46f4-9a6a-8a820426256b', - id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', - type: 'signal', - index: 'myFakeSignalIndex', - depth: 1, - }, - ]; - expect(signal).toEqual(expected); - }); - - describe('removeClashes', () => { - test('it will call renameClashes with a regular doc and not mutate it if it does not have a signal clash', () => { - const doc = sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'); - const output = removeClashes(doc); - expect(output).toBe(doc); // reference check - }); - - test('it will call renameClashes with a regular doc and not change anything', () => { - const doc = sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'); - const output = removeClashes(doc); - expect(output).toEqual(doc); // deep equal check - }); - - test('it will remove a "signal" numeric clash', () => { - const sampleDoc = sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'); - const doc = { - ...sampleDoc, - _source: { - ...sampleDoc._source, - signal: 127, - }, - } as unknown as BaseSignalHit; - const output = removeClashes(doc); - expect(output).toEqual(sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71')); - }); - - test('it will remove a "signal" object clash', () => { - const sampleDoc = sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'); - const doc = { - ...sampleDoc, - _source: { - ...sampleDoc._source, - signal: { child_1: { child_2: 'Test nesting' } }, - }, - } as unknown as BaseSignalHit; - const output = removeClashes(doc); - expect(output).toEqual(sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71')); - }); - - test('it will not remove a "signal" if that is signal is one of our signals', () => { - const sampleDoc = sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'); - const doc = { - ...sampleDoc, - _source: { - ...sampleDoc._source, - signal: { rule: { id: '123' } }, - }, - } as unknown as BaseSignalHit; - const output = removeClashes(doc); - const expected = { - ...sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'), - _source: { - ...sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71')._source, - signal: { rule: { id: '123' } }, - }, - }; - expect(output).toEqual(expected); - }); - }); -}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts deleted file mode 100644 index 5e26466557217..0000000000000 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts +++ /dev/null @@ -1,130 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { SearchTypes } from '../../../../common/detection_engine/types'; -import { RulesSchema } from '../../../../common/detection_engine/schemas/response/rules_schema'; -import { SIGNALS_TEMPLATE_VERSION } from '../routes/index/get_signals_template'; -import { isEventTypeSignal } from './build_event_type_signal'; -import { Signal, Ancestor, BaseSignalHit, ThresholdResult, SimpleHit } from './types'; -import { getValidDateFromDoc } from './utils'; - -/** - * Takes a parent signal or event document and extracts the information needed for the corresponding entry in the child - * signal's `signal.parents` array. - * @param doc The parent signal or event - */ -export const buildParent = (doc: BaseSignalHit): Ancestor => { - if (doc._source?.signal != null) { - return { - rule: doc._source?.signal.rule.id, - id: doc._id, - type: 'signal', - index: doc._index, - // We first look for signal.depth and use that if it exists. If it doesn't exist, this should be a pre-7.10 signal - // and should have signal.parent.depth instead. signal.parent.depth in this case is treated as equivalent to signal.depth. - depth: doc._source?.signal.depth ?? doc._source?.signal.parent?.depth ?? 1, - }; - } else { - return { - id: doc._id, - type: 'event', - index: doc._index, - depth: 0, - }; - } -}; - -/** - * Takes a parent signal or event document with N ancestors and adds the parent document to the ancestry array, - * creating an array of N+1 ancestors. - * @param doc The parent signal/event for which to extend the ancestry. - */ -export const buildAncestors = (doc: BaseSignalHit): Ancestor[] => { - const newAncestor = buildParent(doc); - const existingAncestors = doc._source?.signal?.ancestors; - if (existingAncestors != null) { - return [...existingAncestors, newAncestor]; - } else { - return [newAncestor]; - } -}; - -/** - * This removes any signal named clashes such as if a source index has - * "signal" but is not a signal object we put onto the object. If this - * is our "signal object" then we don't want to remove it. - * @param doc The source index doc to a signal. - */ -export const removeClashes = (doc: BaseSignalHit): BaseSignalHit => { - // @ts-expect-error @elastic/elasticsearch _source is optional - const { signal, ...noSignal } = doc._source; - if (signal == null || isEventTypeSignal(doc as SimpleHit)) { - return doc; - } else { - return { - ...doc, - _source: { ...noSignal }, - }; - } -}; - -/** - * Builds the `signal.*` fields that are common across all signals. - * @param docs The parent signals/events of the new signal to be built. - * @param rule The rule that is generating the new signal. - */ -export const buildSignal = (docs: BaseSignalHit[], rule: RulesSchema, reason: string): Signal => { - const _meta = { - version: SIGNALS_TEMPLATE_VERSION, - }; - const removedClashes = docs.map(removeClashes); - const parents = removedClashes.map(buildParent); - const depth = parents.reduce((acc, parent) => Math.max(parent.depth, acc), 0) + 1; - const ancestors = removedClashes.reduce( - (acc: Ancestor[], doc) => acc.concat(buildAncestors(doc)), - [] - ); - return { - _meta, - parents, - ancestors, - status: 'open', - rule, - reason, - depth, - }; -}; - -const isThresholdResult = (thresholdResult: SearchTypes): thresholdResult is ThresholdResult => { - return typeof thresholdResult === 'object'; -}; - -/** - * Creates signal fields that are only available in the special case where a signal has only 1 parent signal/event. - * We copy the original time from the document as "original_time" since we override the timestamp with the current date time. - * @param doc The parent signal/event of the new signal to be built. - */ -export const additionalSignalFields = (doc: BaseSignalHit) => { - const thresholdResult = doc._source?.threshold_result; - if (thresholdResult != null && !isThresholdResult(thresholdResult)) { - throw new Error(`threshold_result failed to validate: ${thresholdResult}`); - } - const originalTime = getValidDateFromDoc({ - doc, - timestampOverride: undefined, - }); - return { - parent: buildParent(removeClashes(doc)), - original_time: originalTime != null ? originalTime.toISOString() : undefined, - original_event: doc._source?.event ?? undefined, - threshold_result: thresholdResult, - original_signal: - doc._source?.signal != null && !isEventTypeSignal(doc as SimpleHit) - ? doc._source?.signal - : undefined, - }; -}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_factory.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_factory.ts deleted file mode 100644 index a8334cf0a4396..0000000000000 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_factory.ts +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { performance } from 'perf_hooks'; -import { countBy, isEmpty, get } from 'lodash'; - -import { ElasticsearchClient, Logger } from 'kibana/server'; -import { BuildRuleMessage } from './rule_messages'; -import { RefreshTypes } from '../types'; -import { BaseHit } from '../../../../common/detection_engine/types'; -import { errorAggregator, makeFloatString } from './utils'; -import { withSecuritySpan } from '../../../utils/with_security_span'; - -export interface GenericBulkCreateResponse { - success: boolean; - bulkCreateDuration: string; - createdItemsCount: number; - createdItems: Array; - errors: string[]; -} - -export const bulkCreateFactory = - ( - logger: Logger, - esClient: ElasticsearchClient, - buildRuleMessage: BuildRuleMessage, - refreshForBulkCreate: RefreshTypes, - indexNameOverride?: string - ) => - async (wrappedDocs: Array>): Promise> => { - if (wrappedDocs.length === 0) { - return { - errors: [], - success: true, - bulkCreateDuration: '0', - createdItemsCount: 0, - createdItems: [], - }; - } - - const bulkBody = wrappedDocs.flatMap((wrappedDoc) => [ - { - create: { - _index: indexNameOverride ?? wrappedDoc._index, - _id: wrappedDoc._id, - }, - }, - wrappedDoc._source, - ]); - const start = performance.now(); - - const response = await withSecuritySpan('writeAlertsBulk', () => - esClient.bulk({ - refresh: refreshForBulkCreate, - body: bulkBody, - }) - ); - - const end = performance.now(); - logger.debug( - buildRuleMessage( - `individual bulk process time took: ${makeFloatString(end - start)} milliseconds` - ) - ); - logger.debug(buildRuleMessage(`took property says bulk took: ${response.took} milliseconds`)); - const createdItems = wrappedDocs - .map((doc, index) => ({ - _id: response.items[index].create?._id ?? '', - _index: response.items[index].create?._index ?? '', - ...doc._source, - })) - .filter((_, index) => get(response.items[index], 'create.status') === 201); - const createdItemsCount = createdItems.length; - const duplicateSignalsCount = countBy(response.items, 'create.status')['409']; - const errorCountByMessage = errorAggregator(response, [409]); - - logger.debug(buildRuleMessage(`bulk created ${createdItemsCount} signals`)); - if (duplicateSignalsCount > 0) { - logger.debug(buildRuleMessage(`ignored ${duplicateSignalsCount} duplicate signals`)); - } - if (!isEmpty(errorCountByMessage)) { - logger.error( - buildRuleMessage( - `[-] bulkResponse had errors with responses of: ${JSON.stringify(errorCountByMessage)}` - ) - ); - return { - errors: Object.keys(errorCountByMessage), - success: false, - bulkCreateDuration: makeFloatString(end - start), - createdItemsCount, - createdItems, - }; - } else { - return { - errors: [], - success: true, - bulkCreateDuration: makeFloatString(end - start), - createdItemsCount, - createdItems, - }; - } - }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_ml_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_ml_signals.ts index 2453e92dc3c0a..a757e178ea48a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_ml_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_ml_signals.ts @@ -14,7 +14,7 @@ import { AlertInstanceState, AlertServices, } from '../../../../../alerting/server'; -import { GenericBulkCreateResponse } from './bulk_create_factory'; +import { GenericBulkCreateResponse } from '../rule_types/factories'; import { AnomalyResults, Anomaly } from '../../machine_learning'; import { BuildRuleMessage } from './rule_messages'; import { BulkCreate, WrapHits } from './types'; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/query.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/query.ts index 120bf2c2ebfce..5f1ab1c2dd5ff 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/query.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/query.ts @@ -84,7 +84,6 @@ export const queryExecutor = async ({ eventsTelemetry, id: completeRule.alertId, inputIndexPattern: inputIndex, - signalsIndex: ruleParams.outputIndex, filter: esFilter, pageSize: searchAfterSize, buildReasonMessage: buildReasonMessageForQueryAlert, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/get_filter.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/get_filter.ts index f849900ec75e1..f113e84c88ba8 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/get_filter.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/get_filter.ts @@ -21,8 +21,8 @@ import { AlertServices, } from '../../../../../alerting/server'; import { PartialFilter } from '../types'; -import { QueryFilter } from './types'; import { withSecuritySpan } from '../../../utils/with_security_span'; +import { ESBoolQuery } from '../../../../common/typed_json'; interface GetFilterArgs { type: Type; @@ -53,7 +53,7 @@ export const getFilter = async ({ type, query, lists, -}: GetFilterArgs): Promise => { +}: GetFilterArgs): Promise => { const queryFilter = () => { if (query != null && language != null && index != null) { return getQueryFilter(query, language, filters || [], index, lists); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts index 7d22d58efdd6f..52d0a04eee1ec 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts @@ -15,7 +15,6 @@ import { sampleDocWithSortId, } from './__mocks__/es_results'; import { searchAfterAndBulkCreate } from './search_after_bulk_create'; -import { DEFAULT_SIGNALS_INDEX } from '../../../../common/constants'; import { alertsMock, AlertServicesMock } from '../../../../../alerting/server/mocks'; import uuid from 'uuid'; import { listMock } from '../../../../../lists/server/mocks'; @@ -27,17 +26,19 @@ import { getRuleRangeTuples } from './utils'; // eslint-disable-next-line @kbn/eslint/no-restricted-paths import { elasticsearchClientMock } from 'src/core/server/elasticsearch/client/mocks'; import { getCompleteRuleMock, getQueryRuleParams } from '../schemas/rule_schemas.mock'; -import { bulkCreateFactory } from './bulk_create_factory'; -import { wrapHitsFactory } from './wrap_hits_factory'; +import { bulkCreateFactory } from '../rule_types/factories/bulk_create_factory'; +import { wrapHitsFactory } from '../rule_types/factories/wrap_hits_factory'; import { mockBuildRuleMessage } from './__mocks__/build_rule_message.mock'; -import { errors as esErrors } from '@elastic/elasticsearch'; import { BuildReasonMessage } from './reason_formatters'; import { QueryRuleParams } from '../schemas/rule_schemas'; +import { createPersistenceServicesMock } from '../../../../../rule_registry/server/utils/create_persistence_rule_type_wrapper.mock'; +import { PersistenceServices } from '../../../../../rule_registry/server'; const buildRuleMessage = mockBuildRuleMessage; describe('searchAfterAndBulkCreate', () => { let mockService: AlertServicesMock; + let mockPersistenceServices: jest.Mocked; let buildReasonMessage: BuildReasonMessage; let bulkCreate: BulkCreate; let wrapHits: WrapHits; @@ -46,6 +47,9 @@ describe('searchAfterAndBulkCreate', () => { const someGuids = Array.from({ length: 13 }).map(() => uuid.v4()); const sampleParams = getQueryRuleParams(); const queryCompleteRule = getCompleteRuleMock(sampleParams); + const defaultFilter = { + match_all: {}, + }; sampleParams.maxSignals = 30; let tuple: RuleRangeTuple; beforeEach(() => { @@ -65,17 +69,18 @@ describe('searchAfterAndBulkCreate', () => { maxSignals: sampleParams.maxSignals, buildRuleMessage, }).tuples[0]; + mockPersistenceServices = createPersistenceServicesMock(); bulkCreate = bulkCreateFactory( mockLogger, - mockService.scopedClusterClient.asCurrentUser, + mockPersistenceServices.alertWithPersistence, buildRuleMessage, false ); wrapHits = wrapHitsFactory({ completeRule: queryCompleteRule, - signalsIndex: DEFAULT_SIGNALS_INDEX, mergeStrategy: 'missingFields', ignoreFields: [], + spaceId: 'default', }); }); @@ -86,17 +91,9 @@ describe('searchAfterAndBulkCreate', () => { ) ); - mockService.scopedClusterClient.asCurrentUser.bulk.mockResponseOnce({ - took: 100, - errors: false, - items: [ - { - // @ts-expect-error not full response interface - create: { - status: 201, - }, - }, - ], + mockPersistenceServices.alertWithPersistence.mockResolvedValueOnce({ + createdAlerts: [{ _id: '1', _index: '.internal.alerts-security.alerts-default-000001' }], + errors: {}, }); mockService.scopedClusterClient.asCurrentUser.search.mockResolvedValueOnce( @@ -105,17 +102,9 @@ describe('searchAfterAndBulkCreate', () => { ) ); - mockService.scopedClusterClient.asCurrentUser.bulk.mockResponseOnce({ - took: 100, - errors: false, - items: [ - { - // @ts-expect-error not full response interface - create: { - status: 201, - }, - }, - ], + mockPersistenceServices.alertWithPersistence.mockResolvedValueOnce({ + createdAlerts: [{ _id: '2', _index: '.internal.alerts-security.alerts-default-000001' }], + errors: {}, }); mockService.scopedClusterClient.asCurrentUser.search.mockResolvedValueOnce( @@ -124,17 +113,9 @@ describe('searchAfterAndBulkCreate', () => { ) ); - mockService.scopedClusterClient.asCurrentUser.bulk.mockResponseOnce({ - took: 100, - errors: false, - items: [ - { - // @ts-expect-error not full response interface - create: { - status: 201, - }, - }, - ], + mockPersistenceServices.alertWithPersistence.mockResolvedValueOnce({ + createdAlerts: [{ _id: '3', _index: '.internal.alerts-security.alerts-default-000001' }], + errors: {}, }); mockService.scopedClusterClient.asCurrentUser.search.mockResolvedValueOnce( @@ -143,17 +124,9 @@ describe('searchAfterAndBulkCreate', () => { ) ); - mockService.scopedClusterClient.asCurrentUser.bulk.mockResponseOnce({ - took: 100, - errors: false, - items: [ - { - // @ts-expect-error not full response interface - create: { - status: 201, - }, - }, - ], + mockPersistenceServices.alertWithPersistence.mockResolvedValueOnce({ + createdAlerts: [{ _id: '4', _index: '.internal.alerts-security.alerts-default-000001' }], + errors: {}, }); mockService.scopedClusterClient.asCurrentUser.search.mockResolvedValueOnce( @@ -185,9 +158,8 @@ describe('searchAfterAndBulkCreate', () => { eventsTelemetry: undefined, id: sampleRuleGuid, inputIndexPattern, - signalsIndex: DEFAULT_SIGNALS_INDEX, pageSize: 1, - filter: undefined, + filter: defaultFilter, buildReasonMessage, buildRuleMessage, bulkCreate, @@ -205,17 +177,9 @@ describe('searchAfterAndBulkCreate', () => { repeatedSearchResultsWithSortId(4, 1, someGuids.slice(0, 3)) ) ); - mockService.scopedClusterClient.asCurrentUser.bulk.mockResponseOnce({ - took: 100, - errors: false, - items: [ - { - // @ts-expect-error not full response interface - create: { - status: 201, - }, - }, - ], + mockPersistenceServices.alertWithPersistence.mockResolvedValueOnce({ + createdAlerts: [{ _id: '1', _index: '.internal.alerts-security.alerts-default-000001' }], + errors: {}, }); mockService.scopedClusterClient.asCurrentUser.search.mockResolvedValueOnce( @@ -224,17 +188,9 @@ describe('searchAfterAndBulkCreate', () => { ) ); - mockService.scopedClusterClient.asCurrentUser.bulk.mockResponseOnce({ - took: 100, - errors: false, - items: [ - { - // @ts-expect-error not full response interface - create: { - status: 201, - }, - }, - ], + mockPersistenceServices.alertWithPersistence.mockResolvedValueOnce({ + createdAlerts: [{ _id: '2', _index: '.internal.alerts-security.alerts-default-000001' }], + errors: {}, }); mockService.scopedClusterClient.asCurrentUser.search.mockResolvedValueOnce( @@ -243,17 +199,9 @@ describe('searchAfterAndBulkCreate', () => { ) ); - mockService.scopedClusterClient.asCurrentUser.bulk.mockResponseOnce({ - took: 100, - errors: false, - items: [ - { - // @ts-expect-error not full response interface - create: { - status: 201, - }, - }, - ], + mockPersistenceServices.alertWithPersistence.mockResolvedValueOnce({ + createdAlerts: [{ _id: '3', _index: '.internal.alerts-security.alerts-default-000001' }], + errors: {}, }); mockService.scopedClusterClient.asCurrentUser.search.mockResolvedValueOnce( @@ -284,9 +232,8 @@ describe('searchAfterAndBulkCreate', () => { eventsTelemetry: undefined, id: sampleRuleGuid, inputIndexPattern, - signalsIndex: DEFAULT_SIGNALS_INDEX, pageSize: 1, - filter: undefined, + filter: defaultFilter, buildReasonMessage, buildRuleMessage, bulkCreate, @@ -305,35 +252,14 @@ describe('searchAfterAndBulkCreate', () => { ) ); - mockService.scopedClusterClient.asCurrentUser.bulk.mockResponseOnce({ - took: 100, - errors: false, - items: [ - { - // @ts-expect-error not full response interface - create: { - status: 201, - }, - }, - { - // @ts-expect-error not full response interface - create: { - status: 201, - }, - }, - { - // @ts-expect-error not full response interface - create: { - status: 201, - }, - }, - { - // @ts-expect-error not full response interface - create: { - status: 201, - }, - }, + mockPersistenceServices.alertWithPersistence.mockResolvedValueOnce({ + createdAlerts: [ + { _id: '1', _index: '.internal.alerts-security.alerts-default-000001' }, + { _id: '2', _index: '.internal.alerts-security.alerts-default-000001' }, + { _id: '3', _index: '.internal.alerts-security.alerts-default-000001' }, + { _id: '4', _index: '.internal.alerts-security.alerts-default-000001' }, ], + errors: {}, }); mockService.scopedClusterClient.asCurrentUser.search.mockResolvedValueOnce( @@ -364,9 +290,8 @@ describe('searchAfterAndBulkCreate', () => { eventsTelemetry: undefined, id: sampleRuleGuid, inputIndexPattern, - signalsIndex: DEFAULT_SIGNALS_INDEX, pageSize: 1, - filter: undefined, + filter: defaultFilter, buildReasonMessage, buildRuleMessage, bulkCreate, @@ -424,9 +349,8 @@ describe('searchAfterAndBulkCreate', () => { eventsTelemetry: undefined, id: sampleRuleGuid, inputIndexPattern, - signalsIndex: DEFAULT_SIGNALS_INDEX, pageSize: 1, - filter: undefined, + filter: defaultFilter, buildReasonMessage, buildRuleMessage, bulkCreate, @@ -439,39 +363,14 @@ describe('searchAfterAndBulkCreate', () => { }); test('should return success when empty string sortId present', async () => { - mockService.scopedClusterClient.asCurrentUser.bulk.mockResponseOnce({ - took: 100, - errors: false, - items: [ - { - create: { - _id: someGuids[0], - _index: 'myfakeindex', - status: 201, - }, - }, - { - create: { - _id: someGuids[1], - _index: 'myfakeindex', - status: 201, - }, - }, - { - create: { - _id: someGuids[2], - _index: 'myfakeindex', - status: 201, - }, - }, - { - create: { - _id: someGuids[3], - _index: 'myfakeindex', - status: 201, - }, - }, + mockPersistenceServices.alertWithPersistence.mockResolvedValueOnce({ + createdAlerts: [ + { _id: '1', _index: '.internal.alerts-security.alerts-default-000001' }, + { _id: '2', _index: '.internal.alerts-security.alerts-default-000001' }, + { _id: '3', _index: '.internal.alerts-security.alerts-default-000001' }, + { _id: '4', _index: '.internal.alerts-security.alerts-default-000001' }, ], + errors: {}, }); mockService.scopedClusterClient.asCurrentUser.search .mockResolvedValueOnce( @@ -502,9 +401,8 @@ describe('searchAfterAndBulkCreate', () => { eventsTelemetry: undefined, id: sampleRuleGuid, inputIndexPattern, - signalsIndex: DEFAULT_SIGNALS_INDEX, pageSize: 1, - filter: undefined, + filter: defaultFilter, buildReasonMessage, buildRuleMessage, bulkCreate, @@ -558,9 +456,8 @@ describe('searchAfterAndBulkCreate', () => { eventsTelemetry: undefined, id: sampleRuleGuid, inputIndexPattern, - signalsIndex: DEFAULT_SIGNALS_INDEX, pageSize: 1, - filter: undefined, + filter: defaultFilter, buildReasonMessage, buildRuleMessage, bulkCreate, @@ -579,35 +476,14 @@ describe('searchAfterAndBulkCreate', () => { ) ); - mockService.scopedClusterClient.asCurrentUser.bulk.mockResponseOnce({ - took: 100, - errors: false, - items: [ - { - // @ts-expect-error not full response interface - create: { - status: 201, - }, - }, - { - // @ts-expect-error not full response interface - create: { - status: 201, - }, - }, - { - // @ts-expect-error not full response interface - create: { - status: 201, - }, - }, - { - // @ts-expect-error not full response interface - create: { - status: 201, - }, - }, + mockPersistenceServices.alertWithPersistence.mockResolvedValueOnce({ + createdAlerts: [ + { _id: '1', _index: '.internal.alerts-security.alerts-default-000001' }, + { _id: '2', _index: '.internal.alerts-security.alerts-default-000001' }, + { _id: '3', _index: '.internal.alerts-security.alerts-default-000001' }, + { _id: '4', _index: '.internal.alerts-security.alerts-default-000001' }, ], + errors: {}, }); const exceptionItem = getExceptionListItemSchemaMock(); @@ -632,9 +508,8 @@ describe('searchAfterAndBulkCreate', () => { eventsTelemetry: undefined, id: sampleRuleGuid, inputIndexPattern, - signalsIndex: DEFAULT_SIGNALS_INDEX, pageSize: 1, - filter: undefined, + filter: defaultFilter, buildReasonMessage, buildRuleMessage, bulkCreate, @@ -653,35 +528,14 @@ describe('searchAfterAndBulkCreate', () => { ) ); - mockService.scopedClusterClient.asCurrentUser.bulk.mockResponseOnce({ - took: 100, - errors: false, - items: [ - { - // @ts-expect-error not full response interface - create: { - status: 201, - }, - }, - { - // @ts-expect-error not full response interface - create: { - status: 201, - }, - }, - { - // @ts-expect-error not full response interface - create: { - status: 201, - }, - }, - { - // @ts-expect-error not full response interface - create: { - status: 201, - }, - }, + mockPersistenceServices.alertWithPersistence.mockResolvedValueOnce({ + createdAlerts: [ + { _id: '1', _index: '.internal.alerts-security.alerts-default-000001' }, + { _id: '2', _index: '.internal.alerts-security.alerts-default-000001' }, + { _id: '3', _index: '.internal.alerts-security.alerts-default-000001' }, + { _id: '4', _index: '.internal.alerts-security.alerts-default-000001' }, ], + errors: {}, }); mockService.scopedClusterClient.asCurrentUser.search.mockResolvedValueOnce( @@ -708,9 +562,8 @@ describe('searchAfterAndBulkCreate', () => { eventsTelemetry: undefined, id: sampleRuleGuid, inputIndexPattern, - signalsIndex: DEFAULT_SIGNALS_INDEX, pageSize: 1, - filter: undefined, + filter: defaultFilter, buildReasonMessage, buildRuleMessage, bulkCreate, @@ -722,58 +575,6 @@ describe('searchAfterAndBulkCreate', () => { expect(lastLookBackDate).toEqual(new Date('2020-04-20T21:27:45+0000')); }); - test('if unsuccessful first bulk create', async () => { - const exceptionItem = getExceptionListItemSchemaMock(); - exceptionItem.entries = [ - { - field: 'source.ip', - operator: 'included', - type: 'list', - list: { - id: 'ci-badguys.txt', - type: 'ip', - }, - }, - ]; - mockService.scopedClusterClient.asCurrentUser.search.mockResolvedValueOnce( - elasticsearchClientMock.createSuccessTransportRequestPromise( - repeatedSearchResultsWithSortId(4, 1, someGuids.slice(0, 3)) - ) - ); - mockService.scopedClusterClient.asCurrentUser.bulk.mockReturnValue( - elasticsearchClientMock.createErrorTransportRequestPromise( - new esErrors.ResponseError( - elasticsearchClientMock.createApiResponse({ - statusCode: 400, - body: { error: { type: 'bulk_error_type' } }, - }) - ) - ) - ); - const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({ - listClient, - exceptionsList: [exceptionItem], - tuple, - completeRule: queryCompleteRule, - services: mockService, - logger: mockLogger, - eventsTelemetry: undefined, - id: sampleRuleGuid, - inputIndexPattern, - signalsIndex: DEFAULT_SIGNALS_INDEX, - pageSize: 1, - filter: undefined, - buildReasonMessage, - buildRuleMessage, - bulkCreate, - wrapHits, - }); - expect(mockLogger.error).toHaveBeenCalled(); - expect(success).toEqual(false); - expect(createdSignalsCount).toEqual(0); - expect(lastLookBackDate).toEqual(new Date('2020-04-20T21:27:45+0000')); - }); - test('should return success with 0 total hits', async () => { const exceptionItem = getExceptionListItemSchemaMock(); exceptionItem.entries = [ @@ -808,9 +609,8 @@ describe('searchAfterAndBulkCreate', () => { eventsTelemetry: undefined, id: sampleRuleGuid, inputIndexPattern, - signalsIndex: DEFAULT_SIGNALS_INDEX, pageSize: 1, - filter: undefined, + filter: defaultFilter, buildReasonMessage, buildRuleMessage, bulkCreate, @@ -855,9 +655,8 @@ describe('searchAfterAndBulkCreate', () => { eventsTelemetry: undefined, id: sampleRuleGuid, inputIndexPattern, - signalsIndex: DEFAULT_SIGNALS_INDEX, pageSize: 1, - filter: undefined, + filter: defaultFilter, buildReasonMessage, buildRuleMessage, bulkCreate, @@ -895,6 +694,16 @@ describe('searchAfterAndBulkCreate', () => { ) ); + mockPersistenceServices.alertWithPersistence.mockResolvedValueOnce({ + createdAlerts: [{ _id: '1', _index: '.internal.alerts-security.alerts-default-000001' }], + errors: { + 'error on creation': { + count: 1, + statusCode: 500, + }, + }, + }); + mockService.scopedClusterClient.asCurrentUser.bulk.mockResponseOnce(bulkItem); // adds the response with errors we are testing mockService.scopedClusterClient.asCurrentUser.search.mockResolvedValueOnce( @@ -903,17 +712,9 @@ describe('searchAfterAndBulkCreate', () => { ) ); - mockService.scopedClusterClient.asCurrentUser.bulk.mockResponseOnce({ - took: 100, - errors: false, - items: [ - { - // @ts-expect-error not full response interface - create: { - status: 201, - }, - }, - ], + mockPersistenceServices.alertWithPersistence.mockResolvedValueOnce({ + createdAlerts: [{ _id: '2', _index: '.internal.alerts-security.alerts-default-000001' }], + errors: {}, }); mockService.scopedClusterClient.asCurrentUser.search.mockResolvedValueOnce( @@ -922,17 +723,9 @@ describe('searchAfterAndBulkCreate', () => { ) ); - mockService.scopedClusterClient.asCurrentUser.bulk.mockResponseOnce({ - took: 100, - errors: false, - items: [ - { - // @ts-expect-error not full response interface - create: { - status: 201, - }, - }, - ], + mockPersistenceServices.alertWithPersistence.mockResolvedValueOnce({ + createdAlerts: [{ _id: '3', _index: '.internal.alerts-security.alerts-default-000001' }], + errors: {}, }); mockService.scopedClusterClient.asCurrentUser.search.mockResolvedValueOnce( @@ -941,17 +734,9 @@ describe('searchAfterAndBulkCreate', () => { ) ); - mockService.scopedClusterClient.asCurrentUser.bulk.mockResponseOnce({ - took: 100, - errors: false, - items: [ - { - // @ts-expect-error not full response interface - create: { - status: 201, - }, - }, - ], + mockPersistenceServices.alertWithPersistence.mockResolvedValueOnce({ + createdAlerts: [{ _id: '4', _index: '.internal.alerts-security.alerts-default-000001' }], + errors: {}, }); mockService.scopedClusterClient.asCurrentUser.search.mockResolvedValueOnce( @@ -970,9 +755,8 @@ describe('searchAfterAndBulkCreate', () => { eventsTelemetry: undefined, id: sampleRuleGuid, inputIndexPattern, - signalsIndex: DEFAULT_SIGNALS_INDEX, pageSize: 1, - filter: undefined, + filter: defaultFilter, buildReasonMessage, buildRuleMessage, bulkCreate, @@ -992,17 +776,9 @@ describe('searchAfterAndBulkCreate', () => { ) ); - mockService.scopedClusterClient.asCurrentUser.bulk.mockResponseOnce({ - took: 100, - errors: false, - items: [ - { - // @ts-expect-error not full response interface - create: { - status: 201, - }, - }, - ], + mockPersistenceServices.alertWithPersistence.mockResolvedValueOnce({ + createdAlerts: [{ _id: '1', _index: '.internal.alerts-security.alerts-default-000001' }], + errors: {}, }); mockService.scopedClusterClient.asCurrentUser.search.mockResolvedValueOnce( @@ -1011,17 +787,9 @@ describe('searchAfterAndBulkCreate', () => { ) ); - mockService.scopedClusterClient.asCurrentUser.bulk.mockResponseOnce({ - took: 100, - errors: false, - items: [ - { - // @ts-expect-error not full response interface - create: { - status: 201, - }, - }, - ], + mockPersistenceServices.alertWithPersistence.mockResolvedValueOnce({ + createdAlerts: [{ _id: '2', _index: '.internal.alerts-security.alerts-default-000001' }], + errors: {}, }); mockService.scopedClusterClient.asCurrentUser.search.mockResolvedValueOnce( @@ -1030,17 +798,9 @@ describe('searchAfterAndBulkCreate', () => { ) ); - mockService.scopedClusterClient.asCurrentUser.bulk.mockResponseOnce({ - took: 100, - errors: false, - items: [ - { - // @ts-expect-error not full response interface - create: { - status: 201, - }, - }, - ], + mockPersistenceServices.alertWithPersistence.mockResolvedValueOnce({ + createdAlerts: [{ _id: '3', _index: '.internal.alerts-security.alerts-default-000001' }], + errors: {}, }); mockService.scopedClusterClient.asCurrentUser.search.mockResolvedValueOnce( @@ -1061,9 +821,8 @@ describe('searchAfterAndBulkCreate', () => { eventsTelemetry: undefined, id: sampleRuleGuid, inputIndexPattern, - signalsIndex: DEFAULT_SIGNALS_INDEX, pageSize: 1, - filter: undefined, + filter: defaultFilter, buildReasonMessage, buildRuleMessage, bulkCreate, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts index 99230627cb6b8..69c001898b217 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts @@ -76,7 +76,6 @@ export const searchAfterAndBulkCreate = async ({ to: tuple.to.toISOString(), services, logger, - // @ts-expect-error please, declare a type explicitly instead of unknown filter, pageSize: Math.ceil(Math.min(tuple.maxSignals, pageSize)), timestampOverride: ruleParams.timestampOverride, @@ -165,7 +164,7 @@ export const searchAfterAndBulkCreate = async ({ success: bulkSuccess, createdSignalsCount: createdCount, createdSignals: createdItems, - bulkCreateTimes: bulkDuration ? [bulkDuration] : undefined, + bulkCreateTimes: [bulkDuration], errors: bulkErrors, }), ]); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_event_signal.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_event_signal.ts index c1beb55e90a85..0117cb61acbf1 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_event_signal.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_event_signal.ts @@ -134,7 +134,6 @@ export const createEventSignal = async ({ logger, pageSize: searchAfterSize, services, - signalsIndex: outputIndex, sortOrder: 'desc', trackTotalHits: false, tuple, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signal.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signal.ts index 220bebbaa4d21..a07de583d8bab 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signal.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signal.ts @@ -86,7 +86,6 @@ export const createThreatSignal = async ({ logger, pageSize: searchAfterSize, services, - signalsIndex: outputIndex, sortOrder: 'desc', trackTotalHits: false, tuple, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/bulk_create_threshold_signals.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/bulk_create_threshold_signals.test.ts index 2c14e4bed62a8..4f68be017ad67 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/bulk_create_threshold_signals.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/bulk_create_threshold_signals.test.ts @@ -5,10 +5,8 @@ * 2.0. */ -import { loggingSystemMock } from '../../../../../../../../src/core/server/mocks'; import { ThresholdNormalized } from '../../../../../common/detection_engine/schemas/common/schemas'; import { sampleDocSearchResultsNoSortId } from '../__mocks__/es_results'; -import { sampleThresholdSignalHistory } from '../__mocks__/threshold_signal_history.mock'; import { calculateThresholdSignalUuid } from '../utils'; import { transformThresholdResultsToEcs } from './bulk_create_threshold_signals'; @@ -60,12 +58,8 @@ describe('transformThresholdNormalizedResultsToEcs', () => { 'test', startedAt, from, - undefined, - loggingSystemMock.createLogger(), threshold, - '1234', - undefined, - sampleThresholdSignalHistory() + '1234' ); const _id = calculateThresholdSignalUuid( '1234', @@ -158,12 +152,8 @@ describe('transformThresholdNormalizedResultsToEcs', () => { 'test', startedAt, from, - undefined, - loggingSystemMock.createLogger(), threshold, - '1234', - undefined, - sampleThresholdSignalHistory() + '1234' ); expect(transformedResults).toEqual({ took: 10, @@ -226,12 +216,8 @@ describe('transformThresholdNormalizedResultsToEcs', () => { 'test', startedAt, from, - undefined, - loggingSystemMock.createLogger(), threshold, - '1234', - undefined, - sampleThresholdSignalHistory() + '1234' ); const _id = calculateThresholdSignalUuid('1234', startedAt, [], ''); expect(transformedResults).toEqual({ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/bulk_create_threshold_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/bulk_create_threshold_signals.ts index f098f33b2ffc7..2148d4feacdae 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/bulk_create_threshold_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/bulk_create_threshold_signals.ts @@ -9,10 +9,7 @@ import { TIMESTAMP } from '@kbn/rule-data-utils'; import { get } from 'lodash/fp'; import set from 'set-value'; -import { - ThresholdNormalized, - TimestampOverrideOrUndefined, -} from '../../../../../common/detection_engine/schemas/common/schemas'; +import { ThresholdNormalized } from '../../../../../common/detection_engine/schemas/common/schemas'; import { Logger } from '../../../../../../../../src/core/server'; import { AlertInstanceContext, @@ -21,7 +18,7 @@ import { } from '../../../../../../alerting/server'; import { BaseHit } from '../../../../../common/detection_engine/types'; import { TermAggregationBucket } from '../../../types'; -import { GenericBulkCreateResponse } from '../bulk_create_factory'; +import { GenericBulkCreateResponse } from '../../rule_types/factories/bulk_create_factory'; import { calculateThresholdSignalUuid, getThresholdAggregationParts } from '../utils'; import { buildReasonMessageForThresholdAlert } from '../reason_formatters'; import type { @@ -54,12 +51,8 @@ const getTransformedHits = ( inputIndex: string, startedAt: Date, from: Date, - logger: Logger, threshold: ThresholdNormalized, - ruleId: string, - filter: unknown, - timestampOverride: TimestampOverrideOrUndefined, - signalHistory: ThresholdSignalHistory + ruleId: string ) => { if (results.aggregations == null) { return []; @@ -184,24 +177,16 @@ export const transformThresholdResultsToEcs = ( inputIndex: string, startedAt: Date, from: Date, - filter: unknown, - logger: Logger, threshold: ThresholdNormalized, - ruleId: string, - timestampOverride: TimestampOverrideOrUndefined, - signalHistory: ThresholdSignalHistory + ruleId: string ): SignalSearchResponse => { const transformedHits = getTransformedHits( results, inputIndex, startedAt, from, - logger, threshold, - ruleId, - filter, - timestampOverride, - signalHistory + ruleId ); const thresholdResults = { ...results, @@ -228,12 +213,8 @@ export const bulkCreateThresholdSignals = async ( params.inputIndexPattern.join(','), params.startedAt, params.from, - params.filter, - params.logger, ruleParams.threshold, - ruleParams.ruleId, - ruleParams.timestampOverride, - params.signalHistory + ruleParams.ruleId ); return params.bulkCreate( diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts index a5803dc354040..44154a8727f38 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts @@ -6,7 +6,6 @@ */ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; -import { BoolQuery } from '@kbn/es-query'; import moment from 'moment'; import type { ExceptionListItemSchema } from '@kbn/securitysolution-io-ts-list-types'; import { Status } from '../../../../common/detection_engine/schemas/common/schemas'; @@ -31,7 +30,7 @@ import { Logger } from '../../../../../../../src/core/server'; import { BuildRuleMessage } from './rule_messages'; import { ITelemetryEventsSender } from '../../telemetry/sender'; import { CompleteRule, RuleParams } from '../schemas/rule_schemas'; -import { GenericBulkCreateResponse } from './bulk_create_factory'; +import { GenericBulkCreateResponse } from '../rule_types/factories'; import { EcsFieldMap } from '../../../../../rule_registry/common/assets/field_maps/ecs_field_map'; import { TypeOfFieldMap } from '../../../../../rule_registry/common/field_map'; import { BuildReasonMessage } from './reason_formatters'; @@ -275,13 +274,6 @@ export interface AlertAttributes { export type BulkResponseErrorAggregation = Record; -/** - * TODO: Remove this if/when the return filter has its own type exposed - */ -export interface QueryFilter { - bool: BoolQuery; -} - export type SignalsEnrichment = (signals: SignalSearchResponse) => Promise; export type BulkCreate = >( @@ -314,9 +306,8 @@ export interface SearchAfterAndBulkCreateParams { eventsTelemetry: ITelemetryEventsSender | undefined; id: string; inputIndexPattern: string[]; - signalsIndex: string; pageSize: number; - filter: unknown; + filter: estypes.QueryDslQueryContainer; buildRuleMessage: BuildRuleMessage; buildReasonMessage: BuildReasonMessage; enrichment?: SignalsEnrichment; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/wrap_hits_factory.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/wrap_hits_factory.ts deleted file mode 100644 index 22af4dcdb9f4a..0000000000000 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/wrap_hits_factory.ts +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { WrapHits, WrappedSignalHit } from './types'; -import { generateId } from './utils'; -import { buildBulkBody } from './build_bulk_body'; -import { filterDuplicateSignals } from './filter_duplicate_signals'; -import type { ConfigType } from '../../../config'; -import { CompleteRule, RuleParams } from '../schemas/rule_schemas'; - -export const wrapHitsFactory = - ({ - completeRule, - signalsIndex, - mergeStrategy, - ignoreFields, - }: { - completeRule: CompleteRule; - signalsIndex: string; - mergeStrategy: ConfigType['alertMergeStrategy']; - ignoreFields: ConfigType['alertIgnoreFields']; - }): WrapHits => - (events, buildReasonMessage) => { - const wrappedDocs: WrappedSignalHit[] = events.flatMap((doc) => [ - { - _index: signalsIndex, - _id: generateId(doc._index, doc._id, String(doc._version), completeRule.alertId ?? ''), - _source: buildBulkBody(completeRule, doc, mergeStrategy, ignoreFields, buildReasonMessage), - }, - ]); - - return filterDuplicateSignals(completeRule.alertId, wrappedDocs, false); - }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/wrap_sequences_factory.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/wrap_sequences_factory.ts deleted file mode 100644 index 3b93ae824849a..0000000000000 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/wrap_sequences_factory.ts +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { WrappedSignalHit, WrapSequences } from './types'; -import { buildSignalGroupFromSequence } from './build_bulk_body'; -import { ConfigType } from '../../../config'; -import { CompleteRule, RuleParams } from '../schemas/rule_schemas'; - -export const wrapSequencesFactory = - ({ - completeRule, - signalsIndex, - mergeStrategy, - ignoreFields, - }: { - completeRule: CompleteRule; - signalsIndex: string; - mergeStrategy: ConfigType['alertMergeStrategy']; - ignoreFields: ConfigType['alertIgnoreFields']; - }): WrapSequences => - (sequences, buildReasonMessage) => - sequences.reduce( - (acc: WrappedSignalHit[], sequence) => [ - ...acc, - ...buildSignalGroupFromSequence( - sequence, - completeRule, - signalsIndex, - mergeStrategy, - ignoreFields, - buildReasonMessage - ), - ], - [] - );