From 0f8c746401e01cf2509a6329df5a3862f14307f1 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Mon, 30 Nov 2020 19:09:35 +0000 Subject: [PATCH 01/13] Move threshold dupe detection logic to its own function --- .../signals/signal_rule_alert_type.ts | 51 +++------- ....ts => threshold_find_previous_signals.ts} | 0 .../signals/threshold_get_bucket_filters.ts | 96 +++++++++++++++++++ 3 files changed, 109 insertions(+), 38 deletions(-) rename x-pack/plugins/security_solution/server/lib/detection_engine/signals/{find_previous_threshold_signals.ts => threshold_find_previous_signals.ts} (100%) create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold_get_bucket_filters.ts diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts index f0b1825c7cc99..821617f014991 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -8,7 +8,6 @@ import { Logger, KibanaRequest } from 'src/core/server'; -import { Filter } from 'src/plugins/data/common'; import { SIGNALS_ID, DEFAULT_SEARCH_AFTER_PAGE_SIZE, @@ -30,7 +29,6 @@ import { RuleAlertAttributes, EqlSignalSearchResponse, BaseSignalHit, - ThresholdQueryBucket, } from './types'; import { getGapBetweenRuns, @@ -48,9 +46,9 @@ import { signalParamsSchema } from './signal_params_schema'; import { siemRuleActionGroups } from './siem_rule_action_groups'; import { findMlSignals } from './find_ml_signals'; import { findThresholdSignals } from './find_threshold_signals'; -import { findPreviousThresholdSignals } from './find_previous_threshold_signals'; import { bulkCreateMlSignals } from './bulk_create_ml_signals'; import { bulkCreateThresholdSignals } from './bulk_create_threshold_signals'; +import { getThresholdBucketFilters } from './threshold_get_bucket_filters'; import { scheduleNotificationActions, NotificationRuleTypeParams, @@ -292,21 +290,11 @@ export const signalRulesAlertType = ({ ]); } else if (isThresholdRule(type) && threshold) { const inputIndex = await getInputIndex(services, version, index); - const esFilter = await getFilter({ - type, - filters, - language, - query, - savedId, - services, - index: inputIndex, - lists: exceptionItems ?? [], - }); const { - searchResult: previousSignals, + filters: bucketFilters, searchErrors: previousSearchErrors, - } = await findPreviousThresholdSignals({ + } = await getThresholdBucketFilters({ indexPattern: [outputIndex], from, to, @@ -318,29 +306,15 @@ export const signalRulesAlertType = ({ buildRuleMessage, }); - previousSignals.aggregations.threshold.buckets.forEach((bucket: ThresholdQueryBucket) => { - esFilter.bool.filter.push(({ - bool: { - must_not: { - bool: { - must: [ - { - term: { - [threshold.field ?? 'signal.rule.rule_id']: bucket.key, - }, - }, - { - range: { - [timestampOverride ?? '@timestamp']: { - lte: bucket.lastSignalTimestamp.value_as_string, - }, - }, - }, - ], - }, - }, - }, - } as unknown) as Filter); + const esFilter = await getFilter({ + type, + filters: filters?.concat(bucketFilters), + language, + query, + savedId, + services, + index: inputIndex, + lists: exceptionItems ?? [], }); const { searchResult: thresholdResults, searchErrors } = await findThresholdSignals({ @@ -384,6 +358,7 @@ export const signalRulesAlertType = ({ tags, buildRuleMessage, }); + result = mergeReturns([ result, createSearchAfterReturnTypeFromResponse({ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_previous_threshold_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold_find_previous_signals.ts similarity index 100% rename from x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_previous_threshold_signals.ts rename to x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold_find_previous_signals.ts diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold_get_bucket_filters.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold_get_bucket_filters.ts new file mode 100644 index 0000000000000..e89fa7003b2e6 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold_get_bucket_filters.ts @@ -0,0 +1,96 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { Filter } from 'src/plugins/data/common'; +import { ESFilter } from '../../../../../../typings/elasticsearch'; + +import { TimestampOverrideOrUndefined } from '../../../../common/detection_engine/schemas/common/schemas'; +import { AlertServices } from '../../../../../alerts/server'; +import { Logger } from '../../../../../../../src/core/server'; +import { ThresholdQueryBucket } from './types'; +import { BuildRuleMessage } from './rule_messages'; +import { findPreviousThresholdSignals } from './threshold_find_previous_signals'; + +interface GetThresholdBucketFiltersParams { + from: string; + to: string; + indexPattern: string[]; + services: AlertServices; + logger: Logger; + ruleId: string; + bucketByField: string; + timestampOverride: TimestampOverrideOrUndefined; + buildRuleMessage: BuildRuleMessage; +} + +export const getThresholdBucketFilters = async ({ + from, + to, + indexPattern, + services, + logger, + ruleId, + bucketByField, + timestampOverride, + buildRuleMessage, +}: GetThresholdBucketFiltersParams): Promise<{ + filters: Filter[]; + searchErrors: string[]; +}> => { + const { searchResult, searchErrors } = await findPreviousThresholdSignals({ + indexPattern, + from, + to, + services, + logger, + ruleId, + bucketByField, + timestampOverride, + buildRuleMessage, + }); + + /* + const filter: ESFilter = { + bool: { + must_not: [], + }, + }; + */ + + const filters: ESFilter[] = []; + + searchResult.aggregations.threshold.buckets.forEach((bucket: ThresholdQueryBucket) => { + filters.push({ + bool: { + must: [ + { + term: { + [bucketByField ?? 'signal.rule.rule_id']: bucket.key, + }, + }, + { + range: { + [timestampOverride ?? '@timestamp']: { + lte: bucket.lastSignalTimestamp.value_as_string, + }, + }, + }, + ], + }, + }); + }); + + return { + filters: [ + ({ + bool: { + must_not: filters, + }, + } as unknown) as Filter, + ], + searchErrors, + }; +}; From f7419726847f6b776c26e0de697a392deb36016c Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Mon, 30 Nov 2020 19:10:25 +0000 Subject: [PATCH 02/13] Minor fixup --- .../signals/threshold_get_bucket_filters.ts | 9 --------- 1 file changed, 9 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold_get_bucket_filters.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold_get_bucket_filters.ts index e89fa7003b2e6..e797ce514c3a8 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold_get_bucket_filters.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold_get_bucket_filters.ts @@ -52,16 +52,7 @@ export const getThresholdBucketFilters = async ({ buildRuleMessage, }); - /* - const filter: ESFilter = { - bool: { - must_not: [], - }, - }; - */ - const filters: ESFilter[] = []; - searchResult.aggregations.threshold.buckets.forEach((bucket: ThresholdQueryBucket) => { filters.push({ bool: { From da111d0be4153e3436edb517f9f2dc77b906650e Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Wed, 2 Dec 2020 16:48:36 +0000 Subject: [PATCH 03/13] Refactor and remove property injection for threshold signals --- .../signals/bulk_create_threshold_signals.ts | 83 +------------------ .../signals/threshold_get_bucket_filters.ts | 43 +++++----- 2 files changed, 26 insertions(+), 100 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts index a98aae4ec8107..89edd64197ccc 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts @@ -5,7 +5,7 @@ */ import uuidv5 from 'uuid/v5'; -import { reduce, get, isEmpty } from 'lodash/fp'; +import { get, isEmpty } from 'lodash/fp'; import set from 'set-value'; import { @@ -17,7 +17,7 @@ import { AlertServices } from '../../../../../alerts/server'; import { RuleAlertAction } from '../../../../common/detection_engine/types'; import { RuleTypeParams, RefreshTypes } from '../types'; import { singleBulkCreate, SingleBulkCreateResponse } from './single_bulk_create'; -import { SignalSearchResponse, SignalSourceHit, ThresholdAggregationBucket } from './types'; +import { SignalSearchResponse, ThresholdAggregationBucket } from './types'; import { BuildRuleMessage } from './rule_messages'; // used to generate constant Threshold Signals ID when run with the same params @@ -48,81 +48,6 @@ interface BulkCreateThresholdSignalsParams { buildRuleMessage: BuildRuleMessage; } -interface FilterObject { - bool?: { - filter?: FilterObject | FilterObject[]; - should?: Array>>; - }; -} - -const injectFirstMatch = ( - hit: SignalSourceHit, - match: object | Record -): Record | undefined => { - if (match != null) { - for (const key of Object.keys(match)) { - return { [key]: get(key, hit._source) } as Record; - } - } -}; - -const getNestedQueryFilters = ( - hit: SignalSourceHit, - filtersObj: FilterObject -): Record => { - if (Array.isArray(filtersObj.bool?.filter)) { - return reduce( - (acc, filterItem) => { - const nestedFilter = getNestedQueryFilters(hit, filterItem); - - if (nestedFilter) { - return { ...acc, ...nestedFilter }; - } - - return acc; - }, - {}, - filtersObj.bool?.filter - ); - } else { - return ( - (filtersObj.bool?.should && - filtersObj.bool?.should[0] && - (injectFirstMatch(hit, filtersObj.bool.should[0].match) || - injectFirstMatch(hit, filtersObj.bool.should[0].match_phrase))) ?? - {} - ); - } -}; - -export const getThresholdSignalQueryFields = (hit: SignalSourceHit, filter: unknown) => { - const filters = get('bool.filter', filter); - - return reduce( - (acc, item) => { - if (item.match_phrase) { - return { ...acc, ...injectFirstMatch(hit, item.match_phrase) }; - } - - if (item.bool?.should && (item.bool.should[0].match || item.bool.should[0].match_phrase)) { - return { - ...acc, - ...(injectFirstMatch(hit, item.bool.should[0].match) || - injectFirstMatch(hit, item.bool.should[0].match_phrase)), - }; - } - - if (item.bool?.filter) { - return { ...acc, ...getNestedQueryFilters(hit, item) }; - } - - return acc; - }, - {}, - filters - ); -}; - const getTransformedHits = ( results: SignalSearchResponse, inputIndex: string, @@ -153,7 +78,6 @@ const getTransformedHits = ( count: totalResults, value: ruleId, }, - ...getThresholdSignalQueryFields(hit, filter), }; return [ @@ -183,11 +107,8 @@ const getTransformedHits = ( count: docCount, value: get(threshold.field, hit._source), }, - ...getThresholdSignalQueryFields(hit, filter), }; - set(source, threshold.field, key); - return { _index: inputIndex, _id: uuidv5(`${ruleId}${startedAt}${threshold.field}${key}`, NAMESPACE_ID), diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold_get_bucket_filters.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold_get_bucket_filters.ts index e797ce514c3a8..9eff26995ad6e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold_get_bucket_filters.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold_get_bucket_filters.ts @@ -52,27 +52,32 @@ export const getThresholdBucketFilters = async ({ buildRuleMessage, }); - const filters: ESFilter[] = []; - searchResult.aggregations.threshold.buckets.forEach((bucket: ThresholdQueryBucket) => { - filters.push({ - bool: { - must: [ - { - term: { - [bucketByField ?? 'signal.rule.rule_id']: bucket.key, - }, - }, - { - range: { - [timestampOverride ?? '@timestamp']: { - lte: bucket.lastSignalTimestamp.value_as_string, + const filters = searchResult.aggregations.threshold.buckets.reduce( + (acc: ESFilter[], bucket: ThresholdQueryBucket): ESFilter[] => { + return [ + ...acc, + { + bool: { + filter: [ + { + term: { + [bucketByField || 'signal.rule.rule_id']: bucket.key, + }, + }, + { + range: { + [timestampOverride ?? '@timestamp']: { + lte: bucket.lastSignalTimestamp.value_as_string, + }, + }, }, - }, + ], }, - ], - }, - }); - }); + } as ESFilter, + ]; + }, + [] as ESFilter[] + ); return { filters: [ From 539fa49cc9ff14da131ff990247ff262f2d79702 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Thu, 3 Dec 2020 02:37:55 +0000 Subject: [PATCH 04/13] Only show aggregatable fields for threshold rule grouping --- .../index_patterns/fetcher/index_patterns_fetcher.ts | 4 +++- .../fetcher/lib/field_capabilities/field_capabilities.ts | 4 +++- .../static/forms/hook_form_lib/hooks/use_field.ts | 2 +- .../common/search_strategy/index_fields/index.ts | 1 + .../public/common/containers/source/index.tsx | 7 ++++--- .../detections/components/rules/step_define_rule/index.tsx | 7 ++++++- .../server/search_strategy/index_fields/index.ts | 3 ++- 7 files changed, 20 insertions(+), 8 deletions(-) diff --git a/src/plugins/data/server/index_patterns/fetcher/index_patterns_fetcher.ts b/src/plugins/data/server/index_patterns/fetcher/index_patterns_fetcher.ts index 24dad39088b8f..0c5feef1f72f3 100644 --- a/src/plugins/data/server/index_patterns/fetcher/index_patterns_fetcher.ts +++ b/src/plugins/data/server/index_patterns/fetcher/index_patterns_fetcher.ts @@ -65,14 +65,16 @@ export class IndexPatternsFetcher { pattern: string | string[]; metaFields?: string[]; fieldCapsOptions?: { allow_no_indices: boolean }; + filters?: { aggregatable: boolean }; type?: string; rollupIndex?: string; }): Promise { - const { pattern, metaFields, fieldCapsOptions, type, rollupIndex } = options; + const { pattern, metaFields, fieldCapsOptions, filters, type, rollupIndex } = options; const fieldCapsResponse = await getFieldCapabilities( this.elasticsearchClient, pattern, metaFields, + filters, { allow_no_indices: fieldCapsOptions ? fieldCapsOptions.allow_no_indices diff --git a/src/plugins/data/server/index_patterns/fetcher/lib/field_capabilities/field_capabilities.ts b/src/plugins/data/server/index_patterns/fetcher/lib/field_capabilities/field_capabilities.ts index b9e3e8aae0899..1fb1915bfb2ed 100644 --- a/src/plugins/data/server/index_patterns/fetcher/lib/field_capabilities/field_capabilities.ts +++ b/src/plugins/data/server/index_patterns/fetcher/lib/field_capabilities/field_capabilities.ts @@ -39,6 +39,7 @@ export async function getFieldCapabilities( callCluster: ElasticsearchClient, indices: string | string[] = [], metaFields: string[] = [], + filters?: { aggregatable: boolean }, fieldCapsOptions?: { allow_no_indices: boolean } ) { const esFieldCaps = await callFieldCapsApi(callCluster, indices, fieldCapsOptions); @@ -69,7 +70,8 @@ export async function getFieldCapabilities( readFromDocValues: false, }) ) - .map(mergeOverrides); + .map(mergeOverrides) + .filter((field) => (filters?.aggregatable ? field.aggregatable === true : true)); return sortBy(allFieldsUnsorted, 'name'); } diff --git a/src/plugins/es_ui_shared/static/forms/hook_form_lib/hooks/use_field.ts b/src/plugins/es_ui_shared/static/forms/hook_form_lib/hooks/use_field.ts index eb67842bff833..83d4a6ec3db03 100644 --- a/src/plugins/es_ui_shared/static/forms/hook_form_lib/hooks/use_field.ts +++ b/src/plugins/es_ui_shared/static/forms/hook_form_lib/hooks/use_field.ts @@ -42,7 +42,7 @@ export const useField = ( ) => { const { type = FIELD_TYPES.TEXT, - defaultValue = '', // The value to use a fallback mecanism when no initial value is passed + defaultValue = '', // The value to use a fallback mechanism when no initial value is passed initialValue = config.defaultValue ?? '', // The value explicitly passed isIncludedInOutput = true, label = '', diff --git a/x-pack/plugins/security_solution/common/search_strategy/index_fields/index.ts b/x-pack/plugins/security_solution/common/search_strategy/index_fields/index.ts index 259a767f8cf70..9bccceeea360a 100644 --- a/x-pack/plugins/security_solution/common/search_strategy/index_fields/index.ts +++ b/x-pack/plugins/security_solution/common/search_strategy/index_fields/index.ts @@ -51,6 +51,7 @@ export type BeatFields = Record; export interface IndexFieldsStrategyRequest extends IEsSearchRequest { indices: string[]; onlyCheckIfIndicesExist: boolean; + filters?: { aggregatable: boolean }; } export interface IndexFieldsStrategyResponse extends IEsSearchResponse { diff --git a/x-pack/plugins/security_solution/public/common/containers/source/index.tsx b/x-pack/plugins/security_solution/public/common/containers/source/index.tsx index f245857f3d0db..af5f2295dcd0b 100644 --- a/x-pack/plugins/security_solution/public/common/containers/source/index.tsx +++ b/x-pack/plugins/security_solution/public/common/containers/source/index.tsx @@ -121,7 +121,8 @@ interface FetchIndexReturn { export const useFetchIndex = ( indexNames: string[], - onlyCheckIfIndicesExist: boolean = false + onlyCheckIfIndicesExist: boolean = false, + filters?: { aggregatable: boolean } ): [boolean, FetchIndexReturn] => { const { data, notifications } = useKibana().services; const abortCtrl = useRef(new AbortController()); @@ -144,7 +145,7 @@ export const useFetchIndex = ( setLoading(true); const searchSubscription$ = data.search .search( - { indices: iNames, onlyCheckIfIndicesExist }, + { indices: iNames, onlyCheckIfIndicesExist, filters }, { abortSignal: abortCtrl.current.signal, strategy: 'securitySolutionIndexFields', @@ -193,7 +194,7 @@ export const useFetchIndex = ( abortCtrl.current.abort(); }; }, - [data.search, notifications.toasts, onlyCheckIfIndicesExist] + [data.search, filters, notifications.toasts, onlyCheckIfIndicesExist] ); useEffect(() => { diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx index f06d4bdef74cb..ce101afea58b8 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx @@ -167,7 +167,12 @@ const StepDefineRuleComponent: FC = ({ const ruleType = formRuleType || initialState.ruleType; const queryBarQuery = formQuery != null ? formQuery.query.query : '' || initialState.queryBar.query.query; - const [indexPatternsLoading, { browserFields, indexPatterns }] = useFetchIndex(index); + const filters = isThresholdRule(ruleType) ? { aggregatable: true } : undefined; + const [indexPatternsLoading, { browserFields, indexPatterns }] = useFetchIndex( + index, + false, + filters + ); const [ threatIndexPatternsLoading, { browserFields: threatBrowserFields, indexPatterns: threatIndexPatterns }, diff --git a/x-pack/plugins/security_solution/server/search_strategy/index_fields/index.ts b/x-pack/plugins/security_solution/server/search_strategy/index_fields/index.ts index 81ee4dc7c9ad2..9438ffd2e1547 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/index_fields/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/index_fields/index.ts @@ -37,6 +37,7 @@ export const securitySolutionIndexFieldsProvider = (): ISearchStrategy< .map((index) => indexPatternsFetcher.getFieldsForWildcard({ pattern: index, + filters: request.filters, }) ) .map((p) => p.catch((e) => false)) @@ -118,7 +119,7 @@ const missingFields: FieldDescriptor[] = [ * and should avoid any and all creation of new arrays, iterating over the arrays or performing * any n^2 operations. * @param indexesAlias The index alias - * @param index The index its self + * @param index The index itself * @param indexesAliasIdx The index within the alias */ export const createFieldItem = ( From 64823744a39eda21cf3ee5222f4a023885e692a0 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Thu, 3 Dec 2020 15:50:04 +0000 Subject: [PATCH 05/13] Add threshold rule kql filter to timeline --- .../public/detections/components/alerts_table/actions.tsx | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/x-pack/plugins/security_solution/public/detections/components/alerts_table/actions.tsx b/x-pack/plugins/security_solution/public/detections/components/alerts_table/actions.tsx index e3defaea2ec67..021982ea8c4ad 100644 --- a/x-pack/plugins/security_solution/public/detections/components/alerts_table/actions.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/alerts_table/actions.tsx @@ -11,6 +11,7 @@ import { get, getOr, isEmpty, find } from 'lodash/fp'; import moment from 'moment'; import { i18n } from '@kbn/i18n'; +import { buildQueryFilter, Filter } from '../../../../../../../src/plugins/data/common'; import { TimelineId, TimelineStatus, TimelineType } from '../../../../common/types/timeline'; import { updateAlertStatus } from '../../containers/detection_engine/alerts/api'; import { SendAlertToTimelineActionProps, UpdateAlertStatusActionProps } from './types'; @@ -289,6 +290,13 @@ export const sendAlertToTimelineAction = async ({ end: to, }, eventType: 'all', + filters: [ + buildQueryFilter( + ecsData.signal?.rule?.filters, + ecsData._index ?? '', + (ecsData.signal?.rule?.filters as Filter).meta?.alias ?? '' + ), + ], kqlQuery: { filterQuery: { kuery: { From 7e2ca17077f2e7f4948f50fbf43aa92ffa09bcad Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Thu, 3 Dec 2020 15:55:39 +0000 Subject: [PATCH 06/13] Remove outdated getThresholdSignalQueryFields tests --- .../bulk_create_threshold_signals.test.ts | 327 ------------------ 1 file changed, 327 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.test.ts index 6a75d0655cf59..41bc2aa258807 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.test.ts @@ -3,330 +3,3 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ - -import { sampleDocNoSortIdNoVersion } from './__mocks__/es_results'; -import { getThresholdSignalQueryFields } from './bulk_create_threshold_signals'; - -describe('getThresholdSignalQueryFields', () => { - it('should return proper fields for match_phrase filters', () => { - const mockHit = { - ...sampleDocNoSortIdNoVersion(), - _source: { - '@timestamp': '2020-11-03T02:31:47.431Z', - event: { - dataset: 'traefik.access', - module: 'traefik', - }, - traefik: { - access: { - entryPointName: 'web-secure', - }, - }, - url: { - domain: 'kibana.siem.estc.dev', - }, - }, - }; - const mockFilters = { - bool: { - must: [], - filter: [ - { - bool: { - filter: [ - { - bool: { - should: [ - { - match_phrase: { - 'event.module': 'traefik', - }, - }, - ], - minimum_should_match: 1, - }, - }, - { - bool: { - filter: [ - { - bool: { - should: [ - { - match_phrase: { - 'event.dataset': 'traefik.access', - }, - }, - ], - minimum_should_match: 1, - }, - }, - { - bool: { - should: [ - { - match_phrase: { - 'traefik.access.entryPointName': 'web-secure', - }, - }, - ], - minimum_should_match: 1, - }, - }, - ], - }, - }, - ], - }, - }, - { - match_phrase: { - 'url.domain': 'kibana.siem.estc.dev', - }, - }, - ], - should: [], - must_not: [], - }, - }; - - expect(getThresholdSignalQueryFields(mockHit, mockFilters)).toEqual({ - 'event.dataset': 'traefik.access', - 'event.module': 'traefik', - 'traefik.access.entryPointName': 'web-secure', - 'url.domain': 'kibana.siem.estc.dev', - }); - }); - - it('should return proper fields object for nested match filters', () => { - const mockHit = { - ...sampleDocNoSortIdNoVersion(), - _source: { - '@timestamp': '2020-11-03T02:31:47.431Z', - event: { - dataset: 'traefik.access', - module: 'traefik', - }, - url: { - domain: 'kibana.siem.estc.dev', - }, - }, - }; - const filters = { - bool: { - must: [], - filter: [ - { - bool: { - filter: [ - { - bool: { - should: [ - { - match_phrase: { - 'event.module': 'traefik', - }, - }, - ], - minimum_should_match: 1, - }, - }, - { - bool: { - should: [ - { - match: { - 'event.dataset': 'traefik.*', - }, - }, - ], - minimum_should_match: 1, - }, - }, - ], - }, - }, - ], - should: [], - must_not: [], - }, - }; - - expect(getThresholdSignalQueryFields(mockHit, filters)).toEqual({ - 'event.dataset': 'traefik.access', - 'event.module': 'traefik', - }); - }); - - it('should return proper object for simple match filters', () => { - const mockHit = { - ...sampleDocNoSortIdNoVersion(), - _source: { - '@timestamp': '2020-11-03T02:31:47.431Z', - event: { - dataset: 'traefik.access', - module: 'traefik', - }, - }, - }; - const filters = { - bool: { - must: [], - filter: [ - { - bool: { - should: [ - { - match: { - 'event.module': 'traefik', - }, - }, - ], - minimum_should_match: 1, - }, - }, - { - match_phrase: { - 'event.dataset': 'traefik.access', - }, - }, - ], - should: [], - must_not: [], - }, - }; - - expect(getThresholdSignalQueryFields(mockHit, filters)).toEqual({ - 'event.dataset': 'traefik.access', - 'event.module': 'traefik', - }); - }); - - it('should return proper object for simple match_phrase filters', () => { - const mockHit = { - ...sampleDocNoSortIdNoVersion(), - _source: { - '@timestamp': '2020-11-03T02:31:47.431Z', - event: { - dataset: 'traefik.access', - module: 'traefik', - }, - }, - }; - const filters = { - bool: { - must: [], - filter: [ - { - bool: { - should: [ - { - match_phrase: { - 'event.module': 'traefik', - }, - }, - ], - minimum_should_match: 1, - }, - }, - { - match_phrase: { - 'event.dataset': 'traefik.access', - }, - }, - ], - should: [], - must_not: [], - }, - }; - - expect(getThresholdSignalQueryFields(mockHit, filters)).toEqual({ - 'event.module': 'traefik', - 'event.dataset': 'traefik.access', - }); - }); - - it('should return proper object for exists filters', () => { - const mockHit = { - ...sampleDocNoSortIdNoVersion(), - _source: { - '@timestamp': '2020-11-03T02:31:47.431Z', - event: { - module: 'traefik', - }, - }, - }; - const filters = { - bool: { - should: [ - { - bool: { - should: [ - { - exists: { - field: 'process.name', - }, - }, - ], - minimum_should_match: 1, - }, - }, - { - bool: { - should: [ - { - exists: { - field: 'event.type', - }, - }, - ], - minimum_should_match: 1, - }, - }, - ], - minimum_should_match: 1, - }, - }; - expect(getThresholdSignalQueryFields(mockHit, filters)).toEqual({}); - }); - - it('should NOT add invalid characters from CIDR such as the "/" proper object for simple match_phrase filters', () => { - const mockHit = { - ...sampleDocNoSortIdNoVersion(), - _source: { - '@timestamp': '2020-11-03T02:31:47.431Z', - destination: { - ip: '192.168.0.16', - }, - event: { - module: 'traefik', - }, - }, - }; - const filters = { - bool: { - must: [], - filter: [ - { - bool: { - should: [ - { - match: { - 'destination.ip': '192.168.0.0/16', - }, - }, - ], - minimum_should_match: 1, - }, - }, - ], - should: [], - must_not: [], - }, - }; - - expect(getThresholdSignalQueryFields(mockHit, filters)).toEqual({ - 'destination.ip': '192.168.0.16', - }); - }); -}); From 6f9d4689b2cc5a571a2b0d606b5c09ec77eb6397 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Wed, 16 Dec 2020 15:47:41 +0000 Subject: [PATCH 07/13] Filter aggregatable fields on client --- .../search_strategy/index_fields/index.ts | 2 +- .../public/common/containers/source/index.tsx | 2 +- .../rules/step_define_rule/index.tsx | 31 ++++++++++++++----- 3 files changed, 25 insertions(+), 10 deletions(-) diff --git a/x-pack/plugins/security_solution/common/search_strategy/index_fields/index.ts b/x-pack/plugins/security_solution/common/search_strategy/index_fields/index.ts index 9bccceeea360a..5f8010a2ffa66 100644 --- a/x-pack/plugins/security_solution/common/search_strategy/index_fields/index.ts +++ b/x-pack/plugins/security_solution/common/search_strategy/index_fields/index.ts @@ -51,7 +51,7 @@ export type BeatFields = Record; export interface IndexFieldsStrategyRequest extends IEsSearchRequest { indices: string[]; onlyCheckIfIndicesExist: boolean; - filters?: { aggregatable: boolean }; + filters?: { aggregatable?: boolean }; } export interface IndexFieldsStrategyResponse extends IEsSearchResponse { diff --git a/x-pack/plugins/security_solution/public/common/containers/source/index.tsx b/x-pack/plugins/security_solution/public/common/containers/source/index.tsx index bfc1065d66324..2acaee02775a0 100644 --- a/x-pack/plugins/security_solution/public/common/containers/source/index.tsx +++ b/x-pack/plugins/security_solution/public/common/containers/source/index.tsx @@ -122,7 +122,7 @@ interface FetchIndexReturn { export const useFetchIndex = ( indexNames: string[], onlyCheckIfIndicesExist: boolean = false, - filters?: { aggregatable: boolean } + filters?: { aggregatable?: boolean } ): [boolean, FetchIndexReturn] => { const { data, notifications } = useKibana().services; const abortCtrl = useRef(new AbortController()); diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx index ce101afea58b8..1fe1b809d4f30 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx @@ -52,7 +52,7 @@ import { } from '../../../../../common/detection_engine/utils'; import { EqlQueryBar } from '../eql_query_bar'; import { ThreatMatchInput } from '../threatmatch_input'; -import { useFetchIndex } from '../../../../common/containers/source'; +import { BrowserField, BrowserFields, useFetchIndex } from '../../../../common/containers/source'; import { PreviewQuery, Threshold } from '../query_preview'; const CommonUseField = getUseField({ component: Field }); @@ -167,12 +167,27 @@ const StepDefineRuleComponent: FC = ({ const ruleType = formRuleType || initialState.ruleType; const queryBarQuery = formQuery != null ? formQuery.query.query : '' || initialState.queryBar.query.query; - const filters = isThresholdRule(ruleType) ? { aggregatable: true } : undefined; - const [indexPatternsLoading, { browserFields, indexPatterns }] = useFetchIndex( - index, - false, - filters + const [indexPatternsLoading, { browserFields, indexPatterns }] = useFetchIndex(index); + const aggregatableFields = Object.entries(browserFields).reduce( + (groupAcc, [groupName, groupValue]) => { + return { + ...groupAcc, + [groupName]: { + fields: Object.entries(groupValue.fields ?? {}).reduce>( + (fieldAcc, [fieldName, fieldValue]) => { + if (fieldValue.aggregatable === true) { + return { ...fieldAcc, [fieldName]: fieldValue }; + } + return fieldAcc; + }, + {} + ), + } as Partial, + }; + }, + {} ); + const [ threatIndexPatternsLoading, { browserFields: threatBrowserFields, indexPatterns: threatIndexPatterns }, @@ -267,12 +282,12 @@ const StepDefineRuleComponent: FC = ({ const ThresholdInputChildren = useCallback( ({ thresholdField, thresholdValue }) => ( ), - [browserFields] + [aggregatableFields] ); const ThreatMatchInputChildren = useCallback( From 6e94e3b03450d33dbaa3e87bd750457c34ef8c94 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Wed, 16 Dec 2020 18:10:19 +0000 Subject: [PATCH 08/13] Revert "Only show aggregatable fields for threshold rule grouping" This reverts commit 539fa49cc9ff14da131ff990247ff262f2d79702. --- .../index_patterns/fetcher/index_patterns_fetcher.ts | 4 +--- .../fetcher/lib/field_capabilities/field_capabilities.ts | 4 +--- .../static/forms/hook_form_lib/hooks/use_field.ts | 2 +- .../common/search_strategy/index_fields/index.ts | 1 - .../public/common/containers/source/index.tsx | 7 +++---- .../server/search_strategy/index_fields/index.ts | 3 +-- 6 files changed, 7 insertions(+), 14 deletions(-) diff --git a/src/plugins/data/server/index_patterns/fetcher/index_patterns_fetcher.ts b/src/plugins/data/server/index_patterns/fetcher/index_patterns_fetcher.ts index 0c5feef1f72f3..24dad39088b8f 100644 --- a/src/plugins/data/server/index_patterns/fetcher/index_patterns_fetcher.ts +++ b/src/plugins/data/server/index_patterns/fetcher/index_patterns_fetcher.ts @@ -65,16 +65,14 @@ export class IndexPatternsFetcher { pattern: string | string[]; metaFields?: string[]; fieldCapsOptions?: { allow_no_indices: boolean }; - filters?: { aggregatable: boolean }; type?: string; rollupIndex?: string; }): Promise { - const { pattern, metaFields, fieldCapsOptions, filters, type, rollupIndex } = options; + const { pattern, metaFields, fieldCapsOptions, type, rollupIndex } = options; const fieldCapsResponse = await getFieldCapabilities( this.elasticsearchClient, pattern, metaFields, - filters, { allow_no_indices: fieldCapsOptions ? fieldCapsOptions.allow_no_indices diff --git a/src/plugins/data/server/index_patterns/fetcher/lib/field_capabilities/field_capabilities.ts b/src/plugins/data/server/index_patterns/fetcher/lib/field_capabilities/field_capabilities.ts index 1fb1915bfb2ed..b9e3e8aae0899 100644 --- a/src/plugins/data/server/index_patterns/fetcher/lib/field_capabilities/field_capabilities.ts +++ b/src/plugins/data/server/index_patterns/fetcher/lib/field_capabilities/field_capabilities.ts @@ -39,7 +39,6 @@ export async function getFieldCapabilities( callCluster: ElasticsearchClient, indices: string | string[] = [], metaFields: string[] = [], - filters?: { aggregatable: boolean }, fieldCapsOptions?: { allow_no_indices: boolean } ) { const esFieldCaps = await callFieldCapsApi(callCluster, indices, fieldCapsOptions); @@ -70,8 +69,7 @@ export async function getFieldCapabilities( readFromDocValues: false, }) ) - .map(mergeOverrides) - .filter((field) => (filters?.aggregatable ? field.aggregatable === true : true)); + .map(mergeOverrides); return sortBy(allFieldsUnsorted, 'name'); } diff --git a/src/plugins/es_ui_shared/static/forms/hook_form_lib/hooks/use_field.ts b/src/plugins/es_ui_shared/static/forms/hook_form_lib/hooks/use_field.ts index 83d4a6ec3db03..eb67842bff833 100644 --- a/src/plugins/es_ui_shared/static/forms/hook_form_lib/hooks/use_field.ts +++ b/src/plugins/es_ui_shared/static/forms/hook_form_lib/hooks/use_field.ts @@ -42,7 +42,7 @@ export const useField = ( ) => { const { type = FIELD_TYPES.TEXT, - defaultValue = '', // The value to use a fallback mechanism when no initial value is passed + defaultValue = '', // The value to use a fallback mecanism when no initial value is passed initialValue = config.defaultValue ?? '', // The value explicitly passed isIncludedInOutput = true, label = '', diff --git a/x-pack/plugins/security_solution/common/search_strategy/index_fields/index.ts b/x-pack/plugins/security_solution/common/search_strategy/index_fields/index.ts index 5f8010a2ffa66..259a767f8cf70 100644 --- a/x-pack/plugins/security_solution/common/search_strategy/index_fields/index.ts +++ b/x-pack/plugins/security_solution/common/search_strategy/index_fields/index.ts @@ -51,7 +51,6 @@ export type BeatFields = Record; export interface IndexFieldsStrategyRequest extends IEsSearchRequest { indices: string[]; onlyCheckIfIndicesExist: boolean; - filters?: { aggregatable?: boolean }; } export interface IndexFieldsStrategyResponse extends IEsSearchResponse { diff --git a/x-pack/plugins/security_solution/public/common/containers/source/index.tsx b/x-pack/plugins/security_solution/public/common/containers/source/index.tsx index 2acaee02775a0..9bd375b897daf 100644 --- a/x-pack/plugins/security_solution/public/common/containers/source/index.tsx +++ b/x-pack/plugins/security_solution/public/common/containers/source/index.tsx @@ -121,8 +121,7 @@ interface FetchIndexReturn { export const useFetchIndex = ( indexNames: string[], - onlyCheckIfIndicesExist: boolean = false, - filters?: { aggregatable?: boolean } + onlyCheckIfIndicesExist: boolean = false ): [boolean, FetchIndexReturn] => { const { data, notifications } = useKibana().services; const abortCtrl = useRef(new AbortController()); @@ -145,7 +144,7 @@ export const useFetchIndex = ( setLoading(true); const searchSubscription$ = data.search .search( - { indices: iNames, onlyCheckIfIndicesExist, filters }, + { indices: iNames, onlyCheckIfIndicesExist }, { abortSignal: abortCtrl.current.signal, strategy: 'securitySolutionIndexFields', @@ -194,7 +193,7 @@ export const useFetchIndex = ( abortCtrl.current.abort(); }; }, - [data.search, filters, notifications.toasts, onlyCheckIfIndicesExist] + [data.search, notifications.toasts, onlyCheckIfIndicesExist] ); useEffect(() => { diff --git a/x-pack/plugins/security_solution/server/search_strategy/index_fields/index.ts b/x-pack/plugins/security_solution/server/search_strategy/index_fields/index.ts index 9438ffd2e1547..81ee4dc7c9ad2 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/index_fields/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/index_fields/index.ts @@ -37,7 +37,6 @@ export const securitySolutionIndexFieldsProvider = (): ISearchStrategy< .map((index) => indexPatternsFetcher.getFieldsForWildcard({ pattern: index, - filters: request.filters, }) ) .map((p) => p.catch((e) => false)) @@ -119,7 +118,7 @@ const missingFields: FieldDescriptor[] = [ * and should avoid any and all creation of new arrays, iterating over the arrays or performing * any n^2 operations. * @param indexesAlias The index alias - * @param index The index itself + * @param index The index its self * @param indexesAliasIdx The index within the alias */ export const createFieldItem = ( From 04b143248b47a3fcc05526c2efed7307ff5d2291 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Wed, 16 Dec 2020 21:21:30 +0000 Subject: [PATCH 09/13] Fix bug with incorrect calculation of threshold signal dupes when no threshold field present --- .../signals/threshold_get_bucket_filters.ts | 40 ++++++++++--------- 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold_get_bucket_filters.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold_get_bucket_filters.ts index 9eff26995ad6e..bf060da1e76b8 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold_get_bucket_filters.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold_get_bucket_filters.ts @@ -4,6 +4,8 @@ * you may not use this file except in compliance with the Elastic License. */ +import { isEmpty } from 'lodash'; + import { Filter } from 'src/plugins/data/common'; import { ESFilter } from '../../../../../../typings/elasticsearch'; @@ -54,27 +56,29 @@ export const getThresholdBucketFilters = async ({ const filters = searchResult.aggregations.threshold.buckets.reduce( (acc: ESFilter[], bucket: ThresholdQueryBucket): ESFilter[] => { - return [ - ...acc, - { - bool: { - filter: [ - { - term: { - [bucketByField || 'signal.rule.rule_id']: bucket.key, - }, - }, - { - range: { - [timestampOverride ?? '@timestamp']: { - lte: bucket.lastSignalTimestamp.value_as_string, - }, + const filter = { + bool: { + filter: [ + { + range: { + [timestampOverride ?? '@timestamp']: { + lte: bucket.lastSignalTimestamp.value_as_string, }, }, - ], + }, + ], + }, + } as ESFilter; + + if (!isEmpty(bucketByField)) { + (filter.bool.filter as ESFilter[]).push({ + term: { + [bucketByField]: bucket.key, }, - } as ESFilter, - ]; + }); + } + + return [...acc, filter]; }, [] as ESFilter[] ); From 25efc0117706853b63bdf4f9796c71ccb6dc2ff1 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Thu, 17 Dec 2020 15:25:20 +0000 Subject: [PATCH 10/13] Revert "Add threshold rule kql filter to timeline" This reverts commit 64823744a39eda21cf3ee5222f4a023885e692a0. --- .../public/detections/components/alerts_table/actions.tsx | 8 -------- 1 file changed, 8 deletions(-) diff --git a/x-pack/plugins/security_solution/public/detections/components/alerts_table/actions.tsx b/x-pack/plugins/security_solution/public/detections/components/alerts_table/actions.tsx index aacbf8c6be626..54cdd636f7a33 100644 --- a/x-pack/plugins/security_solution/public/detections/components/alerts_table/actions.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/alerts_table/actions.tsx @@ -11,7 +11,6 @@ import { get, getOr, isEmpty, find } from 'lodash/fp'; import moment from 'moment'; import { i18n } from '@kbn/i18n'; -import { buildQueryFilter, Filter } from '../../../../../../../src/plugins/data/common'; import { TimelineId, TimelineStatus, TimelineType } from '../../../../common/types/timeline'; import { updateAlertStatus } from '../../containers/detection_engine/alerts/api'; import { SendAlertToTimelineActionProps, UpdateAlertStatusActionProps } from './types'; @@ -286,13 +285,6 @@ export const sendAlertToTimelineAction = async ({ end: to, }, eventType: 'all', - filters: [ - buildQueryFilter( - ecsData.signal?.rule?.filters, - ecsData._index ?? '', - (ecsData.signal?.rule?.filters as Filter).meta?.alias ?? '' - ), - ], kqlQuery: { filterQuery: { kuery: { From 2e6cf19e16c6f83d0390a8dadfb949a5a7911a4a Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Thu, 17 Dec 2020 22:06:16 +0000 Subject: [PATCH 11/13] Add test skeleton --- .../bulk_create_threshold_signals.test.ts | 75 +++++++++++++++++++ .../signals/bulk_create_threshold_signals.ts | 4 +- 2 files changed, 78 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.test.ts index 41bc2aa258807..75c8a8f5a7e57 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.test.ts @@ -3,3 +3,78 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ + +import uuidv5 from 'uuid/v5'; +import { loggingSystemMock } from '../../../../../../../src/core/server/mocks'; +import { sampleDocNoSortId, sampleDocSearchResultsNoSortId } from './__mocks__/es_results'; +import { NAMESPACE_ID, transformThresholdResultsToEcs } from './bulk_create_threshold_signals'; + +describe('', () => { + it('should return transformed threshold results', () => { + const threshold = { + field: 'source.ip', + value: 1, + }; + const startedAt = new Date('2020-12-17T16:27:00Z'); + const transformedResults = transformThresholdResultsToEcs( + { + ...sampleDocSearchResultsNoSortId('abcd'), + aggregations: { + threshold: { + buckets: [ + { + key: '127.0.0.1', + doc_count: 1, + top_threshold_hits: { + hits: { + hits: [sampleDocNoSortId('abcd')], + }, + }, + }, + ], + }, + }, + }, + 'test', + startedAt, + undefined, + loggingSystemMock.createLogger(), + threshold, + 'abcd', + undefined + ); + const _id = uuidv5(`abcd${startedAt}source.ip127.0.0.1`, NAMESPACE_ID); + expect(transformedResults).toEqual({ + took: 10, + timed_out: false, + _shards: { + total: 10, + successful: 10, + failed: 0, + skipped: 0, + }, + results: { + hits: { + total: 1, + }, + }, + hits: { + total: 100, + max_score: 100, + hits: [ + { + _id, + _index: 'test', + _source: { + '@timestamp': '2020-04-20T21:27:45+0000', + threshold_result: { + count: 1, + value: '127.0.0.1', + }, + }, + }, + ], + }, + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts index 89edd64197ccc..ee5c507c19b64 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts @@ -21,7 +21,7 @@ import { SignalSearchResponse, ThresholdAggregationBucket } from './types'; import { BuildRuleMessage } from './rule_messages'; // used to generate constant Threshold Signals ID when run with the same params -const NAMESPACE_ID = '0684ec03-7201-4ee0-8ee0-3a3f6b2479b2'; +export const NAMESPACE_ID = '0684ec03-7201-4ee0-8ee0-3a3f6b2479b2'; interface BulkCreateThresholdSignalsParams { actions: RuleAlertAction[]; @@ -147,6 +147,8 @@ export const transformThresholdResultsToEcs = ( }, }; + delete thresholdResults.aggregations; // no longer needed + set(thresholdResults, 'results.hits.total', transformedHits.length); return thresholdResults; From 7dc1e975e6571a1368d134c7cdb27880ca7595ef Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Fri, 18 Dec 2020 00:29:29 +0000 Subject: [PATCH 12/13] Finish tests --- .../bulk_create_threshold_signals.test.ts | 10 +++++----- .../signals/bulk_create_threshold_signals.ts | 9 +++------ .../lib/detection_engine/signals/utils.test.ts | 15 +++++++++++++++ .../lib/detection_engine/signals/utils.ts | 18 ++++++++++++++++++ 4 files changed, 41 insertions(+), 11 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.test.ts index 75c8a8f5a7e57..022c07defc9c1 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.test.ts @@ -4,12 +4,12 @@ * you may not use this file except in compliance with the Elastic License. */ -import uuidv5 from 'uuid/v5'; import { loggingSystemMock } from '../../../../../../../src/core/server/mocks'; import { sampleDocNoSortId, sampleDocSearchResultsNoSortId } from './__mocks__/es_results'; -import { NAMESPACE_ID, transformThresholdResultsToEcs } from './bulk_create_threshold_signals'; +import { transformThresholdResultsToEcs } from './bulk_create_threshold_signals'; +import { calculateThresholdSignalUuid } from './utils'; -describe('', () => { +describe('transformThresholdResultsToEcs', () => { it('should return transformed threshold results', () => { const threshold = { field: 'source.ip', @@ -40,10 +40,10 @@ describe('', () => { undefined, loggingSystemMock.createLogger(), threshold, - 'abcd', + '1234', undefined ); - const _id = uuidv5(`abcd${startedAt}source.ip127.0.0.1`, NAMESPACE_ID); + const _id = calculateThresholdSignalUuid('1234', startedAt, 'source.ip', '127.0.0.1'); expect(transformedResults).toEqual({ took: 10, timed_out: false, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts index ee5c507c19b64..3cad33b278749 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts @@ -4,7 +4,6 @@ * you may not use this file except in compliance with the Elastic License. */ -import uuidv5 from 'uuid/v5'; import { get, isEmpty } from 'lodash/fp'; import set from 'set-value'; @@ -18,11 +17,9 @@ import { RuleAlertAction } from '../../../../common/detection_engine/types'; import { RuleTypeParams, RefreshTypes } from '../types'; import { singleBulkCreate, SingleBulkCreateResponse } from './single_bulk_create'; import { SignalSearchResponse, ThresholdAggregationBucket } from './types'; +import { calculateThresholdSignalUuid } from './utils'; import { BuildRuleMessage } from './rule_messages'; -// used to generate constant Threshold Signals ID when run with the same params -export const NAMESPACE_ID = '0684ec03-7201-4ee0-8ee0-3a3f6b2479b2'; - interface BulkCreateThresholdSignalsParams { actions: RuleAlertAction[]; someResult: SignalSearchResponse; @@ -83,7 +80,7 @@ const getTransformedHits = ( return [ { _index: inputIndex, - _id: uuidv5(`${ruleId}${startedAt}${threshold.field}`, NAMESPACE_ID), + _id: calculateThresholdSignalUuid(ruleId, startedAt, threshold.field), _source: source, }, ]; @@ -111,7 +108,7 @@ const getTransformedHits = ( return { _index: inputIndex, - _id: uuidv5(`${ruleId}${startedAt}${threshold.field}${key}`, NAMESPACE_ID), + _id: calculateThresholdSignalUuid(ruleId, startedAt, threshold.field, key), _source: source, }; } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts index dd936776f691a..073e30bbc6e26 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts @@ -36,6 +36,7 @@ import { mergeReturns, createTotalHitsFromSearchResult, lastValidDate, + calculateThresholdSignalUuid, } from './utils'; import { BulkResponseErrorAggregation, SearchAfterAndBulkCreateReturnType } from './types'; import { @@ -1303,4 +1304,18 @@ describe('utils', () => { expect(result).toEqual(4); }); }); + + describe('calculateThresholdSignalUuid', () => { + it('should generate a uuid without key', () => { + const startedAt = new Date('2020-12-17T16:27:00Z'); + const signalUuid = calculateThresholdSignalUuid('abcd', startedAt, 'agent.name'); + expect(signalUuid).toEqual('c0cbe4b7-48de-5734-ae81-d8de3e79839d'); + }); + + it('should generate a uuid with key', () => { + const startedAt = new Date('2019-11-18T13:32:00Z'); + const signalUuid = calculateThresholdSignalUuid('abcd', startedAt, 'host.ip', '1.2.3.4'); + expect(signalUuid).toEqual('f568509e-b570-5d3c-a7ed-7c73fd29ddaf'); + }); + }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts index 2114f21d9cead..18f6e8d127b1b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts @@ -5,6 +5,7 @@ */ import { createHash } from 'crypto'; import moment from 'moment'; +import uuidv5 from 'uuid/v5'; import dateMath from '@elastic/datemath'; import { TimestampOverrideOrUndefined } from '../../../../common/detection_engine/schemas/common/schemas'; @@ -661,3 +662,20 @@ export const createTotalHitsFromSearchResult = ({ : searchResult.hits.total.value; return totalHits; }; + +export const calculateThresholdSignalUuid = ( + ruleId: string, + startedAt: Date, + thresholdField: string, + key?: string +): string => { + // used to generate constant Threshold Signals ID when run with the same params + const NAMESPACE_ID = '0684ec03-7201-4ee0-8ee0-3a3f6b2479b2'; + + let baseString = `${ruleId}${startedAt}${thresholdField}`; + if (key != null) { + baseString = `${baseString}${key}`; + } + + return uuidv5(baseString, NAMESPACE_ID); +}; From 8eb84045096745d6ee9b674c4e24953e6291e9bf Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Sun, 20 Dec 2020 01:35:55 +0000 Subject: [PATCH 13/13] Address comment --- .../lib/detection_engine/signals/signal_rule_alert_type.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts index 455a9a454de3d..3928228357d4c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -323,7 +323,7 @@ export const signalRulesAlertType = ({ const esFilter = await getFilter({ type, - filters: filters?.concat(bucketFilters), + filters: filters ? filters.concat(bucketFilters) : bucketFilters, language, query, savedId,