From 091dca7e2287b97db8baa956bef1e10a977f144f Mon Sep 17 00:00:00 2001 From: Devin Hurley Date: Tue, 14 Jul 2020 16:11:31 -0400 Subject: [PATCH 1/5] wip --- .../signals/filter_events_with_list.ts | 2 +- .../signals/search_after_bulk_create.ts | 20 ++++++++++++++++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filter_events_with_list.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filter_events_with_list.ts index f16de8bf05ef4..cf11500f345d3 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filter_events_with_list.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filter_events_with_list.ts @@ -31,7 +31,7 @@ export const filterEventsAgainstList = async ({ buildRuleMessage, }: FilterEventsAgainstList): Promise => { try { - logger.debug(buildRuleMessage(`exceptionsList: ${JSON.stringify(exceptionsList, null, 2)}`)); + // logger.debug(buildRuleMessage(`exceptionsList: ${JSON.stringify(exceptionsList, null, 2)}`)); if (exceptionsList == null || exceptionsList.length === 0) { logger.debug(buildRuleMessage('about to return original search result')); return eventSearchResult; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts index f3025ead69a05..fcbf421532f7b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts @@ -166,7 +166,6 @@ export const searchAfterAndBulkCreate = async ({ searchResult.hits.hits[searchResult.hits.hits.length - 1]?._source['@timestamp'] ) : null; - searchResultSize += searchResult.hits.hits.length; // filter out the search results that match with the values found in the list. // the resulting set are valid signals that are not on the allowlist. @@ -180,12 +179,24 @@ export const searchAfterAndBulkCreate = async ({ buildRuleMessage, }) : searchResult; + // searchResultSize += filteredEvents.hits.hits.length; if (filteredEvents.hits.total === 0 || filteredEvents.hits.hits.length === 0) { // everything in the events were allowed, so no need to generate signals toReturn.success = true; break; } + // make sure we are not going to create more signals than maxSignals allows + if ( + searchResultSize != null && + searchResultSize + filteredEvents.hits.hits.length > tuple.maxSignals + ) { + filteredEvents.hits.hits = filteredEvents.hits.hits.slice( + 0, + tuple.maxSignals - searchResultSize + ); + } + const { bulkCreateDuration: bulkDuration, createdItemsCount: createdCount, @@ -207,9 +218,11 @@ export const searchAfterAndBulkCreate = async ({ refresh, tags, throttle, + searchResultSize, }); logger.debug(buildRuleMessage(`created ${createdCount} signals`)); toReturn.createdSignalsCount += createdCount; + searchResultSize += createdCount; if (bulkDuration) { toReturn.bulkCreateTimes.push(bulkDuration); } @@ -230,6 +243,11 @@ export const searchAfterAndBulkCreate = async ({ ? filteredEvents.hits.hits[0].sort[0] : undefined; } + logger.debug( + `is searchResultSize (${searchResultSize}) > maxSignals (${tuple.maxSignals})?: ${ + searchResultSize > tuple.maxSignals + }` + ); } catch (exc) { logger.error(buildRuleMessage(`[-] search_after and bulk threw an error ${exc}`)); toReturn.success = false; From 710a06f9d4dc39d10c427eb4ce3c0fded299fec0 Mon Sep 17 00:00:00 2001 From: Devin Hurley Date: Tue, 14 Jul 2020 17:10:43 -0400 Subject: [PATCH 2/5] fix type error --- .../lib/detection_engine/signals/search_after_bulk_create.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts index fcbf421532f7b..1261a30352fa6 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts @@ -218,7 +218,6 @@ export const searchAfterAndBulkCreate = async ({ refresh, tags, throttle, - searchResultSize, }); logger.debug(buildRuleMessage(`created ${createdCount} signals`)); toReturn.createdSignalsCount += createdCount; From 1e5b1b59c0cac02b38fce21875c86107b3d110b9 Mon Sep 17 00:00:00 2001 From: Devin Hurley Date: Tue, 14 Jul 2020 17:14:24 -0400 Subject: [PATCH 3/5] remove logging and a comment --- .../lib/detection_engine/signals/filter_events_with_list.ts | 1 - .../detection_engine/signals/search_after_bulk_create.ts | 6 ------ 2 files changed, 7 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filter_events_with_list.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filter_events_with_list.ts index cf11500f345d3..8af08a02f4152 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filter_events_with_list.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filter_events_with_list.ts @@ -31,7 +31,6 @@ export const filterEventsAgainstList = async ({ buildRuleMessage, }: FilterEventsAgainstList): Promise => { try { - // logger.debug(buildRuleMessage(`exceptionsList: ${JSON.stringify(exceptionsList, null, 2)}`)); if (exceptionsList == null || exceptionsList.length === 0) { logger.debug(buildRuleMessage('about to return original search result')); return eventSearchResult; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts index 1261a30352fa6..1e56c4ac135f9 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts @@ -179,7 +179,6 @@ export const searchAfterAndBulkCreate = async ({ buildRuleMessage, }) : searchResult; - // searchResultSize += filteredEvents.hits.hits.length; if (filteredEvents.hits.total === 0 || filteredEvents.hits.hits.length === 0) { // everything in the events were allowed, so no need to generate signals toReturn.success = true; @@ -242,11 +241,6 @@ export const searchAfterAndBulkCreate = async ({ ? filteredEvents.hits.hits[0].sort[0] : undefined; } - logger.debug( - `is searchResultSize (${searchResultSize}) > maxSignals (${tuple.maxSignals})?: ${ - searchResultSize > tuple.maxSignals - }` - ); } catch (exc) { logger.error(buildRuleMessage(`[-] search_after and bulk threw an error ${exc}`)); toReturn.success = false; From 715837b67f14dbef9f8d167760da005cf6cc7e2b Mon Sep 17 00:00:00 2001 From: Devin Hurley Date: Tue, 14 Jul 2020 17:25:46 -0400 Subject: [PATCH 4/5] rename counter to be more descriptive of its purpose --- .../signals/search_after_bulk_create.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts index 1e56c4ac135f9..8e5b7e81c9991 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts @@ -91,7 +91,7 @@ export const searchAfterAndBulkCreate = async ({ }; let sortId; // tells us where to start our next search_after query - let searchResultSize = 0; + let signalsCreatedCount = 0; /* The purpose of `maxResults` is to ensure we do not perform @@ -127,8 +127,8 @@ export const searchAfterAndBulkCreate = async ({ toReturn.success = false; return toReturn; } - searchResultSize = 0; - while (searchResultSize < tuple.maxSignals) { + signalsCreatedCount = 0; + while (signalsCreatedCount < tuple.maxSignals) { try { logger.debug(buildRuleMessage(`sortIds: ${sortId}`)); const { @@ -187,12 +187,12 @@ export const searchAfterAndBulkCreate = async ({ // make sure we are not going to create more signals than maxSignals allows if ( - searchResultSize != null && - searchResultSize + filteredEvents.hits.hits.length > tuple.maxSignals + signalsCreatedCount != null && + signalsCreatedCount + filteredEvents.hits.hits.length > tuple.maxSignals ) { filteredEvents.hits.hits = filteredEvents.hits.hits.slice( 0, - tuple.maxSignals - searchResultSize + tuple.maxSignals - signalsCreatedCount ); } @@ -220,7 +220,7 @@ export const searchAfterAndBulkCreate = async ({ }); logger.debug(buildRuleMessage(`created ${createdCount} signals`)); toReturn.createdSignalsCount += createdCount; - searchResultSize += createdCount; + signalsCreatedCount += createdCount; if (bulkDuration) { toReturn.bulkCreateTimes.push(bulkDuration); } From 890ef86a922f192181aabf8874e3c4d01900ccc0 Mon Sep 17 00:00:00 2001 From: Devin Hurley Date: Tue, 14 Jul 2020 19:22:54 -0400 Subject: [PATCH 5/5] signalsCreatedCount is initialized to a number and should never be null --- .../lib/detection_engine/signals/search_after_bulk_create.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts index 8e5b7e81c9991..cff5b5d2b6be8 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts @@ -186,10 +186,7 @@ export const searchAfterAndBulkCreate = async ({ } // make sure we are not going to create more signals than maxSignals allows - if ( - signalsCreatedCount != null && - signalsCreatedCount + filteredEvents.hits.hits.length > tuple.maxSignals - ) { + if (signalsCreatedCount + filteredEvents.hits.hits.length > tuple.maxSignals) { filteredEvents.hits.hits = filteredEvents.hits.hits.slice( 0, tuple.maxSignals - signalsCreatedCount