From ec08447e208bfd07cd686bd274cb88a8259b9ce2 Mon Sep 17 00:00:00 2001 From: Yara Tercero Date: Mon, 13 Jul 2020 15:07:07 -0400 Subject: [PATCH 1/8] wip - de exceptions logic --- .../build_exceptions_query.test.ts | 156 +++++------------- .../build_exceptions_query.ts | 40 ++--- .../detection_engine/get_query_filter.ts | 4 +- .../signals/single_search_after.ts | 1 + 4 files changed, 59 insertions(+), 142 deletions(-) diff --git a/x-pack/plugins/security_solution/common/detection_engine/build_exceptions_query.test.ts b/x-pack/plugins/security_solution/common/detection_engine/build_exceptions_query.test.ts index ed0344207d18f..28a7b76c35a1d 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/build_exceptions_query.test.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/build_exceptions_query.test.ts @@ -125,7 +125,7 @@ describe('build_exceptions_query', () => { language: 'kuery', }); - expect(query).toEqual('not host.name:suricata'); + expect(query).toEqual('not host.name:"suricata"'); }); test('it returns formatted string when operator is "excluded"', () => { @@ -139,7 +139,7 @@ describe('build_exceptions_query', () => { language: 'kuery', }); - expect(query).toEqual('host.name:suricata'); + expect(query).toEqual('host.name:"suricata"'); }); }); @@ -155,7 +155,7 @@ describe('build_exceptions_query', () => { language: 'lucene', }); - expect(query).toEqual('NOT host.name:suricata'); + expect(query).toEqual('NOT host.name:"suricata"'); }); test('it returns formatted string when operator is "excluded"', () => { @@ -169,7 +169,7 @@ describe('build_exceptions_query', () => { language: 'lucene', }); - expect(query).toEqual('host.name:suricata'); + expect(query).toEqual('host.name:"suricata"'); }); }); }); @@ -201,7 +201,7 @@ describe('build_exceptions_query', () => { language: 'kuery', }); - expect(exceptionSegment).toEqual('not host.name:(suricata)'); + expect(exceptionSegment).toEqual('not host.name:("suricata")'); }); test('it returns formatted string when operator is "included"', () => { @@ -215,7 +215,7 @@ describe('build_exceptions_query', () => { language: 'kuery', }); - expect(exceptionSegment).toEqual('not host.name:(suricata or auditd)'); + expect(exceptionSegment).toEqual('not host.name:("suricata" or "auditd")'); }); test('it returns formatted string when operator is "excluded"', () => { @@ -229,7 +229,7 @@ describe('build_exceptions_query', () => { language: 'kuery', }); - expect(exceptionSegment).toEqual('host.name:(suricata or auditd)'); + expect(exceptionSegment).toEqual('host.name:("suricata" or "auditd")'); }); }); @@ -245,7 +245,7 @@ describe('build_exceptions_query', () => { language: 'lucene', }); - expect(exceptionSegment).toEqual('NOT host.name:(suricata OR auditd)'); + expect(exceptionSegment).toEqual('NOT host.name:("suricata" OR "auditd")'); }); test('it returns formatted string when operator is "excluded"', () => { @@ -259,7 +259,7 @@ describe('build_exceptions_query', () => { language: 'lucene', }); - expect(exceptionSegment).toEqual('host.name:(suricata OR auditd)'); + expect(exceptionSegment).toEqual('host.name:("suricata" OR "auditd")'); }); test('it returns formatted string when "values" includes only one item', () => { @@ -273,7 +273,7 @@ describe('build_exceptions_query', () => { language: 'lucene', }); - expect(exceptionSegment).toEqual('NOT host.name:(suricata)'); + expect(exceptionSegment).toEqual('NOT host.name:("suricata")'); }); }); }); @@ -295,7 +295,7 @@ describe('build_exceptions_query', () => { }; const result = buildNested({ item, language: 'kuery' }); - expect(result).toEqual('parent:{ nestedField:value-3 }'); + expect(result).toEqual('parent:{ nestedField:"value-3" }'); }); test('it returns formatted query when multiple items in nested entry', () => { @@ -319,52 +319,7 @@ describe('build_exceptions_query', () => { }; const result = buildNested({ item, language: 'kuery' }); - expect(result).toEqual('parent:{ nestedField:value-3 and nestedFieldB:value-4 }'); - }); - }); - - // TODO: Does lucene support nested query syntax? - describe.skip('lucene', () => { - test('it returns formatted query when one item in nested entry', () => { - const item: EntryNested = { - field: 'parent', - type: 'nested', - entries: [ - { - field: 'nestedField', - operator: 'excluded', - type: 'match', - value: 'value-3', - }, - ], - }; - const result = buildNested({ item, language: 'lucene' }); - - expect(result).toEqual('parent:{ nestedField:value-3 }'); - }); - - test('it returns formatted query when multiple items in nested entry', () => { - const item: EntryNested = { - field: 'parent', - type: 'nested', - entries: [ - { - field: 'nestedField', - operator: 'excluded', - type: 'match', - value: 'value-3', - }, - { - field: 'nestedFieldB', - operator: 'excluded', - type: 'match', - value: 'value-4', - }, - ], - }; - const result = buildNested({ item, language: 'lucene' }); - - expect(result).toEqual('parent:{ nestedField:value-3 AND nestedFieldB:value-4 }'); + expect(result).toEqual('parent:{ nestedField:"value-3" and nestedFieldB:"value-4" }'); }); }); }); @@ -397,7 +352,7 @@ describe('build_exceptions_query', () => { language: 'kuery', }); - expect(result).toEqual('not host.name:suricata'); + expect(result).toEqual('not host.name:"suricata"'); }); test('it returns formatted string when "type" is "match_any"', () => { @@ -413,7 +368,7 @@ describe('build_exceptions_query', () => { language: 'kuery', }); - expect(result).toEqual('not host.name:(suricata or auditd)'); + expect(result).toEqual('not host.name:("suricata" or "auditd")'); }); }); @@ -445,7 +400,7 @@ describe('build_exceptions_query', () => { language: 'lucene', }); - expect(result).toEqual('NOT host.name:suricata'); + expect(result).toEqual('NOT host.name:"suricata"'); }); test('it returns formatted string when "type" is "match_any"', () => { @@ -461,7 +416,7 @@ describe('build_exceptions_query', () => { language: 'lucene', }); - expect(result).toEqual('NOT host.name:(suricata OR auditd)'); + expect(result).toEqual('NOT host.name:("suricata" OR "auditd")'); }); }); }); @@ -476,24 +431,22 @@ describe('build_exceptions_query', () => { test('it returns expected query string when single exception in array', () => { const formattedQuery = formatQuery({ - exceptions: ['b:(value-1 or value-2) and not c:*'], + exceptions: ['b:("value-1" or "value-2") and not c:*'], query: 'a:*', language: 'kuery', }); - expect(formattedQuery).toEqual('(a:* and b:(value-1 or value-2) and not c:*)'); + expect(formattedQuery).toEqual('a:* and b:("value-1" or "value-2") and not c:*'); }); test('it returns expected query string when multiple exceptions in array', () => { const formattedQuery = formatQuery({ - exceptions: ['b:(value-1 or value-2) and not c:*', 'not d:*'], + exceptions: ['b:("value-1" or "value-2") and not c:*', 'not d:*'], query: 'a:*', language: 'kuery', }); - expect(formattedQuery).toEqual( - '(a:* and b:(value-1 or value-2) and not c:*) or (a:* and not d:*)' - ); + expect(formattedQuery).toEqual('a:* and b:("value-1" or "value-2") and not c:* and not d:*'); }); }); @@ -508,8 +461,6 @@ describe('build_exceptions_query', () => { }); test('it returns expected query when more than one item in list', () => { - // Equal to query && !(b && !c) -> (query AND NOT b) OR (query AND c) - // https://www.dcode.fr/boolean-expressions-calculator const payload: EntriesArray = [ { field: 'b', @@ -528,14 +479,12 @@ describe('build_exceptions_query', () => { language: 'kuery', lists: payload, }); - const expectedQuery = 'not b:(value-1 or value-2) and c:value-3'; + const expectedQuery = 'not b:("value-1" or "value-2") and c:"value-3"'; expect(query).toEqual(expectedQuery); }); test('it returns expected query when list item includes nested value', () => { - // Equal to query && !(b || !c) -> (query AND NOT b AND c) - // https://www.dcode.fr/boolean-expressions-calculator const lists: EntriesArray = [ { field: 'b', @@ -560,14 +509,12 @@ describe('build_exceptions_query', () => { language: 'kuery', lists, }); - const expectedQuery = 'not b:(value-1 or value-2) and parent:{ nestedField:value-3 }'; + const expectedQuery = 'not b:("value-1" or "value-2") and parent:{ nestedField:"value-3" }'; expect(query).toEqual(expectedQuery); }); test('it returns expected query when list includes multiple items and nested "and" values', () => { - // Equal to query && !((b || !c) && d) -> (query AND NOT b AND c) OR (query AND NOT d) - // https://www.dcode.fr/boolean-expressions-calculator const lists: EntriesArray = [ { field: 'b', @@ -598,13 +545,11 @@ describe('build_exceptions_query', () => { lists, }); const expectedQuery = - 'not b:(value-1 or value-2) and parent:{ nestedField:value-3 } and not d:*'; + 'not b:("value-1" or "value-2") and parent:{ nestedField:"value-3" } and not d:*'; expect(query).toEqual(expectedQuery); }); test('it returns expected query when language is "lucene"', () => { - // Equal to query && !((b || !c) && !d) -> (query AND NOT b AND c) OR (query AND d) - // https://www.dcode.fr/boolean-expressions-calculator const lists: EntriesArray = [ { field: 'b', @@ -635,14 +580,12 @@ describe('build_exceptions_query', () => { lists, }); const expectedQuery = - 'NOT b:(value-1 OR value-2) AND parent:{ nestedField:value-3 } AND _exists_e'; + 'NOT b:("value-1" OR "value-2") AND parent:{ nestedField:"value-3" } AND _exists_e'; expect(query).toEqual(expectedQuery); }); describe('exists', () => { test('it returns expected query when list includes single list item with operator of "included"', () => { - // Equal to query && !(b) -> (query AND NOT b) - // https://www.dcode.fr/boolean-expressions-calculator const lists: EntriesArray = [ { field: 'b', @@ -660,8 +603,6 @@ describe('build_exceptions_query', () => { }); test('it returns expected query when list includes single list item with operator of "excluded"', () => { - // Equal to query && !(!b) -> (query AND b) - // https://www.dcode.fr/boolean-expressions-calculator const lists: EntriesArray = [ { field: 'b', @@ -679,8 +620,6 @@ describe('build_exceptions_query', () => { }); test('it returns expected query when list includes list item with "and" values', () => { - // Equal to query && !(!b || !c) -> (query AND b AND c) - // https://www.dcode.fr/boolean-expressions-calculator const lists: EntriesArray = [ { field: 'b', @@ -704,14 +643,12 @@ describe('build_exceptions_query', () => { language: 'kuery', lists, }); - const expectedQuery = 'b:* and parent:{ c:value-1 }'; + const expectedQuery = 'b:* and parent:{ c:"value-1" }'; expect(query).toEqual(expectedQuery); }); test('it returns expected query when list includes multiple items', () => { - // Equal to query && !((b || !c || d) && e) -> (query AND NOT b AND c AND NOT d) OR (query AND NOT e) - // https://www.dcode.fr/boolean-expressions-calculator const lists: EntriesArray = [ { field: 'b', @@ -746,7 +683,7 @@ describe('build_exceptions_query', () => { language: 'kuery', lists, }); - const expectedQuery = 'not b:* and parent:{ c:value-1 and d:value-2 } and not e:*'; + const expectedQuery = 'not b:* and parent:{ c:"value-1" and d:"value-2" } and not e:*'; expect(query).toEqual(expectedQuery); }); @@ -754,8 +691,6 @@ describe('build_exceptions_query', () => { describe('match', () => { test('it returns expected query when list includes single list item with operator of "included"', () => { - // Equal to query && !(b) -> (query AND NOT b) - // https://www.dcode.fr/boolean-expressions-calculator const lists: EntriesArray = [ { field: 'b', @@ -768,14 +703,12 @@ describe('build_exceptions_query', () => { language: 'kuery', lists, }); - const expectedQuery = 'not b:value'; + const expectedQuery = 'not b:"value"'; expect(query).toEqual(expectedQuery); }); test('it returns expected query when list includes single list item with operator of "excluded"', () => { - // Equal to query && !(!b) -> (query AND b) - // https://www.dcode.fr/boolean-expressions-calculator const lists: EntriesArray = [ { field: 'b', @@ -788,14 +721,12 @@ describe('build_exceptions_query', () => { language: 'kuery', lists, }); - const expectedQuery = 'b:value'; + const expectedQuery = 'b:"value"'; expect(query).toEqual(expectedQuery); }); test('it returns expected query when list includes list item with "and" values', () => { - // Equal to query && !(!b || !c) -> (query AND b AND c) - // https://www.dcode.fr/boolean-expressions-calculator const lists: EntriesArray = [ { field: 'b', @@ -820,14 +751,12 @@ describe('build_exceptions_query', () => { language: 'kuery', lists, }); - const expectedQuery = 'b:value and parent:{ c:valueC }'; + const expectedQuery = 'b:"value" and parent:{ c:"valueC" }'; expect(query).toEqual(expectedQuery); }); test('it returns expected query when list includes multiple items', () => { - // Equal to query && !((b || !c || d) && e) -> (query AND NOT b AND c AND NOT d) OR (query AND NOT e) - // https://www.dcode.fr/boolean-expressions-calculator const lists: EntriesArray = [ { field: 'b', @@ -864,7 +793,8 @@ describe('build_exceptions_query', () => { language: 'kuery', lists, }); - const expectedQuery = 'not b:value and parent:{ c:valueC and d:valueC } and not e:valueC'; + const expectedQuery = + 'not b:"value" and parent:{ c:"valueC" and d:"valueC" } and not e:"valueC"'; expect(query).toEqual(expectedQuery); }); @@ -872,8 +802,6 @@ describe('build_exceptions_query', () => { describe('match_any', () => { test('it returns expected query when list includes single list item with operator of "included"', () => { - // Equal to query && !(b) -> (query AND NOT b) - // https://www.dcode.fr/boolean-expressions-calculator const lists: EntriesArray = [ { field: 'b', @@ -886,14 +814,12 @@ describe('build_exceptions_query', () => { language: 'kuery', lists, }); - const expectedQuery = 'not b:(value or value-1)'; + const expectedQuery = 'not b:("value" or "value-1")'; expect(query).toEqual(expectedQuery); }); test('it returns expected query when list includes single list item with operator of "excluded"', () => { - // Equal to query && !(!b) -> (query AND b) - // https://www.dcode.fr/boolean-expressions-calculator const lists: EntriesArray = [ { field: 'b', @@ -906,14 +832,12 @@ describe('build_exceptions_query', () => { language: 'kuery', lists, }); - const expectedQuery = 'b:(value or value-1)'; + const expectedQuery = 'b:("value" or "value-1")'; expect(query).toEqual(expectedQuery); }); test('it returns expected query when list includes list item with nested values', () => { - // Equal to query && !(!b || c) -> (query AND b AND NOT c) - // https://www.dcode.fr/boolean-expressions-calculator const lists: EntriesArray = [ { field: 'b', @@ -938,14 +862,12 @@ describe('build_exceptions_query', () => { language: 'kuery', lists, }); - const expectedQuery = 'b:(value or value-1) and parent:{ c:valueC }'; + const expectedQuery = 'b:("value" or "value-1") and parent:{ c:"valueC" }'; expect(query).toEqual(expectedQuery); }); test('it returns expected query when list includes multiple items', () => { - // Equal to query && !((b || !c || d) && e) -> ((query AND NOT b AND c AND NOT d) OR (query AND NOT e) - // https://www.dcode.fr/boolean-expressions-calculator const lists: EntriesArray = [ { field: 'b', @@ -964,7 +886,7 @@ describe('build_exceptions_query', () => { language: 'kuery', lists, }); - const expectedQuery = 'not b:(value or value-1) and not e:(valueE or value-4)'; + const expectedQuery = 'not b:("value" or "value-1") and not e:("valueE" or "value-4")'; expect(query).toEqual(expectedQuery); }); @@ -980,8 +902,6 @@ describe('build_exceptions_query', () => { }); test('it returns expected query when lists exist and language is "kuery"', () => { - // Equal to query && !((b || !c || d) && e) -> ((query AND NOT b AND c AND NOT d) OR (query AND NOT e) - // https://www.dcode.fr/boolean-expressions-calculator const payload = getExceptionListItemSchemaMock(); const payload2 = getExceptionListItemSchemaMock(); payload2.entries = [ @@ -1022,14 +942,12 @@ describe('build_exceptions_query', () => { lists: [payload, payload2], }); const expectedQuery = - '(a:* and some.parentField:{ nested.field:some value } and not some.not.nested.field:some value) or (a:* and not b:(value or value-1) and parent:{ c:valueC and d:valueD } and not e:(valueE or value-4))'; + 'a:* and some.parentField:{ nested.field:"some value" } and not some.not.nested.field:"some value" and not b:("value" or "value-1") and parent:{ c:"valueC" and d:"valueD" } and not e:("valueE" or "value-4")'; expect(query).toEqual([{ query: expectedQuery, language: 'kuery' }]); }); test('it returns expected query when lists exist and language is "lucene"', () => { - // Equal to query && !((b || !c || d) && e) -> ((query AND NOT b AND c AND NOT d) OR (query AND NOT e) - // https://www.dcode.fr/boolean-expressions-calculator const payload = getExceptionListItemSchemaMock(); const payload2 = getExceptionListItemSchemaMock(); payload2.entries = [ @@ -1070,7 +988,7 @@ describe('build_exceptions_query', () => { lists: [payload, payload2], }); const expectedQuery = - '(a:* AND some.parentField:{ nested.field:some value } AND NOT some.not.nested.field:some value) OR (a:* AND NOT b:(value OR value-1) AND parent:{ c:valueC AND d:valueD } AND NOT e:(valueE OR value-4))'; + 'a:* AND some.parentField:{ nested.field:"some value" } AND NOT some.not.nested.field:"some value" AND NOT b:("value" OR "value-1") AND parent:{ c:"valueC" AND d:"valueD" } AND NOT e:("valueE" OR "value-4")'; expect(query).toEqual([{ query: expectedQuery, language: 'lucene' }]); }); diff --git a/x-pack/plugins/security_solution/common/detection_engine/build_exceptions_query.ts b/x-pack/plugins/security_solution/common/detection_engine/build_exceptions_query.ts index a69ee809987f7..5fa64ee73b256 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/build_exceptions_query.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/build_exceptions_query.ts @@ -18,7 +18,7 @@ import { entriesNested, ExceptionListItemSchema, } from '../../../lists/common/schemas'; -import { Language, Query } from './schemas/common/schemas'; +import { Language } from './schemas/common/schemas'; type Operators = 'and' | 'or' | 'not'; type LuceneOperators = 'AND' | 'OR' | 'NOT'; @@ -92,7 +92,7 @@ export const buildMatch = ({ const { value, operator, field } = item; const exceptionOperator = operatorBuilder({ operator, language }); - return `${exceptionOperator}${field}:${value}`; + return `${exceptionOperator}${field}:"${value}"`; }; export const buildMatchAny = ({ @@ -110,7 +110,7 @@ export const buildMatchAny = ({ default: const or = getLanguageBooleanOperator({ language, value: 'or' }); const exceptionOperator = operatorBuilder({ operator, language }); - const matchAnyValues = value.map((v) => v); + const matchAnyValues = value.map((v) => `"${v}"`); return `${exceptionOperator}${field}:(${matchAnyValues.join(` ${or} `)})`; } @@ -125,7 +125,7 @@ export const buildNested = ({ }): string => { const { field, entries } = item; const and = getLanguageBooleanOperator({ language, value: 'and' }); - const values = entries.map((entry) => `${entry.field}:${entry.value}`); + const values = entries.map((entry) => `${entry.field}:"${entry.value}"`); return `${field}:{ ${values.join(` ${and} `)} }`; }; @@ -152,24 +152,21 @@ export const evaluateValues = ({ export const formatQuery = ({ exceptions, - query, language, }: { exceptions: string[]; - query: string; language: Language; }): string => { - if (exceptions.length > 0) { - const or = getLanguageBooleanOperator({ language, value: 'or' }); - const and = getLanguageBooleanOperator({ language, value: 'and' }); - const formattedExceptions = exceptions.map((exception) => { - return `(${query} ${and} ${exception})`; - }); - - return formattedExceptions.join(` ${or} `); - } else { - return query; - } + const and = getLanguageBooleanOperator({ language, value: 'and' }); + const formattedExceptions = exceptions.map((exception, index) => { + if (index === 0) { + return exception; + } + + return `${and} ${exception}`; + }); + + return formattedExceptions.join(' '); }; export const buildExceptionItemEntries = ({ @@ -191,19 +188,18 @@ export const buildExceptionItemEntries = ({ }; export const buildQueryExceptions = ({ - query, language, lists, }: { - query: Query; language: Language; lists: ExceptionListItemSchema[] | undefined; }): DataQuery[] => { - if (lists != null) { + if (lists != null && lists.length > 0) { const exceptions = lists.map((exceptionItem) => buildExceptionItemEntries({ lists: exceptionItem.entries, language }) ); - const formattedQuery = formatQuery({ exceptions, language, query }); + const formattedQuery = formatQuery({ exceptions, language }); + return [ { query: formattedQuery, @@ -211,6 +207,6 @@ export const buildQueryExceptions = ({ }, ]; } else { - return [{ query, language }]; + return []; } }; diff --git a/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.ts b/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.ts index ef390c3b44939..a0c6ef369c2dc 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.ts @@ -27,7 +27,9 @@ export const getQueryFilter = ( title: index.join(), }; - const queries: DataQuery[] = buildQueryExceptions({ query, language, lists }); + const initialQuery = [{ query, language }]; + const exceptions = buildQueryExceptions({ language, lists }); + const queries: DataQuery[] = [...initialQuery, ...exceptions]; const config = { allowLeadingWildcards: true, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.ts index 409f374d7df1e..fc1cf26dc9564 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.ts @@ -45,6 +45,7 @@ export const singleSearchAfter = async ({ size: pageSize, searchAfterSortId, }); + const start = performance.now(); const nextSearchAfterResult: SignalSearchResponse = await services.callCluster( 'search', From 54f417105e1630f4c2d06b5a130854251123f251 Mon Sep 17 00:00:00 2001 From: Yara Tercero Date: Tue, 14 Jul 2020 02:34:17 -0400 Subject: [PATCH 2/8] wip - updated filter events with lists, when item did not include large value list was blocking signal creation --- .../scripts/lists/new/items/ip_item.json | 2 +- .../signals/filter_events_with_list.ts | 20 ++++++++++++++++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/lists/server/scripts/lists/new/items/ip_item.json b/x-pack/plugins/lists/server/scripts/lists/new/items/ip_item.json index 563139c40c0ca..c2238890496bb 100644 --- a/x-pack/plugins/lists/server/scripts/lists/new/items/ip_item.json +++ b/x-pack/plugins/lists/server/scripts/lists/new/items/ip_item.json @@ -1,5 +1,5 @@ { "id": "ip_item", "list_id": "ip_list", - "value": "10.4.2.140" + "value": "127.0.0.1" } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filter_events_with_list.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filter_events_with_list.ts index f16de8bf05ef4..8aa3ca3393f49 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filter_events_with_list.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filter_events_with_list.ts @@ -14,6 +14,7 @@ import { EntryList, ExceptionListItemSchema, } from '../../../../../lists/common/schemas'; +import { hasLargeValueList } from './utils'; interface FilterEventsAgainstList { listClient: ListClient; @@ -37,11 +38,28 @@ export const filterEventsAgainstList = async ({ return eventSearchResult; } + const exceptionItemsWithLargeValueLists = exceptionsList.reduce( + (acc, exception) => { + const { entries } = exception; + if (hasLargeValueList(entries)) { + return [...acc, exception]; + } + + return acc; + }, + [] + ); + + if (exceptionItemsWithLargeValueLists.length === 0) { + logger.debug(buildRuleMessage('about to return original search result')); + return eventSearchResult; + } + // narrow unioned type to be single const isStringableType = (val: SearchTypes) => ['string', 'number', 'boolean'].includes(typeof val); // grab the signals with values found in the given exception lists. - const filteredHitsPromises = exceptionsList.map( + const filteredHitsPromises = exceptionItemsWithLargeValueLists.map( async (exceptionItem: ExceptionListItemSchema) => { const { entries } = exceptionItem; From 229bccfaae6a62211df41b3d5ff7a38125e30840 Mon Sep 17 00:00:00 2001 From: Yara Tercero Date: Tue, 14 Jul 2020 08:44:40 -0400 Subject: [PATCH 3/8] updated unit tests post merge conflicts --- .../build_exceptions_query.test.ts | 165 ++++++++---------- .../build_exceptions_query.ts | 2 +- .../detection_engine/get_query_filter.test.ts | 120 +++++-------- 3 files changed, 114 insertions(+), 173 deletions(-) diff --git a/x-pack/plugins/security_solution/common/detection_engine/build_exceptions_query.test.ts b/x-pack/plugins/security_solution/common/detection_engine/build_exceptions_query.test.ts index d84903a763533..c728c2ad83312 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/build_exceptions_query.test.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/build_exceptions_query.test.ts @@ -260,7 +260,7 @@ describe('build_exceptions_query', () => { language: 'kuery', exclude, }); - expect(query).toEqual('not host.name:suricata'); + expect(query).toEqual('not host.name:"suricata"'); }); test('it returns formatted string when operator is "excluded"', () => { const query = buildMatch({ @@ -268,7 +268,7 @@ describe('build_exceptions_query', () => { language: 'kuery', exclude, }); - expect(query).toEqual('host.name:suricata'); + expect(query).toEqual('host.name:"suricata"'); }); }); @@ -279,7 +279,7 @@ describe('build_exceptions_query', () => { language: 'lucene', exclude, }); - expect(query).toEqual('NOT host.name:suricata'); + expect(query).toEqual('NOT host.name:"suricata"'); }); test('it returns formatted string when operator is "excluded"', () => { const query = buildMatch({ @@ -287,7 +287,7 @@ describe('build_exceptions_query', () => { language: 'lucene', exclude, }); - expect(query).toEqual('host.name:suricata'); + expect(query).toEqual('host.name:"suricata"'); }); }); }); @@ -304,7 +304,7 @@ describe('build_exceptions_query', () => { language: 'kuery', exclude, }); - expect(query).toEqual('host.name:suricata'); + expect(query).toEqual('host.name:"suricata"'); }); test('it returns formatted string when operator is "excluded"', () => { const query = buildMatch({ @@ -312,7 +312,7 @@ describe('build_exceptions_query', () => { language: 'kuery', exclude, }); - expect(query).toEqual('not host.name:suricata'); + expect(query).toEqual('not host.name:"suricata"'); }); }); @@ -323,7 +323,7 @@ describe('build_exceptions_query', () => { language: 'lucene', exclude, }); - expect(query).toEqual('host.name:suricata'); + expect(query).toEqual('host.name:"suricata"'); }); test('it returns formatted string when operator is "excluded"', () => { const query = buildMatch({ @@ -331,7 +331,7 @@ describe('build_exceptions_query', () => { language: 'lucene', exclude, }); - expect(query).toEqual('NOT host.name:suricata'); + expect(query).toEqual('NOT host.name:"suricata"'); }); }); }); @@ -362,21 +362,23 @@ describe('build_exceptions_query', () => { }); expect(exceptionSegment).toEqual(''); }); + test('it returns formatted string when "values" includes only one item', () => { const exceptionSegment = buildMatchAny({ item: entryWithIncludedAndOneValue, language: 'kuery', exclude, }); - expect(exceptionSegment).toEqual('not host.name:(suricata)'); + expect(exceptionSegment).toEqual('not host.name:("suricata")'); }); + test('it returns formatted string when operator is "included"', () => { const exceptionSegment = buildMatchAny({ item: matchAnyEntryWithIncludedAndTwoValues, language: 'kuery', exclude, }); - expect(exceptionSegment).toEqual('not host.name:(suricata or auditd)'); + expect(exceptionSegment).toEqual('not host.name:("suricata" or "auditd")'); }); test('it returns formatted string when operator is "excluded"', () => { @@ -385,7 +387,7 @@ describe('build_exceptions_query', () => { language: 'kuery', exclude, }); - expect(exceptionSegment).toEqual('host.name:(suricata or auditd)'); + expect(exceptionSegment).toEqual('host.name:("suricata" or "auditd")'); }); }); @@ -396,7 +398,7 @@ describe('build_exceptions_query', () => { language: 'lucene', exclude, }); - expect(exceptionSegment).toEqual('NOT host.name:(suricata OR auditd)'); + expect(exceptionSegment).toEqual('NOT host.name:("suricata" OR "auditd")'); }); test('it returns formatted string when operator is "excluded"', () => { const exceptionSegment = buildMatchAny({ @@ -404,7 +406,7 @@ describe('build_exceptions_query', () => { language: 'lucene', exclude, }); - expect(exceptionSegment).toEqual('host.name:(suricata OR auditd)'); + expect(exceptionSegment).toEqual('host.name:("suricata" OR "auditd")'); }); test('it returns formatted string when "values" includes only one item', () => { const exceptionSegment = buildMatchAny({ @@ -412,7 +414,7 @@ describe('build_exceptions_query', () => { language: 'lucene', exclude, }); - expect(exceptionSegment).toEqual('NOT host.name:(suricata)'); + expect(exceptionSegment).toEqual('NOT host.name:("suricata")'); }); }); }); @@ -437,7 +439,7 @@ describe('build_exceptions_query', () => { language: 'kuery', exclude, }); - expect(exceptionSegment).toEqual('host.name:(suricata)'); + expect(exceptionSegment).toEqual('host.name:("suricata")'); }); test('it returns formatted string when operator is "included"', () => { const exceptionSegment = buildMatchAny({ @@ -445,7 +447,7 @@ describe('build_exceptions_query', () => { language: 'kuery', exclude, }); - expect(exceptionSegment).toEqual('host.name:(suricata or auditd)'); + expect(exceptionSegment).toEqual('host.name:("suricata" or "auditd")'); }); test('it returns formatted string when operator is "excluded"', () => { @@ -454,7 +456,7 @@ describe('build_exceptions_query', () => { language: 'kuery', exclude, }); - expect(exceptionSegment).toEqual('not host.name:(suricata or auditd)'); + expect(exceptionSegment).toEqual('not host.name:("suricata" or "auditd")'); }); }); @@ -465,7 +467,7 @@ describe('build_exceptions_query', () => { language: 'lucene', exclude, }); - expect(exceptionSegment).toEqual('host.name:(suricata OR auditd)'); + expect(exceptionSegment).toEqual('host.name:("suricata" OR "auditd")'); }); test('it returns formatted string when operator is "excluded"', () => { const exceptionSegment = buildMatchAny({ @@ -473,7 +475,7 @@ describe('build_exceptions_query', () => { language: 'lucene', exclude, }); - expect(exceptionSegment).toEqual('NOT host.name:(suricata OR auditd)'); + expect(exceptionSegment).toEqual('NOT host.name:("suricata" OR "auditd")'); }); test('it returns formatted string when "values" includes only one item', () => { const exceptionSegment = buildMatchAny({ @@ -481,7 +483,7 @@ describe('build_exceptions_query', () => { language: 'lucene', exclude, }); - expect(exceptionSegment).toEqual('host.name:(suricata)'); + expect(exceptionSegment).toEqual('host.name:("suricata")'); }); }); }); @@ -497,7 +499,7 @@ describe('build_exceptions_query', () => { }; const result = buildNested({ item, language: 'kuery' }); - expect(result).toEqual('parent:{ nestedField:value-1 }'); + expect(result).toEqual('parent:{ nestedField:"value-1" }'); }); test('it returns formatted query when multiple items in nested entry', () => { @@ -511,7 +513,7 @@ describe('build_exceptions_query', () => { }; const result = buildNested({ item, language: 'kuery' }); - expect(result).toEqual('parent:{ nestedField:value-1 and nestedFieldB:value-2 }'); + expect(result).toEqual('parent:{ nestedField:"value-1" and nestedFieldB:"value-2" }'); }); }); @@ -525,7 +527,7 @@ describe('build_exceptions_query', () => { }; const result = buildNested({ item, language: 'lucene' }); - expect(result).toEqual('parent:{ nestedField:value-1 }'); + expect(result).toEqual('parent:{ nestedField:"value-1" }'); }); test('it returns formatted query when multiple items in nested entry', () => { @@ -539,7 +541,7 @@ describe('build_exceptions_query', () => { }; const result = buildNested({ item, language: 'lucene' }); - expect(result).toEqual('parent:{ nestedField:value-1 AND nestedFieldB:value-2 }'); + expect(result).toEqual('parent:{ nestedField:"value-1" AND nestedFieldB:"value-2" }'); }); }); }); @@ -555,21 +557,23 @@ describe('build_exceptions_query', () => { }); expect(result).toEqual('not host.name:*'); }); + test('it returns formatted string when "type" is "match"', () => { const result = evaluateValues({ item: matchEntryWithIncluded, language: 'kuery', exclude, }); - expect(result).toEqual('not host.name:suricata'); + expect(result).toEqual('not host.name:"suricata"'); }); + test('it returns formatted string when "type" is "match_any"', () => { const result = evaluateValues({ item: matchAnyEntryWithIncludedAndTwoValues, language: 'kuery', exclude, }); - expect(result).toEqual('not host.name:(suricata or auditd)'); + expect(result).toEqual('not host.name:("suricata" or "auditd")'); }); }); @@ -583,21 +587,23 @@ describe('build_exceptions_query', () => { }); expect(result).toEqual('NOT _exists_host.name'); }); + test('it returns formatted string when "type" is "match"', () => { const result = evaluateValues({ item: matchEntryWithIncluded, language: 'lucene', exclude, }); - expect(result).toEqual('NOT host.name:suricata'); + expect(result).toEqual('NOT host.name:"suricata"'); }); + test('it returns formatted string when "type" is "match_any"', () => { const result = evaluateValues({ item: matchAnyEntryWithIncludedAndTwoValues, language: 'lucene', exclude, }); - expect(result).toEqual('NOT host.name:(suricata OR auditd)'); + expect(result).toEqual('NOT host.name:("suricata" OR "auditd")'); }); }); }); @@ -617,21 +623,23 @@ describe('build_exceptions_query', () => { }); expect(result).toEqual('host.name:*'); }); + test('it returns formatted string when "type" is "match"', () => { const result = evaluateValues({ item: matchEntryWithIncluded, language: 'kuery', exclude, }); - expect(result).toEqual('host.name:suricata'); + expect(result).toEqual('host.name:"suricata"'); }); + test('it returns formatted string when "type" is "match_any"', () => { const result = evaluateValues({ item: matchAnyEntryWithIncludedAndTwoValues, language: 'kuery', exclude, }); - expect(result).toEqual('host.name:(suricata or auditd)'); + expect(result).toEqual('host.name:("suricata" or "auditd")'); }); }); @@ -645,21 +653,23 @@ describe('build_exceptions_query', () => { }); expect(result).toEqual('_exists_host.name'); }); + test('it returns formatted string when "type" is "match"', () => { const result = evaluateValues({ item: matchEntryWithIncluded, language: 'lucene', exclude, }); - expect(result).toEqual('host.name:suricata'); + expect(result).toEqual('host.name:"suricata"'); }); + test('it returns formatted string when "type" is "match_any"', () => { const result = evaluateValues({ item: matchAnyEntryWithIncludedAndTwoValues, language: 'lucene', exclude, }); - expect(result).toEqual('host.name:(suricata OR auditd)'); + expect(result).toEqual('host.name:("suricata" OR "auditd")'); }); }); }); @@ -668,43 +678,34 @@ describe('build_exceptions_query', () => { describe('formatQuery', () => { describe('when query is empty string', () => { - test('it returns query if "exceptions" is empty array', () => { - const formattedQuery = formatQuery({ exceptions: [], query: '', language: 'kuery' }); + test('it returns empty string if "exceptions" is empty array', () => { + const formattedQuery = formatQuery({ exceptions: [], language: 'kuery' }); expect(formattedQuery).toEqual(''); }); + test('it returns expected query string when single exception in array', () => { const formattedQuery = formatQuery({ - exceptions: ['b:(value-1 or value-2) and not c:*'], - query: '', + exceptions: ['b:("value-1" or "value-2") and not c:*'], language: 'kuery', }); - expect(formattedQuery).toEqual('(b:(value-1 or value-2) and not c:*)'); + expect(formattedQuery).toEqual('b:("value-1" or "value-2") and not c:*'); }); }); - test('it returns query if "exceptions" is empty array', () => { - const formattedQuery = formatQuery({ exceptions: [], query: 'a:*', language: 'kuery' }); - expect(formattedQuery).toEqual('a:*'); - }); - test('it returns expected query string when single exception in array', () => { const formattedQuery = formatQuery({ exceptions: ['b:("value-1" or "value-2") and not c:*'], - query: 'a:*', language: 'kuery', }); - expect(formattedQuery).toEqual('(a:* and b:(value-1 or value-2) and not c:*)'); + expect(formattedQuery).toEqual('b:("value-1" or "value-2") and not c:*'); }); test('it returns expected query string when multiple exceptions in array', () => { const formattedQuery = formatQuery({ exceptions: ['b:("value-1" or "value-2") and not c:*', 'not d:*'], - query: 'a:*', language: 'kuery', }); - expect(formattedQuery).toEqual( - '(a:* and b:(value-1 or value-2) and not c:*) or (a:* and not d:*)' - ); + expect(formattedQuery).toEqual('b:("value-1" or "value-2") and not c:* and not d:*'); }); }); @@ -813,9 +814,8 @@ describe('build_exceptions_query', () => { expect(query).toEqual(''); }); + test('it returns expected query when more than one item in list', () => { - // Equal to query && !(b && !c) -> (query AND NOT b) OR (query AND c) - // https://www.dcode.fr/boolean-expressions-calculator const payload: EntriesArray = [ makeMatchAnyEntry({ field: 'b' }), makeMatchEntry({ field: 'c', operator: 'excluded', value: 'value-3' }), @@ -825,14 +825,12 @@ describe('build_exceptions_query', () => { lists: payload, exclude, }); - const expectedQuery = 'b:(value-1 or value-2) and not c:value-3'; + const expectedQuery = 'b:("value-1" or "value-2") and not c:"value-3"'; expect(query).toEqual(expectedQuery); }); test('it returns expected query when list item includes nested value', () => { - // Equal to query && !(b || !c) -> (query AND NOT b AND c) - // https://www.dcode.fr/boolean-expressions-calculator const lists: EntriesArray = [ makeMatchAnyEntry({ field: 'b' }), { @@ -848,14 +846,12 @@ describe('build_exceptions_query', () => { lists, exclude, }); - const expectedQuery = 'b:(value-1 or value-2) and parent:{ nestedField:value-3 }'; + const expectedQuery = 'b:("value-1" or "value-2") and parent:{ nestedField:"value-3" }'; expect(query).toEqual(expectedQuery); }); test('it returns expected query when list includes multiple items and nested "and" values', () => { - // Equal to query && !((b || !c) && d) -> (query AND NOT b AND c) OR (query AND NOT d) - // https://www.dcode.fr/boolean-expressions-calculator const lists: EntriesArray = [ makeMatchAnyEntry({ field: 'b' }), { @@ -872,13 +868,12 @@ describe('build_exceptions_query', () => { lists, exclude, }); - const expectedQuery = 'b:(value-1 or value-2) and parent:{ nestedField:value-3 } and d:*'; + const expectedQuery = + 'b:("value-1" or "value-2") and parent:{ nestedField:"value-3" } and d:*'; expect(query).toEqual(expectedQuery); }); test('it returns expected query when language is "lucene"', () => { - // Equal to query && !((b || !c) && !d) -> (query AND NOT b AND c) OR (query AND d) - // https://www.dcode.fr/boolean-expressions-calculator const lists: EntriesArray = [ makeMatchAnyEntry({ field: 'b' }), { @@ -896,15 +891,13 @@ describe('build_exceptions_query', () => { exclude, }); const expectedQuery = - 'b:(value-1 OR value-2) AND parent:{ nestedField:value-3 } AND NOT _exists_e'; + 'b:("value-1" OR "value-2") AND parent:{ nestedField:"value-3" } AND NOT _exists_e'; expect(query).toEqual(expectedQuery); }); }); describe('exists', () => { test('it returns expected query when list includes single list item with operator of "included"', () => { - // Equal to query && !(b) -> (query AND NOT b) - // https://www.dcode.fr/boolean-expressions-calculator const lists: EntriesArray = [makeExistsEntry({ field: 'b' })]; const query = buildExceptionItemEntries({ language: 'kuery', @@ -917,8 +910,6 @@ describe('build_exceptions_query', () => { }); test('it returns expected query when list includes single list item with operator of "excluded"', () => { - // Equal to query && !(!b) -> (query AND b) - // https://www.dcode.fr/boolean-expressions-calculator const lists: EntriesArray = [makeExistsEntry({ field: 'b', operator: 'excluded' })]; const query = buildExceptionItemEntries({ language: 'kuery', @@ -975,8 +966,6 @@ describe('build_exceptions_query', () => { describe('match', () => { test('it returns expected query when list includes single list item with operator of "included"', () => { - // Equal to query && !(b) -> (query AND NOT b) - // https://www.dcode.fr/boolean-expressions-calculator const lists: EntriesArray = [makeMatchEntry({ field: 'b', value: 'value' })]; const query = buildExceptionItemEntries({ language: 'kuery', @@ -1039,7 +1028,8 @@ describe('build_exceptions_query', () => { lists, exclude, }); - const expectedQuery = 'not b:value and parent:{ c:valueC and d:valueD } and not e:valueE'; + const expectedQuery = + 'not b:"value" and parent:{ c:"valueC" and d:"valueD" } and not e:"valueE"'; expect(query).toEqual(expectedQuery); }); @@ -1047,29 +1037,25 @@ describe('build_exceptions_query', () => { describe('match_any', () => { test('it returns expected query when list includes single list item with operator of "included"', () => { - // Equal to query && !(b) -> (query AND NOT b) - // https://www.dcode.fr/boolean-expressions-calculator const lists: EntriesArray = [makeMatchAnyEntry({ field: 'b' })]; const query = buildExceptionItemEntries({ language: 'kuery', lists, exclude, }); - const expectedQuery = 'not b:(value-1 or value-2)'; + const expectedQuery = 'not b:("value-1" or "value-2")'; expect(query).toEqual(expectedQuery); }); test('it returns expected query when list includes single list item with operator of "excluded"', () => { - // Equal to query && !(!b) -> (query AND b) - // https://www.dcode.fr/boolean-expressions-calculator const lists: EntriesArray = [makeMatchAnyEntry({ field: 'b', operator: 'excluded' })]; const query = buildExceptionItemEntries({ language: 'kuery', lists, exclude, }); - const expectedQuery = 'b:(value-1 or value-2)'; + const expectedQuery = 'b:("value-1" or "value-2")'; expect(query).toEqual(expectedQuery); }); @@ -1088,7 +1074,7 @@ describe('build_exceptions_query', () => { lists, exclude, }); - const expectedQuery = 'b:("value" or "value-1") and parent:{ c:"valueC" }'; + const expectedQuery = 'b:("value-1" or "value-2") and parent:{ c:"valueC" }'; expect(query).toEqual(expectedQuery); }); @@ -1103,7 +1089,7 @@ describe('build_exceptions_query', () => { lists, exclude, }); - const expectedQuery = 'not b:("value" or "value-1") and not e:("valueE" or "value-4")'; + const expectedQuery = 'not b:("value-1" or "value-2") and not c:("value-1" or "value-2")'; expect(query).toEqual(expectedQuery); }); @@ -1111,11 +1097,10 @@ describe('build_exceptions_query', () => { }); describe('buildQueryExceptions', () => { - test('it returns original query if lists is empty array', () => { - const query = buildQueryExceptions({ query: 'host.name: *', language: 'kuery', lists: [] }); - const expectedQuery = 'host.name: *'; + test('it returns empty array if lists is empty array', () => { + const query = buildQueryExceptions({ language: 'kuery', lists: [] }); - expect(query).toEqual([{ query: expectedQuery, language: 'kuery' }]); + expect(query).toEqual([]); }); test('it returns expected query when lists exist and language is "kuery"', () => { @@ -1134,12 +1119,11 @@ describe('build_exceptions_query', () => { makeMatchAnyEntry({ field: 'e' }), ]; const query = buildQueryExceptions({ - query: 'a:*', language: 'kuery', lists: [payload, payload2], }); const expectedQuery = - 'a:* and some.parentField:{ nested.field:"some value" } and not some.not.nested.field:"some value" and not b:("value" or "value-1") and parent:{ c:"valueC" and d:"valueD" } and not e:("valueE" or "value-4")'; + 'some.parentField:{ nested.field:"some value" } and not some.not.nested.field:"some value" and not b:("value-1" or "value-2") and parent:{ c:"valueC" and d:"valueD" } and not e:("value-1" or "value-2")'; expect(query).toEqual([{ query: expectedQuery, language: 'kuery' }]); }); @@ -1160,12 +1144,11 @@ describe('build_exceptions_query', () => { makeMatchAnyEntry({ field: 'e' }), ]; const query = buildQueryExceptions({ - query: 'a:*', language: 'lucene', lists: [payload, payload2], }); const expectedQuery = - 'a:* AND some.parentField:{ nested.field:"some value" } AND NOT some.not.nested.field:"some value" AND NOT b:("value" OR "value-1") AND parent:{ c:"valueC" AND d:"valueD" } AND NOT e:("valueE" OR "value-4")'; + 'some.parentField:{ nested.field:"some value" } AND NOT some.not.nested.field:"some value" AND NOT b:("value-1" OR "value-2") AND parent:{ c:"valueC" AND d:"valueD" } AND NOT e:("value-1" OR "value-2")'; expect(query).toEqual([{ query: expectedQuery, language: 'lucene' }]); }); @@ -1175,21 +1158,17 @@ describe('build_exceptions_query', () => { exclude = false; }); - test('it returns original query if lists is empty array', () => { + test('it returns empty array if lists is empty array', () => { const query = buildQueryExceptions({ - query: 'host.name: *', language: 'kuery', lists: [], exclude, }); - const expectedQuery = 'host.name: *'; - expect(query).toEqual([{ query: expectedQuery, language: 'kuery' }]); + expect(query).toEqual([]); }); test('it returns expected query when lists exist and language is "kuery"', () => { - // Equal to query && !((b || !c || d) && e) -> ((query AND NOT b AND c AND NOT d) OR (query AND NOT e) - // https://www.dcode.fr/boolean-expressions-calculator const payload = getExceptionListItemSchemaMock(); const payload2 = getExceptionListItemSchemaMock(); payload2.entries = [ @@ -1205,20 +1184,17 @@ describe('build_exceptions_query', () => { makeMatchAnyEntry({ field: 'e' }), ]; const query = buildQueryExceptions({ - query: 'a:*', language: 'kuery', lists: [payload, payload2], exclude, }); const expectedQuery = - '(a:* and some.parentField:{ nested.field:some value } and some.not.nested.field:some value) or (a:* and b:(value-1 or value-2) and parent:{ c:valueC and d:valueD } and e:(value-1 or value-2))'; + 'some.parentField:{ nested.field:"some value" } and some.not.nested.field:"some value" and b:("value-1" or "value-2") and parent:{ c:"valueC" and d:"valueD" } and e:("value-1" or "value-2")'; expect(query).toEqual([{ query: expectedQuery, language: 'kuery' }]); }); test('it returns expected query when lists exist and language is "lucene"', () => { - // Equal to query && !((b || !c || d) && e) -> ((query AND NOT b AND c AND NOT d) OR (query AND NOT e) - // https://www.dcode.fr/boolean-expressions-calculator const payload = getExceptionListItemSchemaMock(); const payload2 = getExceptionListItemSchemaMock(); payload2.entries = [ @@ -1234,13 +1210,12 @@ describe('build_exceptions_query', () => { makeMatchAnyEntry({ field: 'e' }), ]; const query = buildQueryExceptions({ - query: 'a:*', language: 'lucene', lists: [payload, payload2], exclude, }); const expectedQuery = - '(a:* AND some.parentField:{ nested.field:some value } AND some.not.nested.field:some value) OR (a:* AND b:(value-1 OR value-2) AND parent:{ c:valueC AND d:valueD } AND e:(value-1 OR value-2))'; + 'some.parentField:{ nested.field:"some value" } AND some.not.nested.field:"some value" AND b:("value-1" OR "value-2") AND parent:{ c:"valueC" AND d:"valueD" } AND e:("value-1" OR "value-2")'; expect(query).toEqual([{ query: expectedQuery, language: 'lucene' }]); }); diff --git a/x-pack/plugins/security_solution/common/detection_engine/build_exceptions_query.ts b/x-pack/plugins/security_solution/common/detection_engine/build_exceptions_query.ts index 1095dfabfc7a2..763ddc5781231 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/build_exceptions_query.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/build_exceptions_query.ts @@ -19,7 +19,7 @@ import { ExceptionListItemSchema, CreateExceptionListItemSchema, } from '../shared_imports'; -import { Language, Query } from './schemas/common/schemas'; +import { Language } from './schemas/common/schemas'; type Operators = 'and' | 'or' | 'not'; type LuceneOperators = 'AND' | 'OR' | 'NOT'; diff --git a/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.test.ts b/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.test.ts index c19ef45605f83..b75bcc14392de 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.test.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.test.ts @@ -362,59 +362,42 @@ describe('get_filter', () => { expect(esQuery).toEqual({ bool: { filter: [ + { bool: { minimum_should_match: 1, should: [{ match: { 'host.name': 'linux' } }] } }, { bool: { filter: [ { - bool: { - minimum_should_match: 1, - should: [ - { - match: { - 'host.name': 'linux', - }, + nested: { + path: 'some.parentField', + query: { + bool: { + minimum_should_match: 1, + should: [ + { + match_phrase: { + 'some.parentField.nested.field': 'some value', + }, + }, + ], }, - ], + }, + score_mode: 'none', }, }, { bool: { - filter: [ - { - nested: { - path: 'some.parentField', - query: { - bool: { - minimum_should_match: 1, - should: [ - { - match: { - 'some.parentField.nested.field': 'some value', - }, - }, - ], - }, - }, - score_mode: 'none', - }, - }, - { - bool: { - must_not: { - bool: { - minimum_should_match: 1, - should: [ - { - match: { - 'some.not.nested.field': 'some value', - }, - }, - ], + must_not: { + bool: { + minimum_should_match: 1, + should: [ + { + match_phrase: { + 'some.not.nested.field': 'some value', }, }, - }, + ], }, - ], + }, }, }, ], @@ -469,52 +452,35 @@ describe('get_filter', () => { expect(esQuery).toEqual({ bool: { filter: [ + { bool: { minimum_should_match: 1, should: [{ match: { 'host.name': 'linux' } }] } }, { bool: { filter: [ { - bool: { - minimum_should_match: 1, - should: [ - { - match: { - 'host.name': 'linux', - }, + nested: { + path: 'some.parentField', + query: { + bool: { + minimum_should_match: 1, + should: [ + { + match_phrase: { + 'some.parentField.nested.field': 'some value', + }, + }, + ], }, - ], + }, + score_mode: 'none', }, }, { bool: { - filter: [ - { - nested: { - path: 'some.parentField', - query: { - bool: { - minimum_should_match: 1, - should: [ - { - match: { - 'some.parentField.nested.field': 'some value', - }, - }, - ], - }, - }, - score_mode: 'none', - }, - }, + minimum_should_match: 1, + should: [ { - bool: { - minimum_should_match: 1, - should: [ - { - match: { - 'some.not.nested.field': 'some value', - }, - }, - ], + match_phrase: { + 'some.not.nested.field': 'some value', }, }, ], From 6f941a20dd27cf7c43f8cb4115cf590f1dc660fd Mon Sep 17 00:00:00 2001 From: Yara Tercero Date: Tue, 14 Jul 2020 08:46:21 -0400 Subject: [PATCH 4/8] pin exceptions language to kuery --- .../common/detection_engine/get_query_filter.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.ts b/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.ts index 3e8a632901020..0fd54c6e3b26a 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.ts @@ -32,7 +32,7 @@ export const getQueryFilter = ( }; const initialQuery = [{ query, language }]; - const exceptions = buildQueryExceptions({ language, lists, exclude: excludeExceptions }); + const exceptions = buildQueryExceptions({ language: 'kuery', lists, exclude: excludeExceptions }); const queries: DataQuery[] = [...initialQuery, ...exceptions]; const config = { From 89b26ab1728e60aa5f9858329b1b2d44dad0a96a Mon Sep 17 00:00:00 2001 From: Yara Tercero Date: Tue, 14 Jul 2020 21:59:25 -0400 Subject: [PATCH 5/8] re-re-revert? - went back to older commit that accomadated signal closure better, unit tests need updating --- .../build_exceptions_query.test.ts | 54 ++++----- .../build_exceptions_query.ts | 89 +++++++-------- .../detection_engine/get_query_filter.ts | 7 ++ .../common/detection_engine/utils.test.ts | 105 ++++++++++++++++++ .../common/detection_engine/utils.ts | 17 +++ .../detection_engine/signals/utils.test.ts | 48 -------- .../lib/detection_engine/signals/utils.ts | 8 +- 7 files changed, 197 insertions(+), 131 deletions(-) create mode 100644 x-pack/plugins/security_solution/common/detection_engine/utils.test.ts create mode 100644 x-pack/plugins/security_solution/common/detection_engine/utils.ts diff --git a/x-pack/plugins/security_solution/common/detection_engine/build_exceptions_query.test.ts b/x-pack/plugins/security_solution/common/detection_engine/build_exceptions_query.test.ts index c728c2ad83312..e5388fcb44234 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/build_exceptions_query.test.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/build_exceptions_query.test.ts @@ -1103,6 +1103,12 @@ describe('build_exceptions_query', () => { expect(query).toEqual([]); }); + test('it returns empty array if lists is undefined', () => { + const query = buildQueryExceptions({ language: 'kuery', lists: undefined }); + + expect(query).toEqual([]); + }); + test('it returns expected query when lists exist and language is "kuery"', () => { const payload = getExceptionListItemSchemaMock(); const payload2 = getExceptionListItemSchemaMock(); @@ -1112,43 +1118,33 @@ describe('build_exceptions_query', () => { field: 'parent', type: 'nested', entries: [ - makeMatchEntry({ field: 'c', operator: 'excluded', value: 'valueC' }), - makeMatchEntry({ field: 'd', operator: 'excluded', value: 'valueD' }), + makeMatchEntry({ field: 'c', operator: 'included', value: 'valueC' }), + makeMatchEntry({ field: 'd', operator: 'included', value: 'valueD' }), ], }, - makeMatchAnyEntry({ field: 'e' }), + makeMatchAnyEntry({ field: 'e', operator: 'excluded' }), ]; const query = buildQueryExceptions({ language: 'kuery', lists: [payload, payload2], }); const expectedQuery = - 'some.parentField:{ nested.field:"some value" } and not some.not.nested.field:"some value" and not b:("value-1" or "value-2") and parent:{ c:"valueC" and d:"valueD" } and not e:("value-1" or "value-2")'; + 'not ((some.parentField:{ nested.field:"some value" } and some.not.nested.field:"some value") or (b:("value-1" or "value-2") and parent:{ c:"valueC" and d:"valueD" } and not e:("value-1" or "value-2")))'; expect(query).toEqual([{ query: expectedQuery, language: 'kuery' }]); }); test('it returns expected query when lists exist and language is "lucene"', () => { const payload = getExceptionListItemSchemaMock(); + payload.entries = [makeMatchAnyEntry({ field: 'a' }), makeMatchAnyEntry({ field: 'b' })]; const payload2 = getExceptionListItemSchemaMock(); - payload2.entries = [ - makeMatchAnyEntry({ field: 'b' }), - { - field: 'parent', - type: 'nested', - entries: [ - makeMatchEntry({ field: 'c', operator: 'excluded', value: 'valueC' }), - makeMatchEntry({ field: 'd', operator: 'excluded', value: 'valueD' }), - ], - }, - makeMatchAnyEntry({ field: 'e' }), - ]; + payload2.entries = [makeMatchAnyEntry({ field: 'c' }), makeMatchAnyEntry({ field: 'd' })]; const query = buildQueryExceptions({ language: 'lucene', lists: [payload, payload2], }); const expectedQuery = - 'some.parentField:{ nested.field:"some value" } AND NOT some.not.nested.field:"some value" AND NOT b:("value-1" OR "value-2") AND parent:{ c:"valueC" AND d:"valueD" } AND NOT e:("value-1" OR "value-2")'; + 'NOT ((a:("value-1" OR "value-2") AND b:("value-1" OR "value-2")) OR (c:("value-1" OR "value-2") AND d:("value-1" OR "value-2")))'; expect(query).toEqual([{ query: expectedQuery, language: 'lucene' }]); }); @@ -1168,6 +1164,12 @@ describe('build_exceptions_query', () => { expect(query).toEqual([]); }); + test('it returns empty array if lists is undefined', () => { + const query = buildQueryExceptions({ language: 'kuery', lists: undefined, exclude }); + + expect(query).toEqual([]); + }); + test('it returns expected query when lists exist and language is "kuery"', () => { const payload = getExceptionListItemSchemaMock(); const payload2 = getExceptionListItemSchemaMock(); @@ -1189,33 +1191,23 @@ describe('build_exceptions_query', () => { exclude, }); const expectedQuery = - 'some.parentField:{ nested.field:"some value" } and some.not.nested.field:"some value" and b:("value-1" or "value-2") and parent:{ c:"valueC" and d:"valueD" } and e:("value-1" or "value-2")'; + '(some.parentField:{ nested.field:"some value" } and some.not.nested.field:"some value") or (b:("value-1" or "value-2") and parent:{ c:"valueC" and d:"valueD" } and e:("value-1" or "value-2"))'; expect(query).toEqual([{ query: expectedQuery, language: 'kuery' }]); }); test('it returns expected query when lists exist and language is "lucene"', () => { const payload = getExceptionListItemSchemaMock(); + payload.entries = [makeMatchAnyEntry({ field: 'a' }), makeMatchAnyEntry({ field: 'b' })]; const payload2 = getExceptionListItemSchemaMock(); - payload2.entries = [ - makeMatchAnyEntry({ field: 'b' }), - { - field: 'parent', - type: 'nested', - entries: [ - makeMatchEntry({ field: 'c', operator: 'excluded', value: 'valueC' }), - makeMatchEntry({ field: 'd', operator: 'excluded', value: 'valueD' }), - ], - }, - makeMatchAnyEntry({ field: 'e' }), - ]; + payload2.entries = [makeMatchAnyEntry({ field: 'c' }), makeMatchAnyEntry({ field: 'd' })]; const query = buildQueryExceptions({ language: 'lucene', lists: [payload, payload2], exclude, }); const expectedQuery = - 'some.parentField:{ nested.field:"some value" } AND some.not.nested.field:"some value" AND b:("value-1" OR "value-2") AND parent:{ c:"valueC" AND d:"valueD" } AND e:("value-1" OR "value-2")'; + '(a:("value-1" OR "value-2") AND b:("value-1" OR "value-2")) OR (c:("value-1" OR "value-2") AND d:("value-1" OR "value-2"))'; expect(query).toEqual([{ query: expectedQuery, language: 'lucene' }]); }); diff --git a/x-pack/plugins/security_solution/common/detection_engine/build_exceptions_query.ts b/x-pack/plugins/security_solution/common/detection_engine/build_exceptions_query.ts index 763ddc5781231..ce9be112ba95b 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/build_exceptions_query.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/build_exceptions_query.ts @@ -20,6 +20,7 @@ import { CreateExceptionListItemSchema, } from '../shared_imports'; import { Language } from './schemas/common/schemas'; +import { hasLargeValueList } from './utils'; type Operators = 'and' | 'or' | 'not'; type LuceneOperators = 'AND' | 'OR' | 'NOT'; @@ -46,18 +47,16 @@ export const getLanguageBooleanOperator = ({ export const operatorBuilder = ({ operator, language, - exclude, }: { operator: Operator; language: Language; - exclude: boolean; }): string => { const not = getLanguageBooleanOperator({ language, value: 'not', }); - if ((exclude && operator === 'included') || (!exclude && operator === 'excluded')) { + if (operator === 'excluded') { return `${not} `; } else { return ''; @@ -67,14 +66,12 @@ export const operatorBuilder = ({ export const buildExists = ({ item, language, - exclude, }: { item: EntryExists; language: Language; - exclude: boolean; }): string => { const { operator, field } = item; - const exceptionOperator = operatorBuilder({ operator, language, exclude }); + const exceptionOperator = operatorBuilder({ operator, language }); switch (language) { case 'kuery': @@ -89,14 +86,12 @@ export const buildExists = ({ export const buildMatch = ({ item, language, - exclude, }: { item: EntryMatch; language: Language; - exclude: boolean; }): string => { const { value, operator, field } = item; - const exceptionOperator = operatorBuilder({ operator, language, exclude }); + const exceptionOperator = operatorBuilder({ operator, language }); return `${exceptionOperator}${field}:"${value}"`; }; @@ -104,11 +99,9 @@ export const buildMatch = ({ export const buildMatchAny = ({ item, language, - exclude, }: { item: EntryMatchAny; language: Language; - exclude: boolean; }): string => { const { value, operator, field } = item; @@ -117,7 +110,7 @@ export const buildMatchAny = ({ return ''; default: const or = getLanguageBooleanOperator({ language, value: 'or' }); - const exceptionOperator = operatorBuilder({ operator, language, exclude }); + const exceptionOperator = operatorBuilder({ operator, language }); const matchAnyValues = value.map((v) => `"${v}"`); return `${exceptionOperator}${field}:(${matchAnyValues.join(` ${or} `)})`; @@ -141,18 +134,16 @@ export const buildNested = ({ export const evaluateValues = ({ item, language, - exclude, }: { item: Entry | EntryNested; language: Language; - exclude: boolean; }): string => { if (entriesExists.is(item)) { - return buildExists({ item, language, exclude }); + return buildExists({ item, language }); } else if (entriesMatch.is(item)) { - return buildMatch({ item, language, exclude }); + return buildMatch({ item, language }); } else if (entriesMatchAny.is(item)) { - return buildMatchAny({ item, language, exclude }); + return buildMatchAny({ item, language }); } else if (entriesNested.is(item)) { return buildNested({ item, language }); } else { @@ -163,40 +154,40 @@ export const evaluateValues = ({ export const formatQuery = ({ exceptions, language, + exclude, }: { exceptions: string[]; language: Language; + exclude: boolean; }): string => { - const and = getLanguageBooleanOperator({ language, value: 'and' }); - const formattedExceptions = exceptions.map((exception, index) => { + const or = getLanguageBooleanOperator({ language, value: 'or' }); + const not = getLanguageBooleanOperator({ language, value: 'not' }); + const formattedExceptionItems = exceptions.map((exceptionItem, index) => { if (index === 0) { - return exception; + return `(${exceptionItem})`; } - return `${and} ${exception}`; + return `${or} (${exceptionItem})`; }); - return formattedExceptions.join(' '); + const exceptionItemsQuery = formattedExceptionItems.join(' '); + return exclude ? `${not} (${exceptionItemsQuery})` : exceptionItemsQuery; }; export const buildExceptionItemEntries = ({ - lists, + entries, language, - exclude, }: { - lists: EntriesArray; + entries: EntriesArray; language: Language; - exclude: boolean; }): string => { const and = getLanguageBooleanOperator({ language, value: 'and' }); - const exceptionItem = lists - .filter(({ type }) => type !== 'list') - .reduce((accum, listItem) => { - const exceptionSegment = evaluateValues({ item: listItem, language, exclude }); - return [...accum, exceptionSegment]; - }, []); - - return exceptionItem.join(` ${and} `); + const exceptionItemEntries = entries.reduce((accum, listItem) => { + const exceptionSegment = evaluateValues({ item: listItem, language }); + return [...accum, exceptionSegment]; + }, []); + + return exceptionItemEntries.join(` ${and} `); }; export const buildQueryExceptions = ({ @@ -208,23 +199,29 @@ export const buildQueryExceptions = ({ lists: Array | undefined; exclude?: boolean; }): DataQuery[] => { - if (lists != null && lists.length > 0) { - const exceptions = lists.reduce((acc, exceptionItem) => { - return [ - ...acc, - ...(exceptionItem.entries !== undefined - ? [buildExceptionItemEntries({ lists: exceptionItem.entries, language, exclude })] - : []), - ]; - }, []); - const formattedQuery = formatQuery({ exceptions, language }); + if (lists == null || (lists != null && lists.length === 0)) { + return []; + } + + const exceptionItems = lists.reduce((acc, exceptionItem) => { + const { entries } = exceptionItem; + + if (entries != null && entries.length > 0 && !hasLargeValueList(entries)) { + return [...acc, buildExceptionItemEntries({ entries, language })]; + } else { + return acc; + } + }, []); + + if (exceptionItems.length === 0) { + return []; + } else { + const formattedQuery = formatQuery({ exceptions: exceptionItems, language, exclude }); return [ { query: formattedQuery, language, }, ]; - } else { - return []; } }; diff --git a/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.ts b/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.ts index 0fd54c6e3b26a..a41589b5d0231 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.ts @@ -32,6 +32,13 @@ export const getQueryFilter = ( }; const initialQuery = [{ query, language }]; + /* + * Pinning exceptions to 'kuery' because lucene + * does not support nested queries, while our exceptions + * UI does, since we can pass both lucene and kql into + * buildEsQuery, this allows us to offer nested queries + * regardless + */ const exceptions = buildQueryExceptions({ language: 'kuery', lists, exclude: excludeExceptions }); const queries: DataQuery[] = [...initialQuery, ...exceptions]; diff --git a/x-pack/plugins/security_solution/common/detection_engine/utils.test.ts b/x-pack/plugins/security_solution/common/detection_engine/utils.test.ts new file mode 100644 index 0000000000000..99680ffe41d44 --- /dev/null +++ b/x-pack/plugins/security_solution/common/detection_engine/utils.test.ts @@ -0,0 +1,105 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { hasLargeValueList, hasNestedEntry } from './utils'; +import { EntriesArray } from '../shared_imports'; + +describe('#hasLargeValueList', () => { + test('it returns false if empty array', () => { + const hasLists = hasLargeValueList([]); + + expect(hasLists).toBeFalsy(); + }); + + test('it returns true if item of type EntryList exists', () => { + const entries: EntriesArray = [ + { + field: 'actingProcess.file.signer', + type: 'list', + operator: 'included', + list: { id: 'some id', type: 'ip' }, + }, + { + field: 'file.signature.signer', + type: 'match', + operator: 'excluded', + value: 'Global Signer', + }, + ]; + const hasLists = hasLargeValueList(entries); + + expect(hasLists).toBeTruthy(); + }); + + test('it returns false if item of type EntryList does not exist', () => { + const entries: EntriesArray = [ + { + field: 'actingProcess.file.signer', + type: 'match', + operator: 'included', + value: 'Elastic, N.V.', + }, + { + field: 'file.signature.signer', + type: 'match', + operator: 'excluded', + value: 'Global Signer', + }, + ]; + const hasLists = hasLargeValueList(entries); + + expect(hasLists).toBeFalsy(); + }); +}); + +describe('#hasNestedEntry', () => { + test('it returns false if empty array', () => { + const hasLists = hasNestedEntry([]); + + expect(hasLists).toBeFalsy(); + }); + + test('it returns true if item of type EntryNested exists', () => { + const entries: EntriesArray = [ + { + field: 'actingProcess.file.signer', + type: 'nested', + entries: [ + { field: 'some field', type: 'match', operator: 'included', value: 'some value' }, + ], + }, + { + field: 'file.signature.signer', + type: 'match', + operator: 'excluded', + value: 'Global Signer', + }, + ]; + const hasLists = hasNestedEntry(entries); + + expect(hasLists).toBeTruthy(); + }); + + test('it returns false if item of type EntryNested does not exist', () => { + const entries: EntriesArray = [ + { + field: 'actingProcess.file.signer', + type: 'match', + operator: 'included', + value: 'Elastic, N.V.', + }, + { + field: 'file.signature.signer', + type: 'match', + operator: 'excluded', + value: 'Global Signer', + }, + ]; + const hasLists = hasNestedEntry(entries); + + expect(hasLists).toBeFalsy(); + }); +}); diff --git a/x-pack/plugins/security_solution/common/detection_engine/utils.ts b/x-pack/plugins/security_solution/common/detection_engine/utils.ts new file mode 100644 index 0000000000000..fa1812235f897 --- /dev/null +++ b/x-pack/plugins/security_solution/common/detection_engine/utils.ts @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { EntriesArray } from '../shared_imports'; + +export const hasLargeValueList = (entries: EntriesArray): boolean => { + const found = entries.filter(({ type }) => type === 'list'); + return found.length > 0; +}; + +export const hasNestedEntry = (entries: EntriesArray): boolean => { + const found = entries.filter(({ type }) => type === 'nested'); + return found.length > 0; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts index 0cc3ca092a4dc..779d54070fd21 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts @@ -23,7 +23,6 @@ import { getGapBetweenRuns, errorAggregator, getListsClient, - hasLargeValueList, getSignalTimeTuples, getExceptions, } from './utils'; @@ -584,53 +583,6 @@ describe('utils', () => { }); }); - describe('#hasLargeValueList', () => { - test('it returns false if empty array', () => { - const hasLists = hasLargeValueList([]); - - expect(hasLists).toBeFalsy(); - }); - - test('it returns true if item of type EntryList exists', () => { - const entries: EntriesArray = [ - { - field: 'actingProcess.file.signer', - type: 'list', - operator: 'included', - list: { id: 'some id', type: 'ip' }, - }, - { - field: 'file.signature.signer', - type: 'match', - operator: 'excluded', - value: 'Global Signer', - }, - ]; - const hasLists = hasLargeValueList(entries); - - expect(hasLists).toBeTruthy(); - }); - - test('it returns false if item of type EntryList does not exist', () => { - const entries: EntriesArray = [ - { - field: 'actingProcess.file.signer', - type: 'match', - operator: 'included', - value: 'Elastic, N.V.', - }, - { - field: 'file.signature.signer', - type: 'match', - operator: 'excluded', - value: 'Global Signer', - }, - ]; - const hasLists = hasLargeValueList(entries); - - expect(hasLists).toBeFalsy(); - }); - }); describe('getSignalTimeTuples', () => { test('should return a single tuple if no gap', () => { const someTuples = getSignalTimeTuples({ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts index 0016765b9dbe9..70433e2c8d989 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts @@ -10,10 +10,11 @@ import dateMath from '@elastic/datemath'; import { Logger, SavedObjectsClientContract } from '../../../../../../../src/core/server'; import { AlertServices, parseDuration } from '../../../../../alerts/server'; import { ExceptionListClient, ListClient, ListPluginSetup } from '../../../../../lists/server'; -import { EntriesArray, ExceptionListItemSchema } from '../../../../../lists/common/schemas'; +import { ExceptionListItemSchema } from '../../../../../lists/common/schemas'; import { ListArrayOrUndefined } from '../../../../common/detection_engine/schemas/types/lists'; import { BulkResponse, BulkResponseErrorAggregation } from './types'; import { BuildRuleMessage } from './rule_messages'; +import { hasLargeValueList } from '../../../../common/detection_engine/utils'; interface SortExceptionsReturn { exceptionsWithValueLists: ExceptionListItemSchema[]; @@ -53,11 +54,6 @@ export const getListsClient = async ({ return { listClient, exceptionsClient }; }; -export const hasLargeValueList = (entries: EntriesArray): boolean => { - const found = entries.filter(({ type }) => type === 'list'); - return found.length > 0; -}; - export const getExceptions = async ({ client, lists, From 6ae3e49b2d52946d49e4f949888f38929369d26c Mon Sep 17 00:00:00 2001 From: Yara Tercero Date: Tue, 14 Jul 2020 22:12:51 -0400 Subject: [PATCH 6/8] fix error on import --- .../lib/detection_engine/signals/filter_events_with_list.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filter_events_with_list.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filter_events_with_list.ts index 8aa3ca3393f49..ea52aecb379fa 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filter_events_with_list.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filter_events_with_list.ts @@ -14,7 +14,7 @@ import { EntryList, ExceptionListItemSchema, } from '../../../../../lists/common/schemas'; -import { hasLargeValueList } from './utils'; +import { hasLargeValueList } from '../../../../common/detection_engine/utils'; interface FilterEventsAgainstList { listClient: ListClient; From ca674afbc206485b2109a1cf9f936ef3a41bb67d Mon Sep 17 00:00:00 2001 From: Yara Tercero Date: Wed, 15 Jul 2020 00:21:47 -0400 Subject: [PATCH 7/8] updates tests --- .../scripts/lists/new/items/keyword_item.json | 2 +- .../build_exceptions_query.test.ts | 847 +++++------------- .../build_exceptions_query.ts | 4 + .../detection_engine/get_query_filter.test.ts | 42 +- .../detection_engine/signals/utils.test.ts | 1 - 5 files changed, 254 insertions(+), 642 deletions(-) diff --git a/x-pack/plugins/lists/server/scripts/lists/new/items/keyword_item.json b/x-pack/plugins/lists/server/scripts/lists/new/items/keyword_item.json index 96d925c157490..0848dc4c1bd94 100644 --- a/x-pack/plugins/lists/server/scripts/lists/new/items/keyword_item.json +++ b/x-pack/plugins/lists/server/scripts/lists/new/items/keyword_item.json @@ -1,4 +1,4 @@ { "list_id": "keyword_list", - "value": "kibana" + "value": "zeek" } diff --git a/x-pack/plugins/security_solution/common/detection_engine/build_exceptions_query.test.ts b/x-pack/plugins/security_solution/common/detection_engine/build_exceptions_query.test.ts index e5388fcb44234..caf2dfb761ed0 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/build_exceptions_query.test.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/build_exceptions_query.test.ts @@ -113,226 +113,97 @@ describe('build_exceptions_query', () => { }); describe('operatorBuilder', () => { - describe("when 'exclude' is true", () => { - describe('and langauge is kuery', () => { - test('it returns "not " when operator is "included"', () => { - const operator = operatorBuilder({ operator: 'included', language: 'kuery', exclude }); - expect(operator).toEqual('not '); - }); - test('it returns empty string when operator is "excluded"', () => { - const operator = operatorBuilder({ operator: 'excluded', language: 'kuery', exclude }); - expect(operator).toEqual(''); - }); + describe('and language is kuery', () => { + test('it returns empty string when operator is "included"', () => { + const operator = operatorBuilder({ operator: 'included', language: 'kuery' }); + expect(operator).toEqual(''); }); - - describe('and language is lucene', () => { - test('it returns "NOT " when operator is "included"', () => { - const operator = operatorBuilder({ operator: 'included', language: 'lucene', exclude }); - expect(operator).toEqual('NOT '); - }); - test('it returns empty string when operator is "excluded"', () => { - const operator = operatorBuilder({ operator: 'excluded', language: 'lucene', exclude }); - expect(operator).toEqual(''); - }); + test('it returns "not " when operator is "excluded"', () => { + const operator = operatorBuilder({ operator: 'excluded', language: 'kuery' }); + expect(operator).toEqual('not '); }); }); - describe("when 'exclude' is false", () => { - beforeEach(() => { - exclude = false; - }); - describe('and language is kuery', () => { - test('it returns empty string when operator is "included"', () => { - const operator = operatorBuilder({ operator: 'included', language: 'kuery', exclude }); - expect(operator).toEqual(''); - }); - test('it returns "not " when operator is "excluded"', () => { - const operator = operatorBuilder({ operator: 'excluded', language: 'kuery', exclude }); - expect(operator).toEqual('not '); - }); + describe('and language is lucene', () => { + test('it returns empty string when operator is "included"', () => { + const operator = operatorBuilder({ operator: 'included', language: 'lucene' }); + expect(operator).toEqual(''); }); - - describe('and language is lucene', () => { - test('it returns empty string when operator is "included"', () => { - const operator = operatorBuilder({ operator: 'included', language: 'lucene', exclude }); - expect(operator).toEqual(''); - }); - test('it returns "NOT " when operator is "excluded"', () => { - const operator = operatorBuilder({ operator: 'excluded', language: 'lucene', exclude }); - expect(operator).toEqual('NOT '); - }); + test('it returns "NOT " when operator is "excluded"', () => { + const operator = operatorBuilder({ operator: 'excluded', language: 'lucene' }); + expect(operator).toEqual('NOT '); }); }); }); describe('buildExists', () => { - describe("when 'exclude' is true", () => { - describe('kuery', () => { - test('it returns formatted wildcard string when operator is "excluded"', () => { - const query = buildExists({ - item: existsEntryWithExcluded, - language: 'kuery', - exclude, - }); - expect(query).toEqual('host.name:*'); - }); - test('it returns formatted wildcard string when operator is "included"', () => { - const query = buildExists({ - item: existsEntryWithIncluded, - language: 'kuery', - exclude, - }); - expect(query).toEqual('not host.name:*'); + describe('kuery', () => { + test('it returns formatted wildcard string when operator is "excluded"', () => { + const query = buildExists({ + item: existsEntryWithExcluded, + language: 'kuery', }); + expect(query).toEqual('not host.name:*'); }); - - describe('lucene', () => { - test('it returns formatted wildcard string when operator is "excluded"', () => { - const query = buildExists({ - item: existsEntryWithExcluded, - language: 'lucene', - exclude, - }); - expect(query).toEqual('_exists_host.name'); - }); - test('it returns formatted wildcard string when operator is "included"', () => { - const query = buildExists({ - item: existsEntryWithIncluded, - language: 'lucene', - exclude, - }); - expect(query).toEqual('NOT _exists_host.name'); + test('it returns formatted wildcard string when operator is "included"', () => { + const query = buildExists({ + item: existsEntryWithIncluded, + language: 'kuery', }); + expect(query).toEqual('host.name:*'); }); }); - describe("when 'exclude' is false", () => { - beforeEach(() => { - exclude = false; - }); - - describe('kuery', () => { - test('it returns formatted wildcard string when operator is "excluded"', () => { - const query = buildExists({ - item: existsEntryWithExcluded, - language: 'kuery', - exclude, - }); - expect(query).toEqual('not host.name:*'); - }); - test('it returns formatted wildcard string when operator is "included"', () => { - const query = buildExists({ - item: existsEntryWithIncluded, - language: 'kuery', - exclude, - }); - expect(query).toEqual('host.name:*'); + describe('lucene', () => { + test('it returns formatted wildcard string when operator is "excluded"', () => { + const query = buildExists({ + item: existsEntryWithExcluded, + language: 'lucene', }); + expect(query).toEqual('NOT _exists_host.name'); }); - - describe('lucene', () => { - test('it returns formatted wildcard string when operator is "excluded"', () => { - const query = buildExists({ - item: existsEntryWithExcluded, - language: 'lucene', - exclude, - }); - expect(query).toEqual('NOT _exists_host.name'); - }); - test('it returns formatted wildcard string when operator is "included"', () => { - const query = buildExists({ - item: existsEntryWithIncluded, - language: 'lucene', - exclude, - }); - expect(query).toEqual('_exists_host.name'); + test('it returns formatted wildcard string when operator is "included"', () => { + const query = buildExists({ + item: existsEntryWithIncluded, + language: 'lucene', }); + expect(query).toEqual('_exists_host.name'); }); }); }); describe('buildMatch', () => { - describe("when 'exclude' is true", () => { - describe('kuery', () => { - test('it returns formatted string when operator is "included"', () => { - const query = buildMatch({ - item: matchEntryWithIncluded, - language: 'kuery', - exclude, - }); - expect(query).toEqual('not host.name:"suricata"'); - }); - test('it returns formatted string when operator is "excluded"', () => { - const query = buildMatch({ - item: matchEntryWithExcluded, - language: 'kuery', - exclude, - }); - expect(query).toEqual('host.name:"suricata"'); + describe('kuery', () => { + test('it returns formatted string when operator is "included"', () => { + const query = buildMatch({ + item: matchEntryWithIncluded, + language: 'kuery', }); + expect(query).toEqual('host.name:"suricata"'); }); - - describe('lucene', () => { - test('it returns formatted string when operator is "included"', () => { - const query = buildMatch({ - item: matchEntryWithIncluded, - language: 'lucene', - exclude, - }); - expect(query).toEqual('NOT host.name:"suricata"'); - }); - test('it returns formatted string when operator is "excluded"', () => { - const query = buildMatch({ - item: matchEntryWithExcluded, - language: 'lucene', - exclude, - }); - expect(query).toEqual('host.name:"suricata"'); + test('it returns formatted string when operator is "excluded"', () => { + const query = buildMatch({ + item: matchEntryWithExcluded, + language: 'kuery', }); + expect(query).toEqual('not host.name:"suricata"'); }); }); - describe("when 'exclude' is false", () => { - beforeEach(() => { - exclude = false; - }); - - describe('kuery', () => { - test('it returns formatted string when operator is "included"', () => { - const query = buildMatch({ - item: matchEntryWithIncluded, - language: 'kuery', - exclude, - }); - expect(query).toEqual('host.name:"suricata"'); - }); - test('it returns formatted string when operator is "excluded"', () => { - const query = buildMatch({ - item: matchEntryWithExcluded, - language: 'kuery', - exclude, - }); - expect(query).toEqual('not host.name:"suricata"'); + describe('lucene', () => { + test('it returns formatted string when operator is "included"', () => { + const query = buildMatch({ + item: matchEntryWithIncluded, + language: 'lucene', }); + expect(query).toEqual('host.name:"suricata"'); }); - - describe('lucene', () => { - test('it returns formatted string when operator is "included"', () => { - const query = buildMatch({ - item: matchEntryWithIncluded, - language: 'lucene', - exclude, - }); - expect(query).toEqual('host.name:"suricata"'); - }); - test('it returns formatted string when operator is "excluded"', () => { - const query = buildMatch({ - item: matchEntryWithExcluded, - language: 'lucene', - exclude, - }); - expect(query).toEqual('NOT host.name:"suricata"'); + test('it returns formatted string when operator is "excluded"', () => { + const query = buildMatch({ + item: matchEntryWithExcluded, + language: 'lucene', }); + expect(query).toEqual('NOT host.name:"suricata"'); }); }); }); @@ -352,150 +223,79 @@ describe('build_exceptions_query', () => { operator: 'excluded', }); - describe("when 'exclude' is true", () => { - describe('kuery', () => { - test('it returns empty string if given an empty array for "values"', () => { - const exceptionSegment = buildMatchAny({ - item: entryWithIncludedAndNoValues, - language: 'kuery', - exclude, - }); - expect(exceptionSegment).toEqual(''); - }); - - test('it returns formatted string when "values" includes only one item', () => { - const exceptionSegment = buildMatchAny({ - item: entryWithIncludedAndOneValue, - language: 'kuery', - exclude, - }); - expect(exceptionSegment).toEqual('not host.name:("suricata")'); + describe('kuery', () => { + test('it returns empty string if given an empty array for "values"', () => { + const exceptionSegment = buildMatchAny({ + item: entryWithIncludedAndNoValues, + language: 'kuery', }); + expect(exceptionSegment).toEqual(''); + }); - test('it returns formatted string when operator is "included"', () => { - const exceptionSegment = buildMatchAny({ - item: matchAnyEntryWithIncludedAndTwoValues, - language: 'kuery', - exclude, - }); - expect(exceptionSegment).toEqual('not host.name:("suricata" or "auditd")'); + test('it returns formatted string when "values" includes only one item', () => { + const exceptionSegment = buildMatchAny({ + item: entryWithIncludedAndOneValue, + language: 'kuery', }); - test('it returns formatted string when operator is "excluded"', () => { - const exceptionSegment = buildMatchAny({ - item: entryWithExcludedAndTwoValues, - language: 'kuery', - exclude, - }); - expect(exceptionSegment).toEqual('host.name:("suricata" or "auditd")'); - }); + expect(exceptionSegment).toEqual('host.name:("suricata")'); }); - describe('lucene', () => { - test('it returns formatted string when operator is "included"', () => { - const exceptionSegment = buildMatchAny({ - item: matchAnyEntryWithIncludedAndTwoValues, - language: 'lucene', - exclude, - }); - expect(exceptionSegment).toEqual('NOT host.name:("suricata" OR "auditd")'); - }); - test('it returns formatted string when operator is "excluded"', () => { - const exceptionSegment = buildMatchAny({ - item: entryWithExcludedAndTwoValues, - language: 'lucene', - exclude, - }); - expect(exceptionSegment).toEqual('host.name:("suricata" OR "auditd")'); - }); - test('it returns formatted string when "values" includes only one item', () => { - const exceptionSegment = buildMatchAny({ - item: entryWithIncludedAndOneValue, - language: 'lucene', - exclude, - }); - expect(exceptionSegment).toEqual('NOT host.name:("suricata")'); + test('it returns formatted string when operator is "included"', () => { + const exceptionSegment = buildMatchAny({ + item: matchAnyEntryWithIncludedAndTwoValues, + language: 'kuery', }); - }); - }); - describe("when 'exclude' is false", () => { - beforeEach(() => { - exclude = false; + expect(exceptionSegment).toEqual('host.name:("suricata" or "auditd")'); }); - describe('kuery', () => { - test('it returns empty string if given an empty array for "values"', () => { - const exceptionSegment = buildMatchAny({ - item: entryWithIncludedAndNoValues, - language: 'kuery', - exclude, - }); - expect(exceptionSegment).toEqual(''); - }); - test('it returns formatted string when "values" includes only one item', () => { - const exceptionSegment = buildMatchAny({ - item: entryWithIncludedAndOneValue, - language: 'kuery', - exclude, - }); - expect(exceptionSegment).toEqual('host.name:("suricata")'); - }); - test('it returns formatted string when operator is "included"', () => { - const exceptionSegment = buildMatchAny({ - item: matchAnyEntryWithIncludedAndTwoValues, - language: 'kuery', - exclude, - }); - expect(exceptionSegment).toEqual('host.name:("suricata" or "auditd")'); + test('it returns formatted string when operator is "excluded"', () => { + const exceptionSegment = buildMatchAny({ + item: entryWithExcludedAndTwoValues, + language: 'kuery', }); - test('it returns formatted string when operator is "excluded"', () => { - const exceptionSegment = buildMatchAny({ - item: entryWithExcludedAndTwoValues, - language: 'kuery', - exclude, - }); - expect(exceptionSegment).toEqual('not host.name:("suricata" or "auditd")'); - }); + expect(exceptionSegment).toEqual('not host.name:("suricata" or "auditd")'); }); + }); - describe('lucene', () => { - test('it returns formatted string when operator is "included"', () => { - const exceptionSegment = buildMatchAny({ - item: matchAnyEntryWithIncludedAndTwoValues, - language: 'lucene', - exclude, - }); - expect(exceptionSegment).toEqual('host.name:("suricata" OR "auditd")'); + describe('lucene', () => { + test('it returns formatted string when operator is "included"', () => { + const exceptionSegment = buildMatchAny({ + item: matchAnyEntryWithIncludedAndTwoValues, + language: 'lucene', }); - test('it returns formatted string when operator is "excluded"', () => { - const exceptionSegment = buildMatchAny({ - item: entryWithExcludedAndTwoValues, - language: 'lucene', - exclude, - }); - expect(exceptionSegment).toEqual('NOT host.name:("suricata" OR "auditd")'); + + expect(exceptionSegment).toEqual('host.name:("suricata" OR "auditd")'); + }); + test('it returns formatted string when operator is "excluded"', () => { + const exceptionSegment = buildMatchAny({ + item: entryWithExcludedAndTwoValues, + language: 'lucene', }); - test('it returns formatted string when "values" includes only one item', () => { - const exceptionSegment = buildMatchAny({ - item: entryWithIncludedAndOneValue, - language: 'lucene', - exclude, - }); - expect(exceptionSegment).toEqual('host.name:("suricata")'); + + expect(exceptionSegment).toEqual('NOT host.name:("suricata" OR "auditd")'); + }); + test('it returns formatted string when "values" includes only one item', () => { + const exceptionSegment = buildMatchAny({ + item: entryWithIncludedAndOneValue, + language: 'lucene', }); + + expect(exceptionSegment).toEqual('host.name:("suricata")'); }); }); }); describe('buildNested', () => { + // NOTE: Only KQL supports nested describe('kuery', () => { test('it returns formatted query when one item in nested entry', () => { const item: EntryNested = { field: 'parent', type: 'nested', - entries: [makeMatchEntry({ field: 'nestedField', operator: 'excluded' })], + entries: [makeMatchEntry({ field: 'nestedField', operator: 'included' })], }; const result = buildNested({ item, language: 'kuery' }); @@ -507,8 +307,8 @@ describe('build_exceptions_query', () => { field: 'parent', type: 'nested', entries: [ - makeMatchEntry({ field: 'nestedField', operator: 'excluded' }), - makeMatchEntry({ field: 'nestedFieldB', operator: 'excluded', value: 'value-2' }), + makeMatchEntry({ field: 'nestedField', operator: 'included' }), + makeMatchEntry({ field: 'nestedFieldB', operator: 'included', value: 'value-2' }), ], }; const result = buildNested({ item, language: 'kuery' }); @@ -516,119 +316,49 @@ describe('build_exceptions_query', () => { expect(result).toEqual('parent:{ nestedField:"value-1" and nestedFieldB:"value-2" }'); }); }); - - // TODO: Does lucene support nested query syntax? - describe.skip('lucene', () => { - test('it returns formatted query when one item in nested entry', () => { - const item: EntryNested = { - field: 'parent', - type: 'nested', - entries: [makeMatchEntry({ field: 'nestedField', operator: 'excluded' })], - }; - const result = buildNested({ item, language: 'lucene' }); - - expect(result).toEqual('parent:{ nestedField:"value-1" }'); - }); - - test('it returns formatted query when multiple items in nested entry', () => { - const item: EntryNested = { - field: 'parent', - type: 'nested', - entries: [ - makeMatchEntry({ field: 'nestedField', operator: 'excluded' }), - makeMatchEntry({ field: 'nestedFieldB', operator: 'excluded', value: 'value-2' }), - ], - }; - const result = buildNested({ item, language: 'lucene' }); - - expect(result).toEqual('parent:{ nestedField:"value-1" AND nestedFieldB:"value-2" }'); - }); - }); }); describe('evaluateValues', () => { - describe("when 'exclude' is true", () => { - describe('kuery', () => { - test('it returns formatted wildcard string when "type" is "exists"', () => { - const result = evaluateValues({ - item: existsEntryWithIncluded, - language: 'kuery', - exclude, - }); - expect(result).toEqual('not host.name:*'); - }); - - test('it returns formatted string when "type" is "match"', () => { - const result = evaluateValues({ - item: matchEntryWithIncluded, - language: 'kuery', - exclude, - }); - expect(result).toEqual('not host.name:"suricata"'); + describe('kuery', () => { + test('it returns formatted wildcard string when "type" is "exists"', () => { + const result = evaluateValues({ + item: existsEntryWithIncluded, + language: 'kuery', }); + expect(result).toEqual('host.name:*'); + }); - test('it returns formatted string when "type" is "match_any"', () => { - const result = evaluateValues({ - item: matchAnyEntryWithIncludedAndTwoValues, - language: 'kuery', - exclude, - }); - expect(result).toEqual('not host.name:("suricata" or "auditd")'); + test('it returns formatted string when "type" is "match"', () => { + const result = evaluateValues({ + item: matchEntryWithIncluded, + language: 'kuery', }); + expect(result).toEqual('host.name:"suricata"'); }); - describe('lucene', () => { - describe('kuery', () => { - test('it returns formatted wildcard string when "type" is "exists"', () => { - const result = evaluateValues({ - item: existsEntryWithIncluded, - language: 'lucene', - exclude, - }); - expect(result).toEqual('NOT _exists_host.name'); - }); - - test('it returns formatted string when "type" is "match"', () => { - const result = evaluateValues({ - item: matchEntryWithIncluded, - language: 'lucene', - exclude, - }); - expect(result).toEqual('NOT host.name:"suricata"'); - }); - - test('it returns formatted string when "type" is "match_any"', () => { - const result = evaluateValues({ - item: matchAnyEntryWithIncludedAndTwoValues, - language: 'lucene', - exclude, - }); - expect(result).toEqual('NOT host.name:("suricata" OR "auditd")'); - }); + test('it returns formatted string when "type" is "match_any"', () => { + const result = evaluateValues({ + item: matchAnyEntryWithIncludedAndTwoValues, + language: 'kuery', }); + expect(result).toEqual('host.name:("suricata" or "auditd")'); }); }); - describe("when 'exclude' is false", () => { - beforeEach(() => { - exclude = false; - }); - + describe('lucene', () => { describe('kuery', () => { test('it returns formatted wildcard string when "type" is "exists"', () => { const result = evaluateValues({ item: existsEntryWithIncluded, - language: 'kuery', - exclude, + language: 'lucene', }); - expect(result).toEqual('host.name:*'); + expect(result).toEqual('_exists_host.name'); }); test('it returns formatted string when "type" is "match"', () => { const result = evaluateValues({ item: matchEntryWithIncluded, - language: 'kuery', - exclude, + language: 'lucene', }); expect(result).toEqual('host.name:"suricata"'); }); @@ -636,76 +366,69 @@ describe('build_exceptions_query', () => { test('it returns formatted string when "type" is "match_any"', () => { const result = evaluateValues({ item: matchAnyEntryWithIncludedAndTwoValues, - language: 'kuery', - exclude, - }); - expect(result).toEqual('host.name:("suricata" or "auditd")'); - }); - }); - - describe('lucene', () => { - describe('kuery', () => { - test('it returns formatted wildcard string when "type" is "exists"', () => { - const result = evaluateValues({ - item: existsEntryWithIncluded, - language: 'lucene', - exclude, - }); - expect(result).toEqual('_exists_host.name'); - }); - - test('it returns formatted string when "type" is "match"', () => { - const result = evaluateValues({ - item: matchEntryWithIncluded, - language: 'lucene', - exclude, - }); - expect(result).toEqual('host.name:"suricata"'); - }); - - test('it returns formatted string when "type" is "match_any"', () => { - const result = evaluateValues({ - item: matchAnyEntryWithIncludedAndTwoValues, - language: 'lucene', - exclude, - }); - expect(result).toEqual('host.name:("suricata" OR "auditd")'); + language: 'lucene', }); + expect(result).toEqual('host.name:("suricata" OR "auditd")'); }); }); }); }); describe('formatQuery', () => { - describe('when query is empty string', () => { - test('it returns empty string if "exceptions" is empty array', () => { - const formattedQuery = formatQuery({ exceptions: [], language: 'kuery' }); - expect(formattedQuery).toEqual(''); + describe('exclude is true', () => { + describe('when query is empty string', () => { + test('it returns empty string if "exceptions" is empty array', () => { + const formattedQuery = formatQuery({ exceptions: [], language: 'kuery', exclude: true }); + expect(formattedQuery).toEqual(''); + }); + + test('it returns expected query string when single exception in array', () => { + const formattedQuery = formatQuery({ + exceptions: ['b:("value-1" or "value-2") and not c:*'], + language: 'kuery', + exclude: true, + }); + expect(formattedQuery).toEqual('not ((b:("value-1" or "value-2") and not c:*))'); + }); }); - test('it returns expected query string when single exception in array', () => { + test('it returns expected query string when multiple exceptions in array', () => { const formattedQuery = formatQuery({ - exceptions: ['b:("value-1" or "value-2") and not c:*'], + exceptions: ['b:("value-1" or "value-2") and not c:*', 'not d:*'], language: 'kuery', + exclude: true, }); - expect(formattedQuery).toEqual('b:("value-1" or "value-2") and not c:*'); + expect(formattedQuery).toEqual( + 'not ((b:("value-1" or "value-2") and not c:*) or (not d:*))' + ); }); }); - test('it returns expected query string when single exception in array', () => { - const formattedQuery = formatQuery({ - exceptions: ['b:("value-1" or "value-2") and not c:*'], - language: 'kuery', + describe('exclude is false', () => { + describe('when query is empty string', () => { + test('it returns empty string if "exceptions" is empty array', () => { + const formattedQuery = formatQuery({ exceptions: [], language: 'kuery', exclude: false }); + expect(formattedQuery).toEqual(''); + }); + + test('it returns expected query string when single exception in array', () => { + const formattedQuery = formatQuery({ + exceptions: ['b:("value-1" or "value-2") and not c:*'], + language: 'kuery', + exclude: false, + }); + expect(formattedQuery).toEqual('(b:("value-1" or "value-2") and not c:*)'); + }); }); - expect(formattedQuery).toEqual('b:("value-1" or "value-2") and not c:*'); - }); - test('it returns expected query string when multiple exceptions in array', () => { - const formattedQuery = formatQuery({ - exceptions: ['b:("value-1" or "value-2") and not c:*', 'not d:*'], - language: 'kuery', + test('it returns expected query string when multiple exceptions in array', () => { + const formattedQuery = formatQuery({ + exceptions: ['b:("value-1" or "value-2") and not c:*', 'not d:*'], + language: 'kuery', + exclude: false, + }); + expect(formattedQuery).toEqual('(b:("value-1" or "value-2") and not c:*) or (not d:*)'); }); - expect(formattedQuery).toEqual('b:("value-1" or "value-2") and not c:* and not d:*'); }); }); @@ -713,73 +436,69 @@ describe('build_exceptions_query', () => { test('it returns empty string if empty lists array passed in', () => { const query = buildExceptionItemEntries({ language: 'kuery', - lists: [], - exclude, + entries: [], }); expect(query).toEqual(''); }); - test('it returns expected query when more than one item in list', () => { + test('it returns expected query when more than one item in exception item', () => { const payload: EntriesArray = [ makeMatchAnyEntry({ field: 'b' }), makeMatchEntry({ field: 'c', operator: 'excluded', value: 'value-3' }), ]; const query = buildExceptionItemEntries({ language: 'kuery', - lists: payload, - exclude, + entries: payload, }); - const expectedQuery = 'not b:("value-1" or "value-2") and c:"value-3"'; + const expectedQuery = 'b:("value-1" or "value-2") and not c:"value-3"'; expect(query).toEqual(expectedQuery); }); - test('it returns expected query when list item includes nested value', () => { - const lists: EntriesArray = [ + test('it returns expected query when exception item includes nested value', () => { + const entries: EntriesArray = [ makeMatchAnyEntry({ field: 'b' }), { field: 'parent', type: 'nested', entries: [ - makeMatchEntry({ field: 'nestedField', operator: 'excluded', value: 'value-3' }), + makeMatchEntry({ field: 'nestedField', operator: 'included', value: 'value-3' }), ], }, ]; const query = buildExceptionItemEntries({ language: 'kuery', - lists, - exclude, + entries, }); - const expectedQuery = 'not b:("value-1" or "value-2") and parent:{ nestedField:"value-3" }'; + const expectedQuery = 'b:("value-1" or "value-2") and parent:{ nestedField:"value-3" }'; expect(query).toEqual(expectedQuery); }); - test('it returns expected query when list includes multiple items and nested "and" values', () => { - const lists: EntriesArray = [ + test('it returns expected query when exception item includes multiple items and nested "and" values', () => { + const entries: EntriesArray = [ makeMatchAnyEntry({ field: 'b' }), { field: 'parent', type: 'nested', entries: [ - makeMatchEntry({ field: 'nestedField', operator: 'excluded', value: 'value-3' }), + makeMatchEntry({ field: 'nestedField', operator: 'included', value: 'value-3' }), ], }, makeExistsEntry({ field: 'd' }), ]; const query = buildExceptionItemEntries({ language: 'kuery', - lists, - exclude, + entries, }); const expectedQuery = - 'not b:("value-1" or "value-2") and parent:{ nestedField:"value-3" } and not d:*'; + 'b:("value-1" or "value-2") and parent:{ nestedField:"value-3" } and d:*'; expect(query).toEqual(expectedQuery); }); test('it returns expected query when language is "lucene"', () => { - const lists: EntriesArray = [ + const entries: EntriesArray = [ makeMatchAnyEntry({ field: 'b' }), { field: 'parent', @@ -792,156 +511,56 @@ describe('build_exceptions_query', () => { ]; const query = buildExceptionItemEntries({ language: 'lucene', - lists, - exclude, + entries, }); const expectedQuery = - 'NOT b:("value-1" OR "value-2") AND parent:{ nestedField:"value-3" } AND _exists_e'; + 'b:("value-1" OR "value-2") AND parent:{ nestedField:"value-3" } AND NOT _exists_e'; expect(query).toEqual(expectedQuery); }); - describe('when "exclude" is false', () => { - beforeEach(() => { - exclude = false; - }); - - test('it returns empty string if empty lists array passed in', () => { - const query = buildExceptionItemEntries({ - language: 'kuery', - lists: [], - exclude, - }); - - expect(query).toEqual(''); - }); - - test('it returns expected query when more than one item in list', () => { - const payload: EntriesArray = [ - makeMatchAnyEntry({ field: 'b' }), - makeMatchEntry({ field: 'c', operator: 'excluded', value: 'value-3' }), - ]; - const query = buildExceptionItemEntries({ - language: 'kuery', - lists: payload, - exclude, - }); - const expectedQuery = 'b:("value-1" or "value-2") and not c:"value-3"'; - - expect(query).toEqual(expectedQuery); - }); - - test('it returns expected query when list item includes nested value', () => { - const lists: EntriesArray = [ - makeMatchAnyEntry({ field: 'b' }), - { - field: 'parent', - type: 'nested', - entries: [ - makeMatchEntry({ field: 'nestedField', operator: 'excluded', value: 'value-3' }), - ], - }, - ]; - const query = buildExceptionItemEntries({ - language: 'kuery', - lists, - exclude, - }); - const expectedQuery = 'b:("value-1" or "value-2") and parent:{ nestedField:"value-3" }'; - - expect(query).toEqual(expectedQuery); - }); - - test('it returns expected query when list includes multiple items and nested "and" values', () => { - const lists: EntriesArray = [ - makeMatchAnyEntry({ field: 'b' }), - { - field: 'parent', - type: 'nested', - entries: [ - makeMatchEntry({ field: 'nestedField', operator: 'excluded', value: 'value-3' }), - ], - }, - makeExistsEntry({ field: 'd' }), - ]; - const query = buildExceptionItemEntries({ - language: 'kuery', - lists, - exclude, - }); - const expectedQuery = - 'b:("value-1" or "value-2") and parent:{ nestedField:"value-3" } and d:*'; - expect(query).toEqual(expectedQuery); - }); - - test('it returns expected query when language is "lucene"', () => { - const lists: EntriesArray = [ - makeMatchAnyEntry({ field: 'b' }), - { - field: 'parent', - type: 'nested', - entries: [ - makeMatchEntry({ field: 'nestedField', operator: 'excluded', value: 'value-3' }), - ], - }, - makeExistsEntry({ field: 'e', operator: 'excluded' }), - ]; - const query = buildExceptionItemEntries({ - language: 'lucene', - lists, - exclude, - }); - const expectedQuery = - 'b:("value-1" OR "value-2") AND parent:{ nestedField:"value-3" } AND NOT _exists_e'; - expect(query).toEqual(expectedQuery); - }); - }); - describe('exists', () => { test('it returns expected query when list includes single list item with operator of "included"', () => { - const lists: EntriesArray = [makeExistsEntry({ field: 'b' })]; + const entries: EntriesArray = [makeExistsEntry({ field: 'b' })]; const query = buildExceptionItemEntries({ language: 'kuery', - lists, - exclude, + entries, }); - const expectedQuery = 'not b:*'; + const expectedQuery = 'b:*'; expect(query).toEqual(expectedQuery); }); test('it returns expected query when list includes single list item with operator of "excluded"', () => { - const lists: EntriesArray = [makeExistsEntry({ field: 'b', operator: 'excluded' })]; + const entries: EntriesArray = [makeExistsEntry({ field: 'b', operator: 'excluded' })]; const query = buildExceptionItemEntries({ language: 'kuery', - lists, - exclude, + entries, }); - const expectedQuery = 'b:*'; + const expectedQuery = 'not b:*'; expect(query).toEqual(expectedQuery); }); - test('it returns expected query when list includes list item with "and" values', () => { - const lists: EntriesArray = [ + test('it returns expected query when exception item includes entry item with "and" values', () => { + const entries: EntriesArray = [ makeExistsEntry({ field: 'b', operator: 'excluded' }), { field: 'parent', type: 'nested', - entries: [makeMatchEntry({ field: 'c', operator: 'excluded', value: 'value-1' })], + entries: [makeMatchEntry({ field: 'c', operator: 'included', value: 'value-1' })], }, ]; const query = buildExceptionItemEntries({ language: 'kuery', - lists, - exclude, + entries, }); - const expectedQuery = 'b:* and parent:{ c:"value-1" }'; + const expectedQuery = 'not b:* and parent:{ c:"value-1" }'; expect(query).toEqual(expectedQuery); }); test('it returns expected query when list includes multiple items', () => { - const lists: EntriesArray = [ + const entries: EntriesArray = [ makeExistsEntry({ field: 'b' }), { field: 'parent', @@ -955,10 +574,9 @@ describe('build_exceptions_query', () => { ]; const query = buildExceptionItemEntries({ language: 'kuery', - lists, - exclude, + entries, }); - const expectedQuery = 'not b:* and parent:{ c:"value-1" and d:"value-2" } and not e:*'; + const expectedQuery = 'b:* and parent:{ c:"value-1" and d:"value-2" } and e:*'; expect(query).toEqual(expectedQuery); }); @@ -966,52 +584,49 @@ describe('build_exceptions_query', () => { describe('match', () => { test('it returns expected query when list includes single list item with operator of "included"', () => { - const lists: EntriesArray = [makeMatchEntry({ field: 'b', value: 'value' })]; + const entries: EntriesArray = [makeMatchEntry({ field: 'b', value: 'value' })]; const query = buildExceptionItemEntries({ language: 'kuery', - lists, - exclude, + entries, }); - const expectedQuery = 'not b:"value"'; + const expectedQuery = 'b:"value"'; expect(query).toEqual(expectedQuery); }); test('it returns expected query when list includes single list item with operator of "excluded"', () => { - const lists: EntriesArray = [ + const entries: EntriesArray = [ makeMatchEntry({ field: 'b', operator: 'excluded', value: 'value' }), ]; const query = buildExceptionItemEntries({ language: 'kuery', - lists, - exclude, + entries, }); - const expectedQuery = 'b:"value"'; + const expectedQuery = 'not b:"value"'; expect(query).toEqual(expectedQuery); }); test('it returns expected query when list includes list item with "and" values', () => { - const lists: EntriesArray = [ + const entries: EntriesArray = [ makeMatchEntry({ field: 'b', operator: 'excluded', value: 'value' }), { field: 'parent', type: 'nested', - entries: [makeMatchEntry({ field: 'c', operator: 'excluded', value: 'valueC' })], + entries: [makeMatchEntry({ field: 'c', operator: 'included', value: 'valueC' })], }, ]; const query = buildExceptionItemEntries({ language: 'kuery', - lists, - exclude, + entries, }); - const expectedQuery = 'b:"value" and parent:{ c:"valueC" }'; + const expectedQuery = 'not b:"value" and parent:{ c:"valueC" }'; expect(query).toEqual(expectedQuery); }); test('it returns expected query when list includes multiple items', () => { - const lists: EntriesArray = [ + const entries: EntriesArray = [ makeMatchEntry({ field: 'b', value: 'value' }), { field: 'parent', @@ -1025,11 +640,9 @@ describe('build_exceptions_query', () => { ]; const query = buildExceptionItemEntries({ language: 'kuery', - lists, - exclude, + entries, }); - const expectedQuery = - 'not b:"value" and parent:{ c:"valueC" and d:"valueD" } and not e:"valueE"'; + const expectedQuery = 'b:"value" and parent:{ c:"valueC" and d:"valueD" } and e:"valueE"'; expect(query).toEqual(expectedQuery); }); @@ -1037,31 +650,29 @@ describe('build_exceptions_query', () => { describe('match_any', () => { test('it returns expected query when list includes single list item with operator of "included"', () => { - const lists: EntriesArray = [makeMatchAnyEntry({ field: 'b' })]; + const entries: EntriesArray = [makeMatchAnyEntry({ field: 'b' })]; const query = buildExceptionItemEntries({ language: 'kuery', - lists, - exclude, + entries, }); - const expectedQuery = 'not b:("value-1" or "value-2")'; + const expectedQuery = 'b:("value-1" or "value-2")'; expect(query).toEqual(expectedQuery); }); test('it returns expected query when list includes single list item with operator of "excluded"', () => { - const lists: EntriesArray = [makeMatchAnyEntry({ field: 'b', operator: 'excluded' })]; + const entries: EntriesArray = [makeMatchAnyEntry({ field: 'b', operator: 'excluded' })]; const query = buildExceptionItemEntries({ language: 'kuery', - lists, - exclude, + entries, }); - const expectedQuery = 'b:("value-1" or "value-2")'; + const expectedQuery = 'not b:("value-1" or "value-2")'; expect(query).toEqual(expectedQuery); }); test('it returns expected query when list includes list item with nested values', () => { - const lists: EntriesArray = [ + const entries: EntriesArray = [ makeMatchAnyEntry({ field: 'b', operator: 'excluded' }), { field: 'parent', @@ -1071,25 +682,23 @@ describe('build_exceptions_query', () => { ]; const query = buildExceptionItemEntries({ language: 'kuery', - lists, - exclude, + entries, }); - const expectedQuery = 'b:("value-1" or "value-2") and parent:{ c:"valueC" }'; + const expectedQuery = 'not b:("value-1" or "value-2") and parent:{ c:"valueC" }'; expect(query).toEqual(expectedQuery); }); test('it returns expected query when list includes multiple items', () => { - const lists: EntriesArray = [ + const entries: EntriesArray = [ makeMatchAnyEntry({ field: 'b' }), makeMatchAnyEntry({ field: 'c' }), ]; const query = buildExceptionItemEntries({ language: 'kuery', - lists, - exclude, + entries, }); - const expectedQuery = 'not b:("value-1" or "value-2") and not c:("value-1" or "value-2")'; + const expectedQuery = 'b:("value-1" or "value-2") and c:("value-1" or "value-2")'; expect(query).toEqual(expectedQuery); }); diff --git a/x-pack/plugins/security_solution/common/detection_engine/build_exceptions_query.ts b/x-pack/plugins/security_solution/common/detection_engine/build_exceptions_query.ts index ce9be112ba95b..fc4fbae02b8fb 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/build_exceptions_query.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/build_exceptions_query.ts @@ -160,6 +160,10 @@ export const formatQuery = ({ language: Language; exclude: boolean; }): string => { + if (exceptions == null || (exceptions != null && exceptions.length === 0)) { + return ''; + } + const or = getLanguageBooleanOperator({ language, value: 'or' }); const not = getLanguageBooleanOperator({ language, value: 'not' }); const formattedExceptionItems = exceptions.map((exceptionItem, index) => { diff --git a/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.test.ts b/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.test.ts index b75bcc14392de..a8eb4e7bbb15b 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.test.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.test.ts @@ -365,28 +365,28 @@ describe('get_filter', () => { { bool: { minimum_should_match: 1, should: [{ match: { 'host.name': 'linux' } }] } }, { bool: { - filter: [ - { - nested: { - path: 'some.parentField', - query: { - bool: { - minimum_should_match: 1, - should: [ - { - match_phrase: { - 'some.parentField.nested.field': 'some value', - }, + must_not: { + bool: { + filter: [ + { + nested: { + path: 'some.parentField', + query: { + bool: { + minimum_should_match: 1, + should: [ + { + match_phrase: { + 'some.parentField.nested.field': 'some value', + }, + }, + ], }, - ], + }, + score_mode: 'none', }, }, - score_mode: 'none', - }, - }, - { - bool: { - must_not: { + { bool: { minimum_should_match: 1, should: [ @@ -398,9 +398,9 @@ describe('get_filter', () => { ], }, }, - }, + ], }, - ], + }, }, }, ], diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts index 779d54070fd21..fa1635324e0f2 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts @@ -9,7 +9,6 @@ import sinon from 'sinon'; import { alertsMock, AlertServicesMock } from '../../../../../alerts/server/mocks'; import { listMock } from '../../../../../lists/server/mocks'; -import { EntriesArray } from '../../../../common/shared_imports'; import { buildRuleMessageFactory } from './rule_messages'; import { ExceptionListClient } from '../../../../../lists/server'; import { getListArrayMock } from '../../../../common/detection_engine/schemas/types/lists.mock'; From d4d8b6ac2a72d01bfeaf749f895caf1a7cc88351 Mon Sep 17 00:00:00 2001 From: Christos Nasikas Date: Wed, 15 Jul 2020 12:33:41 +0300 Subject: [PATCH 8/8] Fix test --- .../signals/get_filter.test.ts | 85 +++++++++---------- 1 file changed, 40 insertions(+), 45 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/get_filter.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/get_filter.test.ts index f34879781e0b0..a5740d7719f47 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/get_filter.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/get_filter.test.ts @@ -192,71 +192,66 @@ describe('get_filter', () => { index: ['auditbeat-*'], lists: [getExceptionListItemSchemaMock()], }); + expect(filter).toEqual({ bool: { + must: [], filter: [ { bool: { - filter: [ + should: [ { - bool: { - minimum_should_match: 1, - should: [ - { - match: { - 'host.name': 'siem', - }, - }, - ], + match: { + 'host.name': 'siem', }, }, - { - bool: { - filter: [ - { - nested: { - path: 'some.parentField', - query: { - bool: { - minimum_should_match: 1, - should: [ - { - match: { - 'some.parentField.nested.field': 'some value', - }, + ], + minimum_should_match: 1, + }, + }, + { + bool: { + must_not: { + bool: { + filter: [ + { + nested: { + path: 'some.parentField', + query: { + bool: { + should: [ + { + match_phrase: { + 'some.parentField.nested.field': 'some value', }, - ], - }, + }, + ], + minimum_should_match: 1, }, - score_mode: 'none', }, + score_mode: 'none', }, - { - bool: { - must_not: { - bool: { - minimum_should_match: 1, - should: [ - { - match: { - 'some.not.nested.field': 'some value', - }, - }, - ], + }, + { + bool: { + should: [ + { + match_phrase: { + 'some.not.nested.field': 'some value', }, }, - }, + ], + minimum_should_match: 1, }, - ], - }, + }, + ], }, - ], + }, }, }, ], - must: [], - must_not: [], should: [], + must_not: [], }, }); });