diff --git a/src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker b/src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker index c064298cf82a0..0473130332e94 100755 --- a/src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker +++ b/src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker @@ -387,6 +387,7 @@ kibana_vars=( xpack.security.sessionTimeout xpack.securitySolution.alertMergeStrategy xpack.securitySolution.alertResultListDefaultDateRange + xpack.securitySolution.alertIgnoreFields xpack.securitySolution.endpointResultListDefaultFirstPageIndex xpack.securitySolution.endpointResultListDefaultPageSize xpack.securitySolution.maxRuleImportExportSize diff --git a/x-pack/plugins/security_solution/server/config.test.ts b/x-pack/plugins/security_solution/server/config.test.ts new file mode 100644 index 0000000000000..67956acd6656f --- /dev/null +++ b/x-pack/plugins/security_solution/server/config.test.ts @@ -0,0 +1,80 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { configSchema } from './config'; + +describe('config', () => { + describe('alertIgnoreFields', () => { + test('should default to an empty array', () => { + expect(configSchema.validate({}).alertIgnoreFields).toEqual([]); + }); + + test('should accept an array of strings', () => { + expect( + configSchema.validate({ alertIgnoreFields: ['foo.bar', 'mars.bar'] }).alertIgnoreFields + ).toEqual(['foo.bar', 'mars.bar']); + }); + + test('should throw if a non string is being sent in', () => { + expect( + () => + configSchema.validate({ + alertIgnoreFields: 5, + }).alertIgnoreFields + ).toThrow('[alertIgnoreFields]: expected value of type [array] but got [number]'); + }); + + test('should throw if we send in an invalid regular expression as a string', () => { + expect( + () => + configSchema.validate({ + alertIgnoreFields: ['/(/'], + }).alertIgnoreFields + ).toThrow( + '[alertIgnoreFields]: "Invalid regular expression: /(/: Unterminated group" at array position 0' + ); + }); + + test('should throw with two errors if we send two invalid regular expressions', () => { + expect( + () => + configSchema.validate({ + alertIgnoreFields: ['/(/', '/(invalid/'], + }).alertIgnoreFields + ).toThrow( + '[alertIgnoreFields]: "Invalid regular expression: /(/: Unterminated group" at array position 0. "Invalid regular expression: /(invalid/: Unterminated group" at array position 1' + ); + }); + + test('should throw with two errors with a valid string mixed in if we send two invalid regular expressions', () => { + expect( + () => + configSchema.validate({ + alertIgnoreFields: ['/(/', 'valid.string', '/(invalid/'], + }).alertIgnoreFields + ).toThrow( + '[alertIgnoreFields]: "Invalid regular expression: /(/: Unterminated group" at array position 0. "Invalid regular expression: /(invalid/: Unterminated group" at array position 2' + ); + }); + + test('should accept a valid regular expression within the string', () => { + expect( + configSchema.validate({ + alertIgnoreFields: ['/(.*)/'], + }).alertIgnoreFields + ).toEqual(['/(.*)/']); + }); + + test('should accept two valid regular expressions', () => { + expect( + configSchema.validate({ + alertIgnoreFields: ['/(.*)/', '/(.valid*)/'], + }).alertIgnoreFields + ).toEqual(['/(.*)/', '/(.valid*)/']); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/config.ts b/x-pack/plugins/security_solution/server/config.ts index b8fdf188d533a..7c551a3906aa4 100644 --- a/x-pack/plugins/security_solution/server/config.ts +++ b/x-pack/plugins/security_solution/server/config.ts @@ -21,12 +21,61 @@ export const configSchema = schema.object({ maxRuleImportPayloadBytes: schema.number({ defaultValue: 10485760 }), maxTimelineImportExportSize: schema.number({ defaultValue: 10000 }), maxTimelineImportPayloadBytes: schema.number({ defaultValue: 10485760 }), + + /** + * This is used within the merge strategies: + * server/lib/detection_engine/signals/source_fields_merging + * + * For determining which strategy for merging "fields" and "_source" together to get + * runtime fields, constant keywords, etc... + * + * "missingFields" (default) This will only merge fields that are missing from the _source and exist in the fields. + * "noFields" This will turn off all merging of runtime fields, constant keywords from fields. + * "allFields" This will merge and overwrite anything found within "fields" into "_source" before indexing the data. + */ alertMergeStrategy: schema.oneOf( [schema.literal('allFields'), schema.literal('missingFields'), schema.literal('noFields')], { defaultValue: 'missingFields', } ), + + /** + * This is used within the merge strategies: + * server/lib/detection_engine/signals/source_fields_merging + * + * For determining if we need to ignore particular "fields" and not merge them with "_source" such as + * runtime fields, constant keywords, etc... + * + * This feature and functionality is mostly as "safety feature" meaning that we have had bugs in the past + * where something down the stack unexpectedly ends up in the fields API which causes documents to not + * be indexable. Rather than changing alertMergeStrategy to be "noFields", you can use this array to add + * any problematic values. + * + * You can use plain dotted notation strings such as "host.name" or a regular expression such as "/host\..+/" + */ + alertIgnoreFields: schema.arrayOf(schema.string(), { + defaultValue: [], + validate(ignoreFields) { + const errors = ignoreFields.flatMap((ignoreField, index) => { + if (ignoreField.startsWith('/') && ignoreField.endsWith('/')) { + try { + new RegExp(ignoreField.slice(1, -1)); + return []; + } catch (error) { + return [`"${error.message}" at array position ${index}`]; + } + } else { + return []; + } + }); + if (errors.length !== 0) { + return errors.join('. '); + } else { + return undefined; + } + }, + }), [SIGNALS_INDEX_KEY]: schema.string({ defaultValue: DEFAULT_SIGNALS_INDEX }), /** diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/index.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/index.ts index f61b0b2a6a2c6..8581687b532ab 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/index.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/index.ts @@ -31,6 +31,7 @@ export const createMockConfig = (): ConfigType => ({ packagerTaskInterval: '60s', validateArtifactDownloads: true, alertMergeStrategy: 'missingFields', + alertIgnoreFields: [], prebuiltRulesFromFileSystem: true, prebuiltRulesFromSavedObjects: false, }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts index 117dcdf0c18da..74d9735da36fc 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts @@ -41,7 +41,8 @@ describe('buildBulkBody', () => { const fakeSignalSourceHit: SignalHitOptionalTimestamp = buildBulkBody( ruleSO, doc, - 'missingFields' + 'missingFields', + [] ); // Timestamp will potentially always be different so remove it for the test delete fakeSignalSourceHit['@timestamp']; @@ -109,7 +110,8 @@ describe('buildBulkBody', () => { const fakeSignalSourceHit: SignalHitOptionalTimestamp = buildBulkBody( ruleSO, doc, - 'missingFields' + 'missingFields', + [] ); // Timestamp will potentially always be different so remove it for the test delete fakeSignalSourceHit['@timestamp']; @@ -191,7 +193,8 @@ describe('buildBulkBody', () => { const fakeSignalSourceHit: SignalHitOptionalTimestamp = buildBulkBody( ruleSO, doc, - 'missingFields' + 'missingFields', + [] ); // Timestamp will potentially always be different so remove it for the test delete fakeSignalSourceHit['@timestamp']; @@ -259,7 +262,8 @@ describe('buildBulkBody', () => { const fakeSignalSourceHit: SignalHitOptionalTimestamp = buildBulkBody( ruleSO, doc, - 'missingFields' + 'missingFields', + [] ); // Timestamp will potentially always be different so remove it for the test delete fakeSignalSourceHit['@timestamp']; @@ -324,7 +328,8 @@ describe('buildBulkBody', () => { const fakeSignalSourceHit: SignalHitOptionalTimestamp = buildBulkBody( ruleSO, doc, - 'missingFields' + 'missingFields', + [] ); // Timestamp will potentially always be different so remove it for the test delete fakeSignalSourceHit['@timestamp']; @@ -388,7 +393,8 @@ describe('buildBulkBody', () => { const { '@timestamp': timestamp, ...fakeSignalSourceHit } = buildBulkBody( ruleSO, doc, - 'missingFields' + 'missingFields', + [] ); const expected: Omit & { someKey: string } = { someKey: 'someValue', @@ -448,7 +454,8 @@ describe('buildBulkBody', () => { const { '@timestamp': timestamp, ...fakeSignalSourceHit } = buildBulkBody( ruleSO, doc, - 'missingFields' + 'missingFields', + [] ); const expected: Omit & { someKey: string } = { someKey: 'someValue', @@ -677,7 +684,8 @@ describe('buildSignalFromEvent', () => { ancestor, ruleSO, true, - 'missingFields' + 'missingFields', + [] ); // Timestamp will potentially always be different so remove it for the test diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts index 2e6f4b9303d89..fec9ba309b996 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts @@ -35,9 +35,10 @@ import type { ConfigType } from '../../../config'; export const buildBulkBody = ( ruleSO: SavedObject, doc: SignalSourceHit, - mergeStrategy: ConfigType['alertMergeStrategy'] + mergeStrategy: ConfigType['alertMergeStrategy'], + ignoreFields: ConfigType['alertIgnoreFields'] ): SignalHit => { - const mergedDoc = getMergeStrategy(mergeStrategy)({ doc }); + const mergedDoc = getMergeStrategy(mergeStrategy)({ doc, ignoreFields }); const rule = buildRuleWithOverrides(ruleSO, mergedDoc._source ?? {}); const signal: Signal = { ...buildSignal([mergedDoc], rule), @@ -68,11 +69,12 @@ export const buildSignalGroupFromSequence = ( sequence: EqlSequence, ruleSO: SavedObject, outputIndex: string, - mergeStrategy: ConfigType['alertMergeStrategy'] + mergeStrategy: ConfigType['alertMergeStrategy'], + ignoreFields: ConfigType['alertIgnoreFields'] ): WrappedSignalHit[] => { const wrappedBuildingBlocks = wrapBuildingBlocks( sequence.events.map((event) => { - const signal = buildSignalFromEvent(event, ruleSO, false, mergeStrategy); + const signal = buildSignalFromEvent(event, ruleSO, false, mergeStrategy, ignoreFields); signal.signal.rule.building_block_type = 'default'; return signal; }), @@ -134,9 +136,10 @@ export const buildSignalFromEvent = ( event: BaseSignalHit, ruleSO: SavedObject, applyOverrides: boolean, - mergeStrategy: ConfigType['alertMergeStrategy'] + mergeStrategy: ConfigType['alertMergeStrategy'], + ignoreFields: ConfigType['alertIgnoreFields'] ): SignalHit => { - const mergedEvent = getMergeStrategy(mergeStrategy)({ doc: event }); + const mergedEvent = getMergeStrategy(mergeStrategy)({ doc: event, ignoreFields }); const rule = applyOverrides ? buildRuleWithOverrides(ruleSO, mergedEvent._source ?? {}) : buildRuleWithoutOverrides(ruleSO); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts index 711db931e9072..74a8d85351068 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts @@ -71,6 +71,7 @@ describe('searchAfterAndBulkCreate', () => { ruleSO, signalsIndex: DEFAULT_SIGNALS_INDEX, mergeStrategy: 'missingFields', + ignoreFields: [], }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.test.ts index aec8b6c552b1d..8d21cf64d27ba 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.test.ts @@ -194,6 +194,7 @@ describe('signal_rule_alert_type', () => { ml: mlMock, lists: listMock.createSetup(), mergeStrategy: 'missingFields', + ignoreFields: [], }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts index 6eef97b05b697..b325dc7335fbd 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -77,6 +77,7 @@ export const signalRulesAlertType = ({ ml, lists, mergeStrategy, + ignoreFields, }: { logger: Logger; eventsTelemetry: TelemetryEventsSender | undefined; @@ -84,6 +85,7 @@ export const signalRulesAlertType = ({ ml: SetupPlugins['ml']; lists: SetupPlugins['lists'] | undefined; mergeStrategy: ConfigType['alertMergeStrategy']; + ignoreFields: ConfigType['alertIgnoreFields']; }): SignalRuleAlertTypeDefinition => { return { id: SIGNALS_ID, @@ -237,12 +239,14 @@ export const signalRulesAlertType = ({ ruleSO: savedObject, signalsIndex: params.outputIndex, mergeStrategy, + ignoreFields, }); const wrapSequences = wrapSequencesFactory({ ruleSO: savedObject, signalsIndex: params.outputIndex, mergeStrategy, + ignoreFields, }); if (isMlRule(type)) { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/strategies/merge_all_fields_with_source.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/strategies/merge_all_fields_with_source.test.ts index b900ea268fd6e..6af82d3a71028 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/strategies/merge_all_fields_with_source.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/strategies/merge_all_fields_with_source.test.ts @@ -44,7 +44,7 @@ describe('merge_all_fields_with_source', () => { test('when source is "undefined", merged doc is "undefined"', () => { const _source: SignalSourceHit['_source'] = {}; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -53,7 +53,7 @@ describe('merge_all_fields_with_source', () => { foo: [], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -62,7 +62,7 @@ describe('merge_all_fields_with_source', () => { foo: 'value', }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -71,7 +71,7 @@ describe('merge_all_fields_with_source', () => { foo: ['value'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -80,7 +80,7 @@ describe('merge_all_fields_with_source', () => { foo: ['value_1', 'value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -89,7 +89,7 @@ describe('merge_all_fields_with_source', () => { foo: { bar: 'some value' }, }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -98,7 +98,7 @@ describe('merge_all_fields_with_source', () => { foo: [{ bar: 'some value' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -107,7 +107,7 @@ describe('merge_all_fields_with_source', () => { foo: [{ bar: 'some value' }, { foo: 'some other value' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); }); @@ -133,7 +133,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -142,7 +142,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': 'value', }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -151,7 +151,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['value'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -160,7 +160,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['value_1', 'value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -169,7 +169,7 @@ describe('merge_all_fields_with_source', () => { foo: { bar: 'some value' }, }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -178,7 +178,7 @@ describe('merge_all_fields_with_source', () => { foo: [{ bar: 'some value' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -187,7 +187,7 @@ describe('merge_all_fields_with_source', () => { foo: [{ bar: 'some value' }, { foo: 'some other value' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); }); @@ -217,7 +217,7 @@ describe('merge_all_fields_with_source', () => { foo: { bar: [] }, }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -226,7 +226,7 @@ describe('merge_all_fields_with_source', () => { foo: { bar: 'value' }, }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -235,7 +235,7 @@ describe('merge_all_fields_with_source', () => { foo: { bar: ['value'] }, }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -244,7 +244,7 @@ describe('merge_all_fields_with_source', () => { foo: { bar: ['value_1', 'value_2'] }, }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -253,7 +253,7 @@ describe('merge_all_fields_with_source', () => { foo: { bar: { mars: 'some value' } }, }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -262,7 +262,7 @@ describe('merge_all_fields_with_source', () => { foo: { bar: [{ mars: 'some value' }] }, }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -271,7 +271,7 @@ describe('merge_all_fields_with_source', () => { foo: { bar: [{ mars: 'some value' }, { mars: 'some other value' }] }, }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); }); @@ -299,7 +299,7 @@ describe('merge_all_fields_with_source', () => { 'bar.foo': [], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -308,7 +308,7 @@ describe('merge_all_fields_with_source', () => { 'bar.foo': 'value', }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -317,7 +317,7 @@ describe('merge_all_fields_with_source', () => { 'bar.foo': ['value'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -326,7 +326,7 @@ describe('merge_all_fields_with_source', () => { 'bar.foo': ['value_1', 'value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -335,7 +335,7 @@ describe('merge_all_fields_with_source', () => { foo: { bar: 'some value' }, }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -344,7 +344,7 @@ describe('merge_all_fields_with_source', () => { foo: [{ bar: 'some value' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -353,7 +353,7 @@ describe('merge_all_fields_with_source', () => { foo: [{ bar: 'some value' }, { foo: 'some other value' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); }); @@ -376,7 +376,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ foo: { bar: 'other_value_1', @@ -389,7 +389,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ foo: { bar: ['other_value_1', 'other_value_2'], @@ -402,7 +402,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ foo: { bar: { zed: 'other_value_1' } }, }); @@ -413,7 +413,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ foo: { bar: [{ zed: 'other_value_1' }, { zed: 'other_value_2' }] }, }); @@ -440,7 +440,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ foo: { bar: 'other_value_1', @@ -453,7 +453,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ foo: { bar: ['other_value_1', 'other_value_2'] }, }); @@ -464,7 +464,7 @@ describe('merge_all_fields_with_source', () => { foo: [{ bar: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ foo: { bar: 'other_value_1', @@ -477,7 +477,7 @@ describe('merge_all_fields_with_source', () => { foo: [{ bar: 'other_value_1' }, { bar: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ foo: [{ bar: 'other_value_1' }, { bar: 'other_value_2' }], }); @@ -503,7 +503,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ 'foo.bar': 'other_value_1', }); @@ -514,7 +514,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(fields); }); @@ -523,7 +523,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ 'foo.bar': { zed: 'other_value_1' }, }); @@ -534,7 +534,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(fields); }); }); @@ -560,7 +560,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ foo: { bar: ['other_value_1'] }, }); @@ -571,7 +571,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ foo: { bar: ['other_value_1', 'other_value_2'] }, }); @@ -582,7 +582,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ foo: { bar: [{ zed: 'other_value_1' }] }, }); @@ -593,7 +593,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ foo: { bar: [{ zed: 'other_value_1' }, { zed: 'other_value_2' }] }, }); @@ -619,7 +619,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(fields); }); @@ -628,7 +628,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(fields); }); @@ -637,7 +637,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(fields); }); @@ -646,7 +646,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(fields); }); }); @@ -670,7 +670,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ foo: { bar: ['other_value_1'] }, }); @@ -681,7 +681,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ foo: { bar: ['other_value_1', 'other_value_2'] }, }); @@ -692,7 +692,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ foo: { bar: [{ zed: 'other_value_1' }] }, }); @@ -703,7 +703,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ foo: { bar: [{ zed: 'other_value_1' }, { zed: 'other_value_2' }] }, }); @@ -729,7 +729,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(fields); }); @@ -738,7 +738,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(fields); }); @@ -747,7 +747,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(fields); }); @@ -756,7 +756,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(fields); }); }); @@ -782,7 +782,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ foo: { bar: ['other_value_1'], @@ -795,7 +795,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ foo: { bar: ['other_value_1', 'other_value_2'], @@ -808,7 +808,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ foo: { bar: [{ zed: 'other_value_1' }], @@ -821,7 +821,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ foo: { bar: [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], @@ -849,7 +849,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(fields); }); @@ -858,7 +858,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(fields); }); @@ -867,7 +867,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(fields); }); @@ -876,7 +876,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(fields); }); }); @@ -902,7 +902,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -911,7 +911,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -920,7 +920,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ foo: { bar: { zed: 'other_value_1' } }, }); @@ -931,7 +931,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ foo: { bar: [{ zed: 'other_value_1' }, { zed: 'other_value_2' }] }, }); @@ -957,7 +957,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -966,7 +966,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -975,7 +975,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ mars: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ 'foo.bar': { mars: 'other_value_1' }, }); @@ -986,7 +986,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ mars: 'other_value_1' }, { mars: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ 'foo.bar': [{ mars: 'other_value_1' }, { mars: 'other_value_2' }], }); @@ -1014,7 +1014,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -1023,7 +1023,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -1032,7 +1032,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ foo: { bar: [{ zed: 'other_value_1' }] }, }); @@ -1043,7 +1043,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ foo: { bar: [{ zed: 'other_value_1' }, { zed: 'other_value_2' }] }, }); @@ -1069,7 +1069,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -1078,7 +1078,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -1087,7 +1087,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ foo: { bar: [{ zed: 'other_value_1' }] }, }); @@ -1098,7 +1098,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ foo: { bar: [{ zed: 'other_value_1' }, { zed: 'other_value_2' }] }, }); @@ -1124,7 +1124,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -1133,7 +1133,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -1142,7 +1142,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ mars: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(fields); }); @@ -1151,7 +1151,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ mars: 'other_value_1' }, { mars: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(fields); }); }); @@ -1175,7 +1175,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -1184,7 +1184,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -1193,7 +1193,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ mars: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(fields); }); @@ -1202,7 +1202,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ mars: 'other_value_1' }, { mars: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(fields); }); }); @@ -1228,7 +1228,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ foo: { bar: 'value_1' }, 'foo.bar': 'other_value_1', @@ -1243,7 +1243,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['value_1', 'value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ foo: { bar: 'value_1' }, // <--- We have duplicated value_1 twice which is a bug 'foo.bar': ['value_1', 'value_2'], // <-- We have merged the array value because we do not understand if we should or not @@ -1270,7 +1270,7 @@ describe('merge_all_fields_with_source', () => { 'bar.keyword': ['bar_other_value_keyword_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ foo: 'foo_other_value_1', bar: 'bar_other_value_1', @@ -1291,7 +1291,7 @@ describe('merge_all_fields_with_source', () => { 'host.hostname.keyword': ['hostname_other_value_keyword_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ host: { hostname: 'hostname_other_value_1', @@ -1316,7 +1316,7 @@ describe('merge_all_fields_with_source', () => { 'foo.host.hostname.keyword': ['hostname_other_value_keyword_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ foo: { host: { @@ -1334,7 +1334,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ foo: 'other_value_1', }); @@ -1354,7 +1354,7 @@ describe('merge_all_fields_with_source', () => { 'host.hostname.keyword': ['hostname_other_value_keyword_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ 'host.name': 'host_name_other_value_1', 'host.hostname': 'hostname_other_value_1', @@ -1373,7 +1373,7 @@ describe('merge_all_fields_with_source', () => { 'foo.host.hostname.keyword': ['hostname_other_value_keyword_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ 'foo.host.name': 'host_name_other_value_1', 'foo.host.hostname': 'hostname_other_value_1', @@ -1388,7 +1388,7 @@ describe('merge_all_fields_with_source', () => { 'foo.bar.zed': ['zed_other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ foo: 'other_value_1', }); @@ -1415,7 +1415,7 @@ describe('merge_all_fields_with_source', () => { 'foo.mars': ['other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -1435,7 +1435,7 @@ describe('merge_all_fields_with_source', () => { 'foo.zed.mars': ['other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); }); @@ -1450,7 +1450,7 @@ describe('merge_all_fields_with_source', () => { foo: [{ bar: ['single_value'], zed: ['single_value'] }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ foo: { bar: 'single_value', zed: 'single_value' }, }); @@ -1469,10 +1469,132 @@ describe('merge_all_fields_with_source', () => { foo: [{ bar: ['single_value'], zed: ['single_value'] }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeAllFieldsWithSource({ doc })._source; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ foo: [{ bar: ['single_value'], zed: ['single_value'] }], }); }); }); + + /** + * Small set of tests to ensure that ignore fields are wired up at the strategy level + */ + describe('ignore fields', () => { + test('Does not merge an ignored field if it does not exist already in the _source', () => { + const _source: SignalSourceHit['_source'] = {}; + const fields: SignalSourceHit['fields'] = { + 'foo.bar': ['other_value_1'], + 'value.should.ignore': ['other_value_2'], // string value should ignore this + '_odd.value': ['other_value_2'], // Regex should ignore this value of: /[_]+/ + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: ['value.should.ignore', '/[_]+/'], + })._source; + expect(merged).toEqual({ + foo: { + bar: 'other_value_1', + }, + }); + }); + + test('Does merge fields when no matching happens', () => { + const _source: SignalSourceHit['_source'] = {}; + const fields: SignalSourceHit['fields'] = { + 'foo.bar': ['other_value_1'], + 'value.should.work': ['other_value_2'], + '_odd.value': ['other_value_2'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: ['other.string', '/[z]+/'], // Neither of these two should match anything + })._source; + expect(merged).toEqual({ + foo: { + bar: 'other_value_1', + }, + _odd: { + value: 'other_value_2', + }, + value: { + should: { + work: 'other_value_2', + }, + }, + }); + }); + + test('Does not update an ignored field and keeps the original value if it matches in the ignoreFields', () => { + const _source: SignalSourceHit['_source'] = { + 'value.should.ignore': ['value_1'], + '_odd.value': ['value_2'], + }; + const fields: SignalSourceHit['fields'] = { + 'foo.bar': ['other_value_1'], + 'value.should.ignore': ['other_value_2'], // string value should ignore this + '_odd.value': ['other_value_2'], // Regex should ignore this value of: /[_]+/ + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: ['value.should.ignore', '/[_]+/'], + })._source; + expect(merged).toEqual({ + foo: { + bar: 'other_value_1', + }, + 'value.should.ignore': ['value_1'], + '_odd.value': ['value_2'], + }); + }); + + test('Does not ignore anything when no matching happens and overwrites the expected fields', () => { + const _source: SignalSourceHit['_source'] = { + 'value.should.ignore': ['value_1'], + '_odd.value': ['value_2'], + }; + const fields: SignalSourceHit['fields'] = { + 'foo.bar': ['other_value_1'], + 'value.should.ignore': ['other_value_2'], + '_odd.value': ['other_value_2'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: ['nothing.to.match', '/[z]+/'], // these match nothing + })._source; + expect(merged).toEqual({ + foo: { + bar: 'other_value_1', + }, + 'value.should.ignore': ['other_value_2'], + '_odd.value': ['other_value_2'], + }); + }); + }); + + /** + * Test that the EQL bug workaround is wired up. Remove this once the bug is fixed. + */ + describe('Works around EQL bug 77152 (https://github.com/elastic/elasticsearch/issues/77152)', () => { + test('Does not merge field that contains _ignored', () => { + const _source: SignalSourceHit['_source'] = {}; + const fields: SignalSourceHit['fields'] = { + 'foo.bar': ['other_value_1'], + _ignored: ['other_value_1'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: [], + })._source; + expect(merged).toEqual({ + foo: { + bar: 'other_value_1', + }, + }); + }); + }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/strategies/merge_all_fields_with_source.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/strategies/merge_all_fields_with_source.ts index da2eea9d2c61e..ade83b88d526b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/strategies/merge_all_fields_with_source.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/strategies/merge_all_fields_with_source.ts @@ -23,14 +23,16 @@ import { isTypeObject } from '../utils/is_type_object'; * on this function and the general strategies. * * @param doc The document with "_source" and "fields" - * @param throwOnFailSafe Defaults to false, but if set to true it will cause a throw if the fail safe is triggered to indicate we need to add a new explicit test condition + * @param ignoreFields Any fields that we should ignore and never merge from "fields". If the value exists + * within doc._source it will be untouched and used. If the value does not exist within the doc._source, + * it will not be added from fields. * @returns The two merged together in one object where we can */ -export const mergeAllFieldsWithSource: MergeStrategyFunction = ({ doc }) => { +export const mergeAllFieldsWithSource: MergeStrategyFunction = ({ doc, ignoreFields }) => { const source = doc._source ?? {}; const fields = doc.fields ?? {}; const fieldEntries = Object.entries(fields); - const filteredEntries = filterFieldEntries(fieldEntries); + const filteredEntries = filterFieldEntries(fieldEntries, ignoreFields); const transformedSource = filteredEntries.reduce( (merged, [fieldsKey, fieldsValue]: [string, FieldsType]) => { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/strategies/merge_missing_fields_with_source.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/strategies/merge_missing_fields_with_source.test.ts index 70d1e79580e84..612bff75792da 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/strategies/merge_missing_fields_with_source.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/strategies/merge_missing_fields_with_source.test.ts @@ -44,7 +44,7 @@ describe('merge_missing_fields_with_source', () => { test('when source is "undefined", merged doc is "undefined"', () => { const _source: SignalSourceHit['_source'] = {}; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -53,7 +53,7 @@ describe('merge_missing_fields_with_source', () => { foo: [], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -62,7 +62,7 @@ describe('merge_missing_fields_with_source', () => { foo: 'value', }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -71,7 +71,7 @@ describe('merge_missing_fields_with_source', () => { foo: ['value'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -80,7 +80,7 @@ describe('merge_missing_fields_with_source', () => { foo: ['value_1', 'value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -89,7 +89,7 @@ describe('merge_missing_fields_with_source', () => { foo: { bar: 'some value' }, }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -98,7 +98,7 @@ describe('merge_missing_fields_with_source', () => { foo: [{ bar: 'some value' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -107,7 +107,7 @@ describe('merge_missing_fields_with_source', () => { foo: [{ bar: 'some value' }, { foo: 'some other value' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); }); @@ -133,7 +133,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -142,7 +142,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': 'value', }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -151,7 +151,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['value'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -160,7 +160,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['value_1', 'value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -169,7 +169,7 @@ describe('merge_missing_fields_with_source', () => { foo: { bar: 'some value' }, }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -178,7 +178,7 @@ describe('merge_missing_fields_with_source', () => { foo: [{ bar: 'some value' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -187,7 +187,7 @@ describe('merge_missing_fields_with_source', () => { foo: [{ bar: 'some value' }, { foo: 'some other value' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); }); @@ -217,7 +217,7 @@ describe('merge_missing_fields_with_source', () => { foo: { bar: [] }, }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -226,7 +226,7 @@ describe('merge_missing_fields_with_source', () => { foo: { bar: 'value' }, }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -235,7 +235,7 @@ describe('merge_missing_fields_with_source', () => { foo: { bar: ['value'] }, }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -244,7 +244,7 @@ describe('merge_missing_fields_with_source', () => { foo: { bar: ['value_1', 'value_2'] }, }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -253,7 +253,7 @@ describe('merge_missing_fields_with_source', () => { foo: { bar: { mars: 'some value' } }, }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -262,7 +262,7 @@ describe('merge_missing_fields_with_source', () => { foo: { bar: [{ mars: 'some value' }] }, }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -271,7 +271,7 @@ describe('merge_missing_fields_with_source', () => { foo: { bar: [{ mars: 'some value' }, { mars: 'some other value' }] }, }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); }); @@ -299,7 +299,7 @@ describe('merge_missing_fields_with_source', () => { 'bar.foo': [], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -308,7 +308,7 @@ describe('merge_missing_fields_with_source', () => { 'bar.foo': 'value', }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -317,7 +317,7 @@ describe('merge_missing_fields_with_source', () => { 'bar.foo': ['value'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -326,7 +326,7 @@ describe('merge_missing_fields_with_source', () => { 'bar.foo': ['value_1', 'value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -335,7 +335,7 @@ describe('merge_missing_fields_with_source', () => { foo: { bar: 'some value' }, }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -344,7 +344,7 @@ describe('merge_missing_fields_with_source', () => { foo: [{ bar: 'some value' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -353,7 +353,7 @@ describe('merge_missing_fields_with_source', () => { foo: [{ bar: 'some value' }, { foo: 'some other value' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); }); @@ -376,7 +376,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ foo: { bar: 'other_value_1', @@ -389,7 +389,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ foo: { bar: ['other_value_1', 'other_value_2'], @@ -402,7 +402,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({}); }); @@ -411,7 +411,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({}); }); }); @@ -436,7 +436,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -445,7 +445,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -454,7 +454,7 @@ describe('merge_missing_fields_with_source', () => { foo: [{ bar: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -463,7 +463,7 @@ describe('merge_missing_fields_with_source', () => { foo: [{ bar: 'other_value_1' }, { bar: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); }); @@ -487,7 +487,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -496,7 +496,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -505,7 +505,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -514,7 +514,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); }); @@ -540,7 +540,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -549,7 +549,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -558,7 +558,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -567,7 +567,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); }); @@ -591,7 +591,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -600,7 +600,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -609,7 +609,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -618,7 +618,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); }); @@ -642,7 +642,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -651,7 +651,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -660,7 +660,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -669,7 +669,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); }); @@ -693,7 +693,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -702,7 +702,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -711,7 +711,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -720,7 +720,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); }); @@ -746,7 +746,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -755,7 +755,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -764,7 +764,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -773,7 +773,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); }); @@ -797,7 +797,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -806,7 +806,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -815,7 +815,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -824,7 +824,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); }); @@ -850,7 +850,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -859,7 +859,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -868,7 +868,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -877,7 +877,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); }); @@ -901,7 +901,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -910,7 +910,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -919,7 +919,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ mars: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -928,7 +928,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ mars: 'other_value_1' }, { mars: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); }); @@ -954,7 +954,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -963,7 +963,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -972,7 +972,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -981,7 +981,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); }); @@ -1005,7 +1005,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -1014,7 +1014,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -1023,7 +1023,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -1032,7 +1032,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); }); @@ -1056,7 +1056,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -1065,7 +1065,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -1074,7 +1074,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ mars: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -1083,7 +1083,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ mars: 'other_value_1' }, { mars: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); }); @@ -1107,7 +1107,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -1116,7 +1116,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -1125,7 +1125,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ mars: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -1134,7 +1134,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ mars: 'other_value_1' }, { mars: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); }); @@ -1160,7 +1160,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -1172,7 +1172,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['value_1', 'value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); }); @@ -1196,7 +1196,7 @@ describe('merge_missing_fields_with_source', () => { 'bar.keyword': ['bar_other_value_keyword_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -1214,7 +1214,7 @@ describe('merge_missing_fields_with_source', () => { 'host.hostname.keyword': ['hostname_other_value_keyword_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -1234,7 +1234,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.host.hostname.keyword': ['hostname_other_value_keyword_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -1245,7 +1245,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ foo: 'other_value_1', }); @@ -1265,7 +1265,7 @@ describe('merge_missing_fields_with_source', () => { 'host.hostname.keyword': ['hostname_other_value_keyword_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -1281,7 +1281,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.host.hostname.keyword': ['hostname_other_value_keyword_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -1293,7 +1293,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar.zed': ['zed_other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({ foo: 'other_value_1', }); @@ -1320,7 +1320,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.mars': ['other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); @@ -1340,7 +1340,7 @@ describe('merge_missing_fields_with_source', () => { 'foo.zed.mars': ['other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); }); @@ -1355,7 +1355,7 @@ describe('merge_missing_fields_with_source', () => { foo: [{ bar: ['single_value'], zed: ['single_value'] }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual({}); }); @@ -1372,8 +1372,82 @@ describe('merge_missing_fields_with_source', () => { foo: [{ bar: ['single_value'], zed: ['single_value'] }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; - const merged = mergeMissingFieldsWithSource({ doc })._source; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; expect(merged).toEqual(_source); }); }); + + /** + * Small set of tests to ensure that ignore fields are wired up at the strategy level + */ + describe('ignore fields', () => { + test('Does not merge an ignored field if it does not exist already in the _source', () => { + const _source: SignalSourceHit['_source'] = {}; + const fields: SignalSourceHit['fields'] = { + 'foo.bar': ['other_value_1'], + 'value.should.ignore': ['other_value_2'], // string value should ignore this + '_odd.value': ['other_value_2'], // Regex should ignore this value of: /[_]+/ + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: ['value.should.ignore', '/[_]+/'], + })._source; + expect(merged).toEqual({ + foo: { + bar: 'other_value_1', + }, + }); + }); + + test('Does merge fields when no matching happens', () => { + const _source: SignalSourceHit['_source'] = {}; + const fields: SignalSourceHit['fields'] = { + 'foo.bar': ['other_value_1'], + 'value.should.work': ['other_value_2'], + '_odd.value': ['other_value_2'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: ['other.string', '/[z]+/'], // Neither of these two should match anything + })._source; + expect(merged).toEqual({ + foo: { + bar: 'other_value_1', + }, + _odd: { + value: 'other_value_2', + }, + value: { + should: { + work: 'other_value_2', + }, + }, + }); + }); + }); + + /** + * Test that the EQL bug workaround is wired up. Remove this once the bug is fixed. + */ + describe('Works around EQL bug 77152 (https://github.com/elastic/elasticsearch/issues/77152)', () => { + test('Does not merge field that contains _ignored', () => { + const _source: SignalSourceHit['_source'] = {}; + const fields: SignalSourceHit['fields'] = { + 'foo.bar': ['other_value_1'], + _ignored: ['other_value_1'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: [], + })._source; + expect(merged).toEqual({ + foo: { + bar: 'other_value_1', + }, + }); + }); + }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/strategies/merge_missing_fields_with_source.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/strategies/merge_missing_fields_with_source.ts index b66c46ccbf0ca..611a3ad879705 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/strategies/merge_missing_fields_with_source.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/strategies/merge_missing_fields_with_source.ts @@ -19,14 +19,16 @@ import { isNestedObject } from '../utils/is_nested_object'; * Merges only missing sections of "doc._source" with its "doc.fields" on a "best effort" basis. See ../README.md for more information * on this function and the general strategies. * @param doc The document with "_source" and "fields" - * @param throwOnFailSafe Defaults to false, but if set to true it will cause a throw if the fail safe is triggered to indicate we need to add a new explicit test condition + * @param ignoreFields Any fields that we should ignore and never merge from "fields". If the value exists + * within doc._source it will be untouched and used. If the value does not exist within the doc._source, + * it will not be added from fields. * @returns The two merged together in one object where we can */ -export const mergeMissingFieldsWithSource: MergeStrategyFunction = ({ doc }) => { +export const mergeMissingFieldsWithSource: MergeStrategyFunction = ({ doc, ignoreFields }) => { const source = doc._source ?? {}; const fields = doc.fields ?? {}; const fieldEntries = Object.entries(fields); - const filteredEntries = filterFieldEntries(fieldEntries); + const filteredEntries = filterFieldEntries(fieldEntries, ignoreFields); const transformedSource = filteredEntries.reduce( (merged, [fieldsKey, fieldsValue]: [string, FieldsType]) => { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/strategies/merge_no_fields.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/strategies/merge_no_fields.ts index 6c2daf2526715..5e26b619fbdfa 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/strategies/merge_no_fields.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/strategies/merge_no_fields.ts @@ -10,6 +10,7 @@ import { MergeStrategyFunction } from '../types'; /** * Does nothing and does not merge source with fields * @param doc The doc to return and do nothing + * @param ignoreFields We do nothing with this value and ignore it if set * @returns The doc as a no operation and do nothing */ -export const mergeNoFields: MergeStrategyFunction = ({ doc }) => doc; +export const mergeNoFields: MergeStrategyFunction = ({ doc, ignoreFields }) => doc; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/types.ts index 1438d2844949c..0b847064d5d62 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/types.ts @@ -14,5 +14,13 @@ export type FieldsType = string[] | number[] | boolean[] | object[]; /** * The type of the merge strategy functions which must implement to be part of the strategy group + * @param doc The document to send in to merge + * @param ignoreFields Fields you want to ignore and not merge. */ -export type MergeStrategyFunction = ({ doc }: { doc: SignalSourceHit }) => SignalSourceHit; +export type MergeStrategyFunction = ({ + doc, + ignoreFields, +}: { + doc: SignalSourceHit; + ignoreFields: string[]; +}) => SignalSourceHit; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/filter_field_entries.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/filter_field_entries.test.ts index 9cc2478290885..031a2013b462e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/filter_field_entries.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/filter_field_entries.test.ts @@ -27,7 +27,9 @@ describe('filter_field_entries', () => { test('returns a single valid fieldEntries as expected', () => { const fieldEntries: Array<[string, FieldsType]> = [['foo.bar', dummyValue]]; - expect(filterFieldEntries(fieldEntries)).toEqual(fieldEntries); + expect(filterFieldEntries(fieldEntries, [])).toEqual( + fieldEntries + ); }); test('removes invalid dotted entries', () => { @@ -37,7 +39,7 @@ describe('filter_field_entries', () => { ['..', dummyValue], ['foo..bar', dummyValue], ]; - expect(filterFieldEntries(fieldEntries)).toEqual([ + expect(filterFieldEntries(fieldEntries, [])).toEqual([ ['foo.bar', dummyValue], ]); }); @@ -49,7 +51,7 @@ describe('filter_field_entries', () => { ['bar.keyword', dummyValue], // <-- "bar.keyword" multi-field should be removed ['bar', dummyValue], ]; - expect(filterFieldEntries(fieldEntries)).toEqual([ + expect(filterFieldEntries(fieldEntries, [])).toEqual([ ['foo', dummyValue], ['bar', dummyValue], ]); @@ -62,7 +64,7 @@ describe('filter_field_entries', () => { ['host.hostname', dummyValue], ['host.hostname.keyword', dummyValue], // <-- multi-field should be removed ]; - expect(filterFieldEntries(fieldEntries)).toEqual([ + expect(filterFieldEntries(fieldEntries, [])).toEqual([ ['host.name', dummyValue], ['host.hostname', dummyValue], ]); @@ -75,9 +77,34 @@ describe('filter_field_entries', () => { ['foo.host.hostname', dummyValue], ['foo.host.hostname.keyword', dummyValue], // <-- multi-field should be removed ]; - expect(filterFieldEntries(fieldEntries)).toEqual([ + expect(filterFieldEntries(fieldEntries, [])).toEqual([ ['foo.host.name', dummyValue], ['foo.host.hostname', dummyValue], ]); }); + + test('ignores fields of "_ignore", for eql bug https://github.com/elastic/elasticsearch/issues/77152', () => { + const fieldEntries: Array<[string, FieldsType]> = [ + ['_ignored', dummyValue], + ['foo.host.hostname', dummyValue], + ]; + expect(filterFieldEntries(fieldEntries, [])).toEqual([ + ['foo.host.hostname', dummyValue], + ]); + }); + + test('ignores fields given strings and regular expressions in the ignoreFields list', () => { + const fieldEntries: Array<[string, FieldsType]> = [ + ['host.name', dummyValue], + ['user.name', dummyValue], // <-- string from ignoreFields should ignore this + ['host.hostname', dummyValue], + ['_odd.value', dummyValue], // <-- regular expression from ignoreFields should ignore this + ]; + expect( + filterFieldEntries(fieldEntries, ['user.name', '/[_]+/']) + ).toEqual([ + ['host.name', dummyValue], + ['host.hostname', dummyValue], + ]); + }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/filter_field_entries.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/filter_field_entries.ts index 221cdabc62847..4ee5fa1db52f5 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/filter_field_entries.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/filter_field_entries.ts @@ -9,6 +9,8 @@ import { isMultiField } from './is_multifield'; import { isInvalidKey } from './is_invalid_key'; import { isTypeObject } from './is_type_object'; import { FieldsType } from '../types'; +import { isIgnored } from './is_ignored'; +import { isEqlBug77152 } from './is_eql_bug_77152'; /** * Filters field entries by removing invalid field entries such as any invalid characters @@ -17,13 +19,18 @@ import { FieldsType } from '../types'; * those and don't try to merge those. * * @param fieldEntries The field entries to filter + * @param ignoreFields Array of fields to ignore. If a value starts and ends with "/", such as: "/[_]+/" then the field will be treated as a regular expression. + * If you have an object structure to ignore such as "{ a: { b: c: {} } } ", then you need to ignore it as the string "a.b.c" * @returns The field entries filtered */ export const filterFieldEntries = ( - fieldEntries: Array<[string, FieldsType]> + fieldEntries: Array<[string, FieldsType]>, + ignoreFields: string[] ): Array<[string, FieldsType]> => { return fieldEntries.filter(([fieldsKey, fieldsValue]: [string, FieldsType]) => { return ( + !isEqlBug77152(fieldsKey) && + !isIgnored(fieldsKey, ignoreFields) && !isInvalidKey(fieldsKey) && !isMultiField(fieldsKey, fieldEntries) && !isTypeObject(fieldsValue) // TODO: Look at not filtering this and instead transform it so it can be inserted correctly in the strategies which does an overwrite of everything from fields diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/index.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/index.ts index baf9efca511e2..87b1097dd9bca 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/index.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/index.ts @@ -7,6 +7,7 @@ export * from './array_in_path_exists'; export * from './filter_field_entries'; export * from './is_array_of_primitives'; +export * from './is_ignored'; export * from './is_invalid_key'; export * from './is_multifield'; export * from './is_nested_object'; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_eql_bug_77152.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_eql_bug_77152.test.ts new file mode 100644 index 0000000000000..47a56e096649b --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_eql_bug_77152.test.ts @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { isEqlBug77152 } from './is_eql_bug_77152'; + +/** + * @deprecated Remove this test once https://github.com/elastic/elasticsearch/issues/77152 is fixed. + */ +describe('is_eql_bug_77152', () => { + beforeAll(() => { + jest.resetAllMocks(); + }); + + afterEach(() => { + jest.resetAllMocks(); + }); + + it('returns true if it encounters the bug which is _ignored is returned in the fields', () => { + expect(isEqlBug77152('_ignored')).toEqual(true); + }); + + it('returns false if it encounters a normal field', () => { + expect(isEqlBug77152('some.field')).toEqual(false); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_eql_bug_77152.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_eql_bug_77152.ts new file mode 100644 index 0000000000000..e9a642cd5c382 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_eql_bug_77152.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/** + * Ignores any field that is "_ignored". Customers are not allowed to have this field and more importantly this shows up as a bug + * from EQL as seen here: https://github.com/elastic/elasticsearch/issues/77152 + * Once this ticket is fixed, please remove this function. + * @param fieldsKey The fields key to match against "_ignored" + * @returns true if it is a "_ignored", otherwise false + * @deprecated Remove this once https://github.com/elastic/elasticsearch/issues/77152 is fixed. + */ +export const isEqlBug77152 = (fieldsKey: string): boolean => { + return fieldsKey === '_ignored'; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_ignored.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_ignored.test.ts new file mode 100644 index 0000000000000..e4a7093ef127c --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_ignored.test.ts @@ -0,0 +1,84 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { isIgnored } from './is_ignored'; + +describe('is_ignored', () => { + beforeAll(() => { + jest.resetAllMocks(); + }); + + afterEach(() => { + jest.resetAllMocks(); + }); + + describe('string matching', () => { + test('it returns false if given an empty array', () => { + expect(isIgnored('simple.value', [])).toEqual(false); + }); + + test('it returns true if a simple string value matches', () => { + expect(isIgnored('simple.value', ['simple.value'])).toEqual(true); + }); + + test('it returns false if a simple string value does not match', () => { + expect(isIgnored('simple', ['simple.value'])).toEqual(false); + }); + + test('it returns true if a simple string value matches with two strings', () => { + expect(isIgnored('simple.value', ['simple.value', 'simple.second.value'])).toEqual(true); + }); + + test('it returns true if a simple string value matches the second string', () => { + expect(isIgnored('simple.second.value', ['simple.value', 'simple.second.value'])).toEqual( + true + ); + }); + + test('it returns false if a simple string value does not match two strings', () => { + expect(isIgnored('simple', ['simple.value', 'simple.second.value'])).toEqual(false); + }); + + test('it returns true if mixed with a regular expression in the list', () => { + expect(isIgnored('simple', ['simple', '/[_]+/'])).toEqual(true); + }); + }); + + describe('regular expression matching', () => { + test('it returns true if a simple regular expression matches', () => { + expect(isIgnored('_ignored', ['/[_]+/'])).toEqual(true); + }); + + test('it returns false if a simple regular expression does not match', () => { + expect(isIgnored('simple', ['/[_]+/'])).toEqual(false); + }); + + test('it returns true if a simple regular expression matches a longer string', () => { + expect(isIgnored('___ignored', ['/[_]+/'])).toEqual(true); + }); + + test('it returns true if mixed with regular stings', () => { + expect(isIgnored('___ignored', ['simple', '/[_]+/'])).toEqual(true); + }); + + test('it returns true with start anchor', () => { + expect(isIgnored('_ignored', ['simple', '/^[_]+/'])).toEqual(true); + }); + + test('it returns false with start anchor', () => { + expect(isIgnored('simple.something_', ['simple', '/^[_]+/'])).toEqual(false); + }); + + test('it returns true with end anchor', () => { + expect(isIgnored('something_', ['simple', '/[_]+$/'])).toEqual(true); + }); + + test('it returns false with end anchor', () => { + expect(isIgnored('_something', ['simple', '/[_]+$/'])).toEqual(false); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_ignored.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_ignored.ts new file mode 100644 index 0000000000000..a418ce735626d --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_ignored.ts @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/** + * Matches against anything you want to ignore and if it matches that field is ignored. + * @param fieldsKey The fields key to match against + * @param ignoreFields Array of fields to ignore. If a value starts and ends with "/", such as: "/[_]+/" then the field will be treated as a regular expression. + * If you have an object structure to ignore such as "{ a: { b: c: {} } } ", then you need to ignore it as the string "a.b.c" + * @returns true if it is a field to ignore, otherwise false + */ +export const isIgnored = (fieldsKey: string, ignoreFields: string[]): boolean => { + return ignoreFields.some((ignoreField) => { + if (ignoreField.startsWith('/') && ignoreField.endsWith('/')) { + return new RegExp(ignoreField.slice(1, -1)).test(fieldsKey); + } else { + return fieldsKey === ignoreField; + } + }); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/wrap_hits_factory.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/wrap_hits_factory.ts index b28c46aae8f82..e4874521f399b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/wrap_hits_factory.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/wrap_hits_factory.ts @@ -15,10 +15,12 @@ export const wrapHitsFactory = ({ ruleSO, signalsIndex, mergeStrategy, + ignoreFields, }: { ruleSO: SearchAfterAndBulkCreateParams['ruleSO']; signalsIndex: string; mergeStrategy: ConfigType['alertMergeStrategy']; + ignoreFields: ConfigType['alertIgnoreFields']; }): WrapHits => (events) => { const wrappedDocs: WrappedSignalHit[] = events.flatMap((doc) => [ { @@ -29,7 +31,7 @@ export const wrapHitsFactory = ({ String(doc._version), ruleSO.attributes.params.ruleId ?? '' ), - _source: buildBulkBody(ruleSO, doc, mergeStrategy), + _source: buildBulkBody(ruleSO, doc, mergeStrategy, ignoreFields), }, ]); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/wrap_sequences_factory.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/wrap_sequences_factory.ts index f0b9e64047692..57d1ddc2d1763 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/wrap_sequences_factory.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/wrap_sequences_factory.ts @@ -13,15 +13,17 @@ export const wrapSequencesFactory = ({ ruleSO, signalsIndex, mergeStrategy, + ignoreFields, }: { ruleSO: SearchAfterAndBulkCreateParams['ruleSO']; signalsIndex: string; mergeStrategy: ConfigType['alertMergeStrategy']; + ignoreFields: ConfigType['alertIgnoreFields']; }): WrapSequences => (sequences) => sequences.reduce( (acc: WrappedSignalHit[], sequence) => [ ...acc, - ...buildSignalGroupFromSequence(sequence, ruleSO, signalsIndex, mergeStrategy), + ...buildSignalGroupFromSequence(sequence, ruleSO, signalsIndex, mergeStrategy, ignoreFields), ], [] ); diff --git a/x-pack/plugins/security_solution/server/plugin.ts b/x-pack/plugins/security_solution/server/plugin.ts index 2f3850ff49f4c..9def54fdd8580 100644 --- a/x-pack/plugins/security_solution/server/plugin.ts +++ b/x-pack/plugins/security_solution/server/plugin.ts @@ -388,6 +388,7 @@ export class Plugin implements IPlugin `--xpack.${key}.enabled=false`), ...(ssl ? [ diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/ignore_fields.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/ignore_fields.ts new file mode 100644 index 0000000000000..409128523ea40 --- /dev/null +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/ignore_fields.ts @@ -0,0 +1,133 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; + +import { FtrProviderContext } from '../../../common/ftr_provider_context'; +import { + createRule, + createSignalsIndex, + deleteAllAlerts, + deleteSignalsIndex, + getEqlRuleForSignalTesting, + getSignalsById, + waitForRuleSuccessOrStatus, + waitForSignalsToBePresent, +} from '../../utils'; + +interface Ignore { + normal_constant?: string; + small_field?: string; + testing_ignored?: string; + testing_regex?: string; +} + +// eslint-disable-next-line import/no-default-export +export default ({ getService }: FtrProviderContext): void => { + /** + * See the config file (detection_engine_api_integration/common/config.ts) for which field values were added to be ignored + * for testing. The values should be in the config around the area of: + * --xpack.securitySolution.alertIgnoreFields=[testing.ignore_1,/[testingRegex] + * meaning that the ignore fields values should be the array: ["testing.ignore_1", "/[testingRegex]/"] + * + * This test exercises the ability to be able to ignore particular values within the fields API and merge strategies. + * These values can be defined in your kibana.yml file as "xpack.securitySolution.alertIgnoreFields". This is useful + * for users that find bugs or regressions within query languages or bugs within the merge strategies + * where one or more fields are causing problems and they need to turn disable that particular field. + * + * Ref: + * https://github.com/elastic/kibana/issues/110802 + * https://github.com/elastic/elasticsearch/issues/77152 + * + * Files ref: + * server/lib/detection_engine/signals/source_fields_merging/utils/is_ignored.ts + * server/lib/detection_engine/signals/source_fields_merging/utils/is_eql_bug_77152.ts + */ + describe('ignore_fields', () => { + const supertest = getService('supertest'); + const esArchiver = getService('esArchiver'); + + before(async () => { + await esArchiver.load('x-pack/test/functional/es_archives/security_solution/ignore_fields'); + }); + + after(async () => { + await esArchiver.unload('x-pack/test/functional/es_archives/security_solution/ignore_fields'); + }); + + beforeEach(async () => { + await createSignalsIndex(supertest); + }); + + afterEach(async () => { + await deleteSignalsIndex(supertest); + await deleteAllAlerts(supertest); + }); + + it('should ignore the field of "testing_ignored"', async () => { + const rule = getEqlRuleForSignalTesting(['ignore_fields']); + + const { id } = await createRule(supertest, rule); + await waitForRuleSuccessOrStatus(supertest, id); + await waitForSignalsToBePresent(supertest, 4, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const hits = signalsOpen.hits.hits + .map((hit) => (hit._source as Ignore).testing_ignored) + .sort(); + + // Value should be "undefined for all records" + expect(hits).to.eql([undefined, undefined, undefined, undefined]); + }); + + it('should ignore the field of "testing_regex"', async () => { + const rule = getEqlRuleForSignalTesting(['ignore_fields']); + + const { id } = await createRule(supertest, rule); + await waitForRuleSuccessOrStatus(supertest, id); + await waitForSignalsToBePresent(supertest, 4, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const hits = signalsOpen.hits.hits.map((hit) => (hit._source as Ignore).testing_regex).sort(); + + // Value should be "undefined for all records" + expect(hits).to.eql([undefined, undefined, undefined, undefined]); + }); + + it('should have the field of "normal_constant"', async () => { + const rule = getEqlRuleForSignalTesting(['ignore_fields']); + + const { id } = await createRule(supertest, rule); + await waitForRuleSuccessOrStatus(supertest, id); + await waitForSignalsToBePresent(supertest, 4, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const hits = signalsOpen.hits.hits + .map((hit) => (hit._source as Ignore).normal_constant) + .sort(); + + // Value should be "constant_value for all records" + expect(hits).to.eql(['constant_value', 'constant_value', 'constant_value', 'constant_value']); + }); + + // TODO: Remove this test once https://github.com/elastic/elasticsearch/issues/77152 is fixed + it('should ignore the field of "_ignored" when using EQL and index the data', async () => { + const rule = getEqlRuleForSignalTesting(['ignore_fields']); + + const { id } = await createRule(supertest, rule); + await waitForRuleSuccessOrStatus(supertest, id); + await waitForSignalsToBePresent(supertest, 4, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const hits = signalsOpen.hits.hits.map((hit) => (hit._source as Ignore).small_field).sort(); + + // We just test a constant value to ensure this did not blow up on us and did index data. + expect(hits).to.eql([ + '1 indexed', + '2 large not indexed', + '3 large not indexed', + '4 large not indexed', + ]); + }); + }); +}; diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/index.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/index.ts index 44e6023bf366a..ef2e03b8e416e 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/index.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/index.ts @@ -47,6 +47,7 @@ export default ({ loadTestFile }: FtrProviderContext): void => { loadTestFile(require.resolve('./delete_signals_migrations')); loadTestFile(require.resolve('./timestamps')); loadTestFile(require.resolve('./runtime')); + loadTestFile(require.resolve('./ignore_fields')); }); // That split here enable us on using a different ciGroup to run the tests diff --git a/x-pack/test/functional/es_archives/security_solution/ignore_fields/data.json b/x-pack/test/functional/es_archives/security_solution/ignore_fields/data.json new file mode 100644 index 0000000000000..7a33785c0bc41 --- /dev/null +++ b/x-pack/test/functional/es_archives/security_solution/ignore_fields/data.json @@ -0,0 +1,51 @@ +{ + "type": "doc", + "value": { + "id": "1", + "index": "ignore_fields", + "source": { + "@timestamp": "2020-10-28T05:00:53.000Z", + "small_field": "1 indexed" + }, + "type": "_doc" + } +} + +{ + "type": "doc", + "value": { + "id": "2", + "index": "ignore_fields", + "source": { + "@timestamp": "2020-10-28T05:01:53.000Z", + "small_field": "2 large not indexed" + }, + "type": "_doc" + } +} + +{ + "type": "doc", + "value": { + "id": "3", + "index": "ignore_fields", + "source": { + "@timestamp": "2020-10-28T05:02:53.000Z", + "small_field": "3 large not indexed" + }, + "type": "_doc" + } +} + +{ + "type": "doc", + "value": { + "id": "4", + "index": "ignore_fields", + "source": { + "@timestamp": "2020-10-28T05:03:53.000Z", + "small_field": "4 large not indexed" + }, + "type": "_doc" + } +} diff --git a/x-pack/test/functional/es_archives/security_solution/ignore_fields/mappings.json b/x-pack/test/functional/es_archives/security_solution/ignore_fields/mappings.json new file mode 100644 index 0000000000000..e2c8ca3c2bc89 --- /dev/null +++ b/x-pack/test/functional/es_archives/security_solution/ignore_fields/mappings.json @@ -0,0 +1,41 @@ +{ + "type": "index", + "value": { + "index": "ignore_fields", + "mappings": { + "dynamic": "strict", + "properties": { + "@timestamp": { + "type": "date" + }, + "testing_ignored": { + "properties": { + "constant": { + "type": "constant_keyword", + "value": "constant_value" + } + } + }, + "testing_regex": { + "type": "constant_keyword", + "value": "constant_value" + }, + "normal_constant": { + "type": "constant_keyword", + "value": "constant_value" + }, + "small_field": { + "type": "keyword", + "ignore_above": 10 + } + } + }, + "settings": { + "index": { + "refresh_interval": "1s", + "number_of_replicas": "1", + "number_of_shards": "1" + } + } + } +}