From 2d11ea9351e01d52b068eed4a0ddc0a0475655a4 Mon Sep 17 00:00:00 2001 From: Aaron Caldwell <aaron.caldwell@elastic.co> Date: Wed, 23 Dec 2020 10:29:40 -0700 Subject: [PATCH 01/10] Clean up old offset logic --- .../alert_types/geo_containment/types.ts | 1 - .../alert_types/geo_containment/alert_type.ts | 2 -- .../geo_containment/geo_containment.ts | 35 +++++-------------- .../geo_containment/tests/alert_type.test.ts | 1 - 4 files changed, 8 insertions(+), 31 deletions(-) diff --git a/x-pack/plugins/stack_alerts/public/alert_types/geo_containment/types.ts b/x-pack/plugins/stack_alerts/public/alert_types/geo_containment/types.ts index d1f64c9298f15..c67043106c0d1 100644 --- a/x-pack/plugins/stack_alerts/public/alert_types/geo_containment/types.ts +++ b/x-pack/plugins/stack_alerts/public/alert_types/geo_containment/types.ts @@ -18,7 +18,6 @@ export interface GeoContainmentAlertParams extends AlertTypeParams { boundaryIndexId: string; boundaryGeoField: string; boundaryNameField?: string; - delayOffsetWithUnits?: string; indexQuery?: Query; boundaryIndexQuery?: Query; } diff --git a/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/alert_type.ts b/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/alert_type.ts index 85dcf1125becd..1562b71e3224e 100644 --- a/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/alert_type.ts +++ b/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/alert_type.ts @@ -97,7 +97,6 @@ export const ParamsSchema = schema.object({ boundaryIndexId: schema.string({ minLength: 1 }), boundaryGeoField: schema.string({ minLength: 1 }), boundaryNameField: schema.maybe(schema.string({ minLength: 1 })), - delayOffsetWithUnits: schema.maybe(schema.string({ minLength: 1 })), indexQuery: schema.maybe(schema.any({})), boundaryIndexQuery: schema.maybe(schema.any({})), }); @@ -113,7 +112,6 @@ export interface GeoContainmentParams extends AlertTypeParams { boundaryIndexId: string; boundaryGeoField: string; boundaryNameField?: string; - delayOffsetWithUnits?: string; indexQuery?: Query; boundaryIndexQuery?: Query; } diff --git a/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/geo_containment.ts b/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/geo_containment.ts index 612eff3014985..4a2ec4ef08e03 100644 --- a/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/geo_containment.ts +++ b/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/geo_containment.ts @@ -78,23 +78,6 @@ export function transformResults( return orderedResults; } -function getOffsetTime(delayOffsetWithUnits: string, oldTime: Date): Date { - const timeUnit = delayOffsetWithUnits.slice(-1); - const time: number = +delayOffsetWithUnits.slice(0, -1); - - const adjustedDate = new Date(oldTime.getTime()); - if (timeUnit === 's') { - adjustedDate.setSeconds(adjustedDate.getSeconds() - time); - } else if (timeUnit === 'm') { - adjustedDate.setMinutes(adjustedDate.getMinutes() - time); - } else if (timeUnit === 'h') { - adjustedDate.setHours(adjustedDate.getHours() - time); - } else if (timeUnit === 'd') { - adjustedDate.setDate(adjustedDate.getDate() - time); - } - return adjustedDate; -} - export function getActiveEntriesAndGenerateAlerts( prevLocationMap: Record<string, LatestEntityLocation>, currLocationMap: Map<string, LatestEntityLocation>, @@ -130,7 +113,14 @@ export function getActiveEntriesAndGenerateAlerts( return allActiveEntriesMap; } export const getGeoContainmentExecutor = (log: Logger): GeoContainmentAlertType['executor'] => - async function ({ previousStartedAt, startedAt, services, params, alertId, state }) { + async function ({ + previousStartedAt: currIntervalStartTime, + startedAt: currIntervalEndTime, + services, + params, + alertId, + state, + }) { const { shapesFilters, shapesIdsNamesMap } = state.shapesFilters ? state : await getShapesFilters( @@ -146,15 +136,6 @@ export const getGeoContainmentExecutor = (log: Logger): GeoContainmentAlertType[ const executeEsQuery = await executeEsQueryFactory(params, services, log, shapesFilters); - let currIntervalStartTime = previousStartedAt; - let currIntervalEndTime = startedAt; - if (params.delayOffsetWithUnits) { - if (currIntervalStartTime) { - currIntervalStartTime = getOffsetTime(params.delayOffsetWithUnits, currIntervalStartTime); - } - currIntervalEndTime = getOffsetTime(params.delayOffsetWithUnits, currIntervalEndTime); - } - // Start collecting data only on the first cycle let currentIntervalResults: SearchResponse<unknown> | undefined; if (!currIntervalStartTime) { diff --git a/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/tests/alert_type.test.ts b/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/tests/alert_type.test.ts index 0592c944de570..b9def821987de 100644 --- a/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/tests/alert_type.test.ts +++ b/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/tests/alert_type.test.ts @@ -34,7 +34,6 @@ describe('alertType', () => { boundaryIndexId: 'testIndex', boundaryGeoField: 'testField', boundaryNameField: 'testField', - delayOffsetWithUnits: 'testOffset', }; expect(alertType.validate?.params?.validate(params)).toBeTruthy(); From 97b9e39d280740142a435a0e7e71f6114cf4e735 Mon Sep 17 00:00:00 2001 From: Aaron Caldwell <aaron.caldwell@elastic.co> Date: Wed, 23 Dec 2020 11:10:03 -0700 Subject: [PATCH 02/10] Shift to storing location array per entity --- .../geo_containment/geo_containment.ts | 54 ++++++++++--------- 1 file changed, 29 insertions(+), 25 deletions(-) diff --git a/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/geo_containment.ts b/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/geo_containment.ts index 4a2ec4ef08e03..1d5bc3fa93547 100644 --- a/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/geo_containment.ts +++ b/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/geo_containment.ts @@ -24,7 +24,7 @@ export function transformResults( results: SearchResponse<unknown> | undefined, dateField: string, geoField: string -): Map<string, LatestEntityLocation> { +): Map<string, LatestEntityLocation[]> { if (!results) { return new Map(); } @@ -64,13 +64,15 @@ export function transformResults( // Get unique .reduce( ( - accu: Map<string, LatestEntityLocation>, + accu: Map<string, LatestEntityLocation[]>, el: LatestEntityLocation & { entityName: string } ) => { const { entityName, ...locationData } = el; if (!accu.has(entityName)) { - accu.set(entityName, locationData); + accu.set(entityName, []); } + // @ts-ignore + accu.get(entityName).push(locationData); return accu; }, new Map() @@ -79,8 +81,8 @@ export function transformResults( } export function getActiveEntriesAndGenerateAlerts( - prevLocationMap: Record<string, LatestEntityLocation>, - currLocationMap: Map<string, LatestEntityLocation>, + prevLocationMap: Record<string, LatestEntityLocation[]>, + currLocationMap: Map<string, LatestEntityLocation[]>, alertInstanceFactory: AlertServices< GeoContainmentInstanceState, GeoContainmentInstanceContext @@ -88,27 +90,29 @@ export function getActiveEntriesAndGenerateAlerts( shapesIdsNamesMap: Record<string, unknown>, currIntervalEndTime: Date ) { - const allActiveEntriesMap: Map<string, LatestEntityLocation> = new Map([ + const allActiveEntriesMap: Map<string, LatestEntityLocation[]> = new Map([ ...Object.entries(prevLocationMap || {}), ...currLocationMap, ]); - allActiveEntriesMap.forEach(({ location, shapeLocationId, dateInShape, docId }, entityName) => { - const containingBoundaryName = shapesIdsNamesMap[shapeLocationId] || shapeLocationId; - const context = { - entityId: entityName, - entityDateTime: dateInShape ? new Date(dateInShape).toISOString() : null, - entityDocumentId: docId, - detectionDateTime: new Date(currIntervalEndTime).toISOString(), - entityLocation: `POINT (${location[0]} ${location[1]})`, - containingBoundaryId: shapeLocationId, - containingBoundaryName, - }; - const alertInstanceId = `${entityName}-${containingBoundaryName}`; - if (shapeLocationId === OTHER_CATEGORY) { - allActiveEntriesMap.delete(entityName); - } else { - alertInstanceFactory(alertInstanceId).scheduleActions(ActionGroupId, context); - } + allActiveEntriesMap.forEach((locationsArr, entityName) => { + locationsArr.forEach(({ location, shapeLocationId, dateInShape, docId }) => { + const containingBoundaryName = shapesIdsNamesMap[shapeLocationId] || shapeLocationId; + const context = { + entityId: entityName, + entityDateTime: dateInShape ? new Date(dateInShape).toISOString() : null, + entityDocumentId: docId, + detectionDateTime: new Date(currIntervalEndTime).toISOString(), + entityLocation: `POINT (${location[0]} ${location[1]})`, + containingBoundaryId: shapeLocationId, + containingBoundaryName, + }; + const alertInstanceId = `${entityName}-${containingBoundaryName}`; + if (shapeLocationId === OTHER_CATEGORY) { + allActiveEntriesMap.delete(entityName); + } else { + alertInstanceFactory(alertInstanceId).scheduleActions(ActionGroupId, context); + } + }); }); return allActiveEntriesMap; } @@ -149,14 +153,14 @@ export const getGeoContainmentExecutor = (log: Logger): GeoContainmentAlertType[ currentIntervalResults = await executeEsQuery(currIntervalStartTime, currIntervalEndTime); } - const currLocationMap: Map<string, LatestEntityLocation> = transformResults( + const currLocationMap: Map<string, LatestEntityLocation[]> = transformResults( currentIntervalResults, params.dateField, params.geoField ); const allActiveEntriesMap = getActiveEntriesAndGenerateAlerts( - state.prevLocationMap as Record<string, LatestEntityLocation>, + state.prevLocationMap as Record<string, LatestEntityLocation[]>, currLocationMap, services.alertInstanceFactory, shapesIdsNamesMap, From 0d01fc1118996fb8a8ef5d6f31f19383dff06b96 Mon Sep 17 00:00:00 2001 From: Aaron Caldwell <aaron.caldwell@elastic.co> Date: Mon, 28 Dec 2020 11:39:40 -0700 Subject: [PATCH 03/10] Preserve latest entity-time combinations, resolve the rest --- .../geo_containment/geo_containment.ts | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/geo_containment.ts b/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/geo_containment.ts index 1d5bc3fa93547..b58de2823248d 100644 --- a/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/geo_containment.ts +++ b/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/geo_containment.ts @@ -95,6 +95,7 @@ export function getActiveEntriesAndGenerateAlerts( ...currLocationMap, ]); allActiveEntriesMap.forEach((locationsArr, entityName) => { + // Generate alerts locationsArr.forEach(({ location, shapeLocationId, dateInShape, docId }) => { const containingBoundaryName = shapesIdsNamesMap[shapeLocationId] || shapeLocationId; const context = { @@ -107,15 +108,23 @@ export function getActiveEntriesAndGenerateAlerts( containingBoundaryName, }; const alertInstanceId = `${entityName}-${containingBoundaryName}`; - if (shapeLocationId === OTHER_CATEGORY) { - allActiveEntriesMap.delete(entityName); - } else { + if (shapeLocationId !== OTHER_CATEGORY) { alertInstanceFactory(alertInstanceId).scheduleActions(ActionGroupId, context); } }); + // Determine what remains active + const latestLocationsArr = locationsArr.filter( + (latestEntityLocation) => latestEntityLocation.dateInShape === locationsArr[0].dateInShape + ); + if (latestLocationsArr[0].shapeLocationId === OTHER_CATEGORY) { + allActiveEntriesMap.delete(entityName); + } else { + allActiveEntriesMap.set(entityName, latestLocationsArr); + } }); return allActiveEntriesMap; } + export const getGeoContainmentExecutor = (log: Logger): GeoContainmentAlertType['executor'] => async function ({ previousStartedAt: currIntervalStartTime, From 44cadf7f5669298b514b76120516186d6c025c93 Mon Sep 17 00:00:00 2001 From: Aaron Caldwell <aaron.caldwell@elastic.co> Date: Mon, 28 Dec 2020 15:16:31 -0700 Subject: [PATCH 04/10] Convert prevLocationMap to Map before passing to function --- .../alert_types/geo_containment/geo_containment.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/geo_containment.ts b/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/geo_containment.ts index b58de2823248d..5314db3ba881e 100644 --- a/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/geo_containment.ts +++ b/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/geo_containment.ts @@ -81,7 +81,7 @@ export function transformResults( } export function getActiveEntriesAndGenerateAlerts( - prevLocationMap: Record<string, LatestEntityLocation[]>, + prevLocationMap: Map<string, LatestEntityLocation[]>, currLocationMap: Map<string, LatestEntityLocation[]>, alertInstanceFactory: AlertServices< GeoContainmentInstanceState, @@ -91,7 +91,7 @@ export function getActiveEntriesAndGenerateAlerts( currIntervalEndTime: Date ) { const allActiveEntriesMap: Map<string, LatestEntityLocation[]> = new Map([ - ...Object.entries(prevLocationMap || {}), + ...prevLocationMap, ...currLocationMap, ]); allActiveEntriesMap.forEach((locationsArr, entityName) => { @@ -168,8 +168,11 @@ export const getGeoContainmentExecutor = (log: Logger): GeoContainmentAlertType[ params.geoField ); + const prevLocationMap: Map<string, LatestEntityLocation[]> = new Map([ + ...Object.entries((state.prevLocationMap as Record<string, LatestEntityLocation[]>) || {}), + ]); const allActiveEntriesMap = getActiveEntriesAndGenerateAlerts( - state.prevLocationMap as Record<string, LatestEntityLocation[]>, + prevLocationMap, currLocationMap, services.alertInstanceFactory, shapesIdsNamesMap, From 9bd0a4cf977b52a7360f24baeb8174071d47b1ce Mon Sep 17 00:00:00 2001 From: Aaron Caldwell <aaron.caldwell@elastic.co> Date: Mon, 25 Jan 2021 10:19:50 -0700 Subject: [PATCH 05/10] Clean up and fix tests --- .../geo_containment/geo_containment.ts | 6 +- .../tests/geo_containment.test.ts | 214 ++++++++++-------- 2 files changed, 127 insertions(+), 93 deletions(-) diff --git a/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/geo_containment.ts b/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/geo_containment.ts index 43b4cf93b7ffe..13fbfa22c6dc8 100644 --- a/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/geo_containment.ts +++ b/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/geo_containment.ts @@ -98,7 +98,6 @@ export function getActiveEntriesAndGenerateAlerts( allActiveEntriesMap.forEach((locationsArr, entityName) => { // Generate alerts locationsArr.forEach(({ location, shapeLocationId, dateInShape, docId }) => { - const containingBoundaryName = shapesIdsNamesMap[shapeLocationId] || shapeLocationId; const context = { entityId: entityName, entityDateTime: dateInShape ? new Date(dateInShape).toISOString() : null, @@ -106,9 +105,9 @@ export function getActiveEntriesAndGenerateAlerts( detectionDateTime: new Date(currIntervalEndTime).toISOString(), entityLocation: `POINT (${location[0]} ${location[1]})`, containingBoundaryId: shapeLocationId, - containingBoundaryName, + containingBoundaryName: shapesIdsNamesMap[shapeLocationId] || shapeLocationId, }; - const alertInstanceId = `${entityName}-${containingBoundaryName}`; + const alertInstanceId = `${entityName}-${context.containingBoundaryName}`; if (shapeLocationId !== OTHER_CATEGORY) { alertInstanceFactory(alertInstanceId).scheduleActions(ActionGroupId, context); } @@ -117,6 +116,7 @@ export function getActiveEntriesAndGenerateAlerts( const latestLocationsArr = locationsArr.filter( (latestEntityLocation) => latestEntityLocation.dateInShape === locationsArr[0].dateInShape ); + // If the latest location is "other", don't carry it through to the next interval if (latestLocationsArr[0].shapeLocationId === OTHER_CATEGORY) { allActiveEntriesMap.delete(entityName); } else { diff --git a/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/tests/geo_containment.test.ts b/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/tests/geo_containment.test.ts index 26b51060c2e73..d9675d1923258 100644 --- a/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/tests/geo_containment.test.ts +++ b/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/tests/geo_containment.test.ts @@ -27,39 +27,47 @@ describe('geo_containment', () => { new Map([ [ '936', - { - dateInShape: '2020-09-28T18:01:41.190Z', - docId: 'N-ng1XQB6yyY-xQxnGSM', - location: [-82.8814151789993, 40.62806099653244], - shapeLocationId: '0DrJu3QB6yyY-xQxv6Ip', - }, + [ + { + dateInShape: '2020-09-28T18:01:41.190Z', + docId: 'N-ng1XQB6yyY-xQxnGSM', + location: [-82.8814151789993, 40.62806099653244], + shapeLocationId: '0DrJu3QB6yyY-xQxv6Ip', + }, + ], ], [ 'AAL2019', - { - dateInShape: '2020-09-28T18:01:41.191Z', - docId: 'iOng1XQB6yyY-xQxnGSM', - location: [-82.22068064846098, 39.006176185794175], - shapeLocationId: '0DrJu3QB6yyY-xQxv6Ip', - }, + [ + { + dateInShape: '2020-09-28T18:01:41.191Z', + docId: 'iOng1XQB6yyY-xQxnGSM', + location: [-82.22068064846098, 39.006176185794175], + shapeLocationId: '0DrJu3QB6yyY-xQxv6Ip', + }, + ], ], [ 'AAL2323', - { - dateInShape: '2020-09-28T18:01:41.191Z', - docId: 'n-ng1XQB6yyY-xQxnGSM', - location: [-84.71324851736426, 41.6677269525826], - shapeLocationId: '0DrJu3QB6yyY-xQxv6Ip', - }, + [ + { + dateInShape: '2020-09-28T18:01:41.191Z', + docId: 'n-ng1XQB6yyY-xQxnGSM', + location: [-84.71324851736426, 41.6677269525826], + shapeLocationId: '0DrJu3QB6yyY-xQxv6Ip', + }, + ], ], [ 'ABD5250', - { - dateInShape: '2020-09-28T18:01:41.192Z', - docId: 'GOng1XQB6yyY-xQxnGWM', - location: [6.073727197945118, 39.07997465226799], - shapeLocationId: '0DrJu3QB6yyY-xQxv6Ip', - }, + [ + { + dateInShape: '2020-09-28T18:01:41.192Z', + docId: 'GOng1XQB6yyY-xQxnGWM', + location: [6.073727197945118, 39.07997465226799], + shapeLocationId: '0DrJu3QB6yyY-xQxv6Ip', + }, + ], ], ]) ); @@ -77,39 +85,47 @@ describe('geo_containment', () => { new Map([ [ '936', - { - dateInShape: '2020-09-28T18:01:41.190Z', - docId: 'N-ng1XQB6yyY-xQxnGSM', - location: [-82.8814151789993, 40.62806099653244], - shapeLocationId: '0DrJu3QB6yyY-xQxv6Ip', - }, + [ + { + dateInShape: '2020-09-28T18:01:41.190Z', + docId: 'N-ng1XQB6yyY-xQxnGSM', + location: [-82.8814151789993, 40.62806099653244], + shapeLocationId: '0DrJu3QB6yyY-xQxv6Ip', + }, + ], ], [ 'AAL2019', - { - dateInShape: '2020-09-28T18:01:41.191Z', - docId: 'iOng1XQB6yyY-xQxnGSM', - location: [-82.22068064846098, 39.006176185794175], - shapeLocationId: '0DrJu3QB6yyY-xQxv6Ip', - }, + [ + { + dateInShape: '2020-09-28T18:01:41.191Z', + docId: 'iOng1XQB6yyY-xQxnGSM', + location: [-82.22068064846098, 39.006176185794175], + shapeLocationId: '0DrJu3QB6yyY-xQxv6Ip', + }, + ], ], [ 'AAL2323', - { - dateInShape: '2020-09-28T18:01:41.191Z', - docId: 'n-ng1XQB6yyY-xQxnGSM', - location: [-84.71324851736426, 41.6677269525826], - shapeLocationId: '0DrJu3QB6yyY-xQxv6Ip', - }, + [ + { + dateInShape: '2020-09-28T18:01:41.191Z', + docId: 'n-ng1XQB6yyY-xQxnGSM', + location: [-84.71324851736426, 41.6677269525826], + shapeLocationId: '0DrJu3QB6yyY-xQxv6Ip', + }, + ], ], [ 'ABD5250', - { - dateInShape: '2020-09-28T18:01:41.192Z', - docId: 'GOng1XQB6yyY-xQxnGWM', - location: [6.073727197945118, 39.07997465226799], - shapeLocationId: '0DrJu3QB6yyY-xQxv6Ip', - }, + [ + { + dateInShape: '2020-09-28T18:01:41.192Z', + docId: 'GOng1XQB6yyY-xQxnGWM', + location: [6.073727197945118, 39.07997465226799], + shapeLocationId: '0DrJu3QB6yyY-xQxv6Ip', + }, + ], ], ]) ); @@ -131,30 +147,36 @@ describe('geo_containment', () => { const currLocationMap = new Map([ [ 'a', - { - location: [0, 0], - shapeLocationId: '123', - dateInShape: 'Wed Dec 09 2020 14:31:31 GMT-0700 (Mountain Standard Time)', - docId: 'docId1', - }, + [ + { + location: [0, 0], + shapeLocationId: '123', + dateInShape: 'Wed Dec 09 2020 14:31:31 GMT-0700 (Mountain Standard Time)', + docId: 'docId1', + }, + ], ], [ 'b', - { - location: [0, 0], - shapeLocationId: '456', - dateInShape: 'Wed Dec 09 2020 15:31:31 GMT-0700 (Mountain Standard Time)', - docId: 'docId2', - }, + [ + { + location: [0, 0], + shapeLocationId: '456', + dateInShape: 'Wed Dec 09 2020 15:31:31 GMT-0700 (Mountain Standard Time)', + docId: 'docId2', + }, + ], ], [ 'c', - { - location: [0, 0], - shapeLocationId: '789', - dateInShape: 'Wed Dec 09 2020 16:31:31 GMT-0700 (Mountain Standard Time)', - docId: 'docId3', - }, + [ + { + location: [0, 0], + shapeLocationId: '789', + dateInShape: 'Wed Dec 09 2020 16:31:31 GMT-0700 (Mountain Standard Time)', + docId: 'docId3', + }, + ], ], ]); @@ -215,7 +237,7 @@ describe('geo_containment', () => { const currentDateTime = new Date(); it('should use currently active entities if no older entity entries', () => { - const emptyPrevLocationMap = {}; + const emptyPrevLocationMap = new Map(); const allActiveEntriesMap = getActiveEntriesAndGenerateAlerts( emptyPrevLocationMap, currLocationMap, @@ -227,14 +249,19 @@ describe('geo_containment', () => { expect(testAlertActionArr).toMatchObject(expectedContext); }); it('should overwrite older identical entity entries', () => { - const prevLocationMapWithIdenticalEntityEntry = { - a: { - location: [0, 0], - shapeLocationId: '999', - dateInShape: 'Wed Dec 09 2020 12:31:31 GMT-0700 (Mountain Standard Time)', - docId: 'docId7', - }, - }; + const prevLocationMapWithIdenticalEntityEntry = new Map([ + [ + 'a', + [ + { + location: [0, 0], + shapeLocationId: '999', + dateInShape: 'Wed Dec 09 2020 12:31:31 GMT-0700 (Mountain Standard Time)', + docId: 'docId7', + }, + ], + ], + ]); const allActiveEntriesMap = getActiveEntriesAndGenerateAlerts( prevLocationMapWithIdenticalEntityEntry, currLocationMap, @@ -246,14 +273,19 @@ describe('geo_containment', () => { expect(testAlertActionArr).toMatchObject(expectedContext); }); it('should preserve older non-identical entity entries', () => { - const prevLocationMapWithNonIdenticalEntityEntry = { - d: { - location: [0, 0], - shapeLocationId: '999', - dateInShape: 'Wed Dec 09 2020 12:31:31 GMT-0700 (Mountain Standard Time)', - docId: 'docId7', - }, - }; + const prevLocationMapWithNonIdenticalEntityEntry = new Map([ + [ + 'd', + [ + { + location: [0, 0], + shapeLocationId: '999', + dateInShape: 'Wed Dec 09 2020 12:31:31 GMT-0700 (Mountain Standard Time)', + docId: 'docId7', + }, + ], + ], + ]); const expectedContextPlusD = [ { actionGroupId: 'Tracked entity contained', @@ -280,13 +312,15 @@ describe('geo_containment', () => { expect(testAlertActionArr).toMatchObject(expectedContextPlusD); }); it('should remove "other" entries and schedule the expected number of actions', () => { - const emptyPrevLocationMap = {}; - const currLocationMapWithOther = new Map(currLocationMap).set('d', { - location: [0, 0], - shapeLocationId: OTHER_CATEGORY, - dateInShape: 'Wed Dec 09 2020 14:31:31 GMT-0700 (Mountain Standard Time)', - docId: 'docId1', - }); + const emptyPrevLocationMap = new Map(); + const currLocationMapWithOther = new Map([...currLocationMap]).set('d', [ + { + location: [0, 0], + shapeLocationId: OTHER_CATEGORY, + dateInShape: 'Wed Dec 09 2020 14:31:31 GMT-0700 (Mountain Standard Time)', + docId: 'docId1', + }, + ]); expect(currLocationMapWithOther).not.toEqual(currLocationMap); const allActiveEntriesMap = getActiveEntriesAndGenerateAlerts( emptyPrevLocationMap, From 8439114989417ecf34e07d9101e4d3a9d81ace7f Mon Sep 17 00:00:00 2001 From: Aaron Caldwell <aaron.caldwell@elastic.co> Date: Mon, 25 Jan 2021 16:32:04 -0700 Subject: [PATCH 06/10] Remove filter on dates --- .../server/alert_types/geo_containment/geo_containment.ts | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/geo_containment.ts b/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/geo_containment.ts index 13fbfa22c6dc8..c3afb92509457 100644 --- a/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/geo_containment.ts +++ b/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/geo_containment.ts @@ -112,15 +112,11 @@ export function getActiveEntriesAndGenerateAlerts( alertInstanceFactory(alertInstanceId).scheduleActions(ActionGroupId, context); } }); - // Determine what remains active - const latestLocationsArr = locationsArr.filter( - (latestEntityLocation) => latestEntityLocation.dateInShape === locationsArr[0].dateInShape - ); // If the latest location is "other", don't carry it through to the next interval - if (latestLocationsArr[0].shapeLocationId === OTHER_CATEGORY) { + if (locationsArr[0].shapeLocationId === OTHER_CATEGORY) { allActiveEntriesMap.delete(entityName); } else { - allActiveEntriesMap.set(entityName, latestLocationsArr); + allActiveEntriesMap.set(entityName, locationsArr); } }); return allActiveEntriesMap; From cb24ca6c6e8de6da003d5cdcc629e76acc87e312 Mon Sep 17 00:00:00 2001 From: Aaron Caldwell <aaron.caldwell@elastic.co> Date: Mon, 25 Jan 2021 16:32:40 -0700 Subject: [PATCH 07/10] Add tests for cases involving multiple final locations --- .../tests/geo_containment.test.ts | 102 ++++++++++++++++++ 1 file changed, 102 insertions(+) diff --git a/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/tests/geo_containment.test.ts b/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/tests/geo_containment.test.ts index d9675d1923258..4ba9412fd3c2d 100644 --- a/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/tests/geo_containment.test.ts +++ b/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/tests/geo_containment.test.ts @@ -311,6 +311,7 @@ describe('geo_containment', () => { expect(allActiveEntriesMap.has('d')).toBeTruthy(); expect(testAlertActionArr).toMatchObject(expectedContextPlusD); }); + it('should remove "other" entries and schedule the expected number of actions', () => { const emptyPrevLocationMap = new Map(); const currLocationMapWithOther = new Map([...currLocationMap]).set('d', [ @@ -332,5 +333,106 @@ describe('geo_containment', () => { expect(allActiveEntriesMap).toEqual(currLocationMap); expect(testAlertActionArr).toMatchObject(expectedContext); }); + + it('should generate multiple alerts per entity if found in multiple shapes in interval', () => { + const emptyPrevLocationMap = new Map(); + const currLocationMapWithThreeMore = new Map([...currLocationMap]).set('d', [ + { + location: [0, 0], + shapeLocationId: '789', + dateInShape: 'Wed Dec 10 2020 14:31:31 GMT-0700 (Mountain Standard Time)', + docId: 'docId1', + }, + { + location: [0, 0], + shapeLocationId: '123', + dateInShape: 'Wed Dec 08 2020 14:31:31 GMT-0700 (Mountain Standard Time)', + docId: 'docId2', + }, + { + location: [0, 0], + shapeLocationId: '456', + dateInShape: 'Wed Dec 07 2020 14:31:31 GMT-0700 (Mountain Standard Time)', + docId: 'docId3', + }, + ]); + getActiveEntriesAndGenerateAlerts( + emptyPrevLocationMap, + currLocationMapWithThreeMore, + alertInstanceFactory, + emptyShapesIdsNamesMap, + currentDateTime + ); + let numEntitiesInShapes = 0; + currLocationMapWithThreeMore.forEach((v) => { + numEntitiesInShapes += v.length; + }); + expect(testAlertActionArr.length).toEqual(numEntitiesInShapes); + }); + + it('should not return entity as active entry if most recent location is "other"', () => { + const emptyPrevLocationMap = new Map(); + const currLocationMapWithOther = new Map([...currLocationMap]).set('d', [ + { + location: [0, 0], + shapeLocationId: OTHER_CATEGORY, + dateInShape: 'Wed Dec 10 2020 14:31:31 GMT-0700 (Mountain Standard Time)', + docId: 'docId1', + }, + { + location: [0, 0], + shapeLocationId: '123', + dateInShape: 'Wed Dec 08 2020 14:31:31 GMT-0700 (Mountain Standard Time)', + docId: 'docId1', + }, + { + location: [0, 0], + shapeLocationId: '456', + dateInShape: 'Wed Dec 07 2020 14:31:31 GMT-0700 (Mountain Standard Time)', + docId: 'docId1', + }, + ]); + expect(currLocationMapWithOther).not.toEqual(currLocationMap); + const allActiveEntriesMap = getActiveEntriesAndGenerateAlerts( + emptyPrevLocationMap, + currLocationMapWithOther, + alertInstanceFactory, + emptyShapesIdsNamesMap, + currentDateTime + ); + expect(allActiveEntriesMap).toEqual(currLocationMap); + }); + + it('should return entity as active entry if "other" not the latest location', () => { + const emptyPrevLocationMap = new Map(); + const currLocationMapWithOther = new Map([...currLocationMap]).set('d', [ + { + location: [0, 0], + shapeLocationId: '123', + dateInShape: 'Wed Dec 10 2020 14:31:31 GMT-0700 (Mountain Standard Time)', + docId: 'docId1', + }, + { + location: [0, 0], + shapeLocationId: OTHER_CATEGORY, + dateInShape: 'Wed Dec 08 2020 14:31:31 GMT-0700 (Mountain Standard Time)', + docId: 'docId1', + }, + { + location: [0, 0], + shapeLocationId: '456', + dateInShape: 'Wed Dec 07 2020 14:31:31 GMT-0700 (Mountain Standard Time)', + docId: 'docId1', + }, + ]); + const allActiveEntriesMap = getActiveEntriesAndGenerateAlerts( + emptyPrevLocationMap, + currLocationMapWithOther, + alertInstanceFactory, + emptyShapesIdsNamesMap, + currentDateTime + ); + expect(allActiveEntriesMap).toEqual(currLocationMapWithOther); + }); }); }); From 1708cefaa7b3d5952311b3b86a0bf416614ccecc Mon Sep 17 00:00:00 2001 From: Aaron Caldwell <aaron.caldwell@elastic.co> Date: Mon, 25 Jan 2021 16:56:43 -0700 Subject: [PATCH 08/10] Cover case where 'other' and earlier entries are removed --- .../alert_types/geo_containment/geo_containment.ts | 10 +++++++++- .../geo_containment/tests/geo_containment.test.ts | 13 +++++++++++-- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/geo_containment.ts b/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/geo_containment.ts index c3afb92509457..22c5bd4ca563d 100644 --- a/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/geo_containment.ts +++ b/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/geo_containment.ts @@ -112,9 +112,17 @@ export function getActiveEntriesAndGenerateAlerts( alertInstanceFactory(alertInstanceId).scheduleActions(ActionGroupId, context); } }); - // If the latest location is "other", don't carry it through to the next interval + let otherIndex; if (locationsArr[0].shapeLocationId === OTHER_CATEGORY) { allActiveEntriesMap.delete(entityName); + } else if ( + locationsArr.some(({ shapeLocationId }, index) => { + otherIndex = index; + return shapeLocationId === OTHER_CATEGORY; + }) + ) { + const afterOtherLocationsArr = locationsArr.slice(0, otherIndex); + allActiveEntriesMap.set(entityName, afterOtherLocationsArr); } else { allActiveEntriesMap.set(entityName, locationsArr); } diff --git a/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/tests/geo_containment.test.ts b/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/tests/geo_containment.test.ts index 4ba9412fd3c2d..c67acaf2cef1f 100644 --- a/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/tests/geo_containment.test.ts +++ b/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/tests/geo_containment.test.ts @@ -403,7 +403,7 @@ describe('geo_containment', () => { expect(allActiveEntriesMap).toEqual(currLocationMap); }); - it('should return entity as active entry if "other" not the latest location', () => { + it('should return entity as active entry if "other" not the latest location but remove "other" and earlier entries', () => { const emptyPrevLocationMap = new Map(); const currLocationMapWithOther = new Map([...currLocationMap]).set('d', [ { @@ -432,7 +432,16 @@ describe('geo_containment', () => { emptyShapesIdsNamesMap, currentDateTime ); - expect(allActiveEntriesMap).toEqual(currLocationMapWithOther); + expect(allActiveEntriesMap).toEqual( + new Map([...currLocationMap]).set('d', [ + { + location: [0, 0], + shapeLocationId: '123', + dateInShape: 'Wed Dec 10 2020 14:31:31 GMT-0700 (Mountain Standard Time)', + docId: 'docId1', + }, + ]) + ); }); }); }); From 5c0f96d91ce6142d19c07c0ae0d3aed28c5fa05e Mon Sep 17 00:00:00 2001 From: Aaron Caldwell <aaron.caldwell@elastic.co> Date: Wed, 27 Jan 2021 15:34:04 -0700 Subject: [PATCH 09/10] Type fix --- .../alert_types/geo_containment/geo_containment.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/geo_containment.ts b/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/geo_containment.ts index 22c5bd4ca563d..0c5cb62092529 100644 --- a/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/geo_containment.ts +++ b/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/geo_containment.ts @@ -68,11 +68,12 @@ export function transformResults( el: LatestEntityLocation & { entityName: string } ) => { const { entityName, ...locationData } = el; - if (!accu.has(entityName)) { - accu.set(entityName, []); + if (entityName) { + if (!accu.has(entityName)) { + accu.set(entityName, []); + } + accu.get(entityName)!.push(locationData); } - // @ts-ignore - accu.get(entityName).push(locationData); return accu; }, new Map() From 95a885304dad67e17f9913a942d630dd56883492 Mon Sep 17 00:00:00 2001 From: Aaron Caldwell <aaron.caldwell@elastic.co> Date: Wed, 27 Jan 2021 16:40:55 -0700 Subject: [PATCH 10/10] Review feedback. Update if/else for readability. Make timestamp differences more clear --- .../geo_containment/geo_containment.ts | 17 +++++++++-------- .../tests/geo_containment.test.ts | 16 ++++++++-------- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/geo_containment.ts b/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/geo_containment.ts index 0c5cb62092529..1648ad9ad2a62 100644 --- a/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/geo_containment.ts +++ b/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/geo_containment.ts @@ -113,16 +113,17 @@ export function getActiveEntriesAndGenerateAlerts( alertInstanceFactory(alertInstanceId).scheduleActions(ActionGroupId, context); } }); - let otherIndex; + if (locationsArr[0].shapeLocationId === OTHER_CATEGORY) { allActiveEntriesMap.delete(entityName); - } else if ( - locationsArr.some(({ shapeLocationId }, index) => { - otherIndex = index; - return shapeLocationId === OTHER_CATEGORY; - }) - ) { - const afterOtherLocationsArr = locationsArr.slice(0, otherIndex); + return; + } + + const otherCatIndex = locationsArr.findIndex( + ({ shapeLocationId }) => shapeLocationId === OTHER_CATEGORY + ); + if (otherCatIndex >= 0) { + const afterOtherLocationsArr = locationsArr.slice(0, otherCatIndex); allActiveEntriesMap.set(entityName, afterOtherLocationsArr); } else { allActiveEntriesMap.set(entityName, locationsArr); diff --git a/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/tests/geo_containment.test.ts b/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/tests/geo_containment.test.ts index c67acaf2cef1f..40ad6454c3673 100644 --- a/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/tests/geo_containment.test.ts +++ b/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/tests/geo_containment.test.ts @@ -162,7 +162,7 @@ describe('geo_containment', () => { { location: [0, 0], shapeLocationId: '456', - dateInShape: 'Wed Dec 09 2020 15:31:31 GMT-0700 (Mountain Standard Time)', + dateInShape: 'Wed Dec 16 2020 15:31:31 GMT-0700 (Mountain Standard Time)', docId: 'docId2', }, ], @@ -173,7 +173,7 @@ describe('geo_containment', () => { { location: [0, 0], shapeLocationId: '789', - dateInShape: 'Wed Dec 09 2020 16:31:31 GMT-0700 (Mountain Standard Time)', + dateInShape: 'Wed Dec 23 2020 16:31:31 GMT-0700 (Mountain Standard Time)', docId: 'docId3', }, ], @@ -346,13 +346,13 @@ describe('geo_containment', () => { { location: [0, 0], shapeLocationId: '123', - dateInShape: 'Wed Dec 08 2020 14:31:31 GMT-0700 (Mountain Standard Time)', + dateInShape: 'Wed Dec 08 2020 12:31:31 GMT-0700 (Mountain Standard Time)', docId: 'docId2', }, { location: [0, 0], shapeLocationId: '456', - dateInShape: 'Wed Dec 07 2020 14:31:31 GMT-0700 (Mountain Standard Time)', + dateInShape: 'Wed Dec 07 2020 10:31:31 GMT-0700 (Mountain Standard Time)', docId: 'docId3', }, ]); @@ -382,13 +382,13 @@ describe('geo_containment', () => { { location: [0, 0], shapeLocationId: '123', - dateInShape: 'Wed Dec 08 2020 14:31:31 GMT-0700 (Mountain Standard Time)', + dateInShape: 'Wed Dec 08 2020 12:31:31 GMT-0700 (Mountain Standard Time)', docId: 'docId1', }, { location: [0, 0], shapeLocationId: '456', - dateInShape: 'Wed Dec 07 2020 14:31:31 GMT-0700 (Mountain Standard Time)', + dateInShape: 'Wed Dec 07 2020 10:31:31 GMT-0700 (Mountain Standard Time)', docId: 'docId1', }, ]); @@ -415,13 +415,13 @@ describe('geo_containment', () => { { location: [0, 0], shapeLocationId: OTHER_CATEGORY, - dateInShape: 'Wed Dec 08 2020 14:31:31 GMT-0700 (Mountain Standard Time)', + dateInShape: 'Wed Dec 08 2020 12:31:31 GMT-0700 (Mountain Standard Time)', docId: 'docId1', }, { location: [0, 0], shapeLocationId: '456', - dateInShape: 'Wed Dec 07 2020 14:31:31 GMT-0700 (Mountain Standard Time)', + dateInShape: 'Wed Dec 07 2020 10:31:31 GMT-0700 (Mountain Standard Time)', docId: 'docId1', }, ]);