From 1f0e4489ee8f83e30e6c3a17cf86262ffc5d465b Mon Sep 17 00:00:00 2001 From: Jonathan Buttner Date: Fri, 26 Jun 2020 17:38:40 -0400 Subject: [PATCH 01/16] Refactor generator for ancestry support --- .../common/endpoint/generate_data.test.ts | 43 +++- .../common/endpoint/generate_data.ts | 210 +++++++++++------- 2 files changed, 165 insertions(+), 88 deletions(-) diff --git a/x-pack/plugins/security_solution/common/endpoint/generate_data.test.ts b/x-pack/plugins/security_solution/common/endpoint/generate_data.test.ts index 4516007580edc..1bfd1bd56d508 100644 --- a/x-pack/plugins/security_solution/common/endpoint/generate_data.test.ts +++ b/x-pack/plugins/security_solution/common/endpoint/generate_data.test.ts @@ -113,6 +113,8 @@ describe('data generator', () => { percentWithRelated: 100, relatedEvents: 0, relatedAlerts: 0, + useAncestryArray: true, + ancestryArraySize: ANCESTRY_LIMIT, }); tree.ancestry.delete(tree.origin.id); }); @@ -150,6 +152,8 @@ describe('data generator', () => { { category: RelatedEventCategory.Network, count: 1 }, ], relatedAlerts, + useAncestryArray: true, + ancestryArraySize: ANCESTRY_LIMIT, }); }); @@ -162,29 +166,46 @@ describe('data generator', () => { }; const verifyAncestry = (event: Event, genTree: Tree) => { - if (event.process.Ext.ancestry.length > 0) { - expect(event.process.parent?.entity_id).toBe(event.process.Ext.ancestry[0]); + if (event.process.Ext.ancestry!.length > 0) { + expect(event.process.parent?.entity_id).toBe(event.process.Ext.ancestry![0]); } - for (let i = 0; i < event.process.Ext.ancestry.length; i++) { - const ancestor = event.process.Ext.ancestry[i]; + for (let i = 0; i < event.process.Ext.ancestry!.length; i++) { + const ancestor = event.process.Ext.ancestry![i]; const parent = genTree.children.get(ancestor) || genTree.ancestry.get(ancestor); expect(ancestor).toBe(parent?.lifecycle[0].process.entity_id); // the next ancestor should be the grandparent - if (i + 1 < event.process.Ext.ancestry.length) { - const grandparent = event.process.Ext.ancestry[i + 1]; + if (i + 1 < event.process.Ext.ancestry!.length) { + const grandparent = event.process.Ext.ancestry![i + 1]; expect(grandparent).toBe(parent?.lifecycle[0].process.parent?.entity_id); } } }; it('has ancestry array defined', () => { - expect(tree.origin.lifecycle[0].process.Ext.ancestry.length).toBe(ANCESTRY_LIMIT); + expect(tree.origin.lifecycle[0].process.Ext.ancestry!.length).toBe(ANCESTRY_LIMIT); for (const event of tree.allEvents) { verifyAncestry(event, tree); } }); + it('creates the right number childrenLevels', () => { + let totalChildren = 0; + for (const level of tree.childrenLevels) { + totalChildren += level.size; + } + expect(totalChildren).toEqual(tree.children.size); + expect(tree.childrenLevels.length).toEqual(generations); + }); + + it('has the right nodes in both the childrenLevels and children map', () => { + for (const level of tree.childrenLevels) { + for (const node of level.values()) { + expect(tree.children.get(node.id)).toEqual(node); + } + } + }); + it('has the right related events for each node', () => { const checkRelatedEvents = (node: TreeNode) => { expect(node.relatedEvents.length).toEqual(4); @@ -290,7 +311,11 @@ describe('data generator', () => { let events: Event[]; beforeEach(() => { - events = generator.createAlertEventAncestry(3, 0, 0, 0, 0); + events = generator.createAlertEventAncestry({ + ancestors: 3, + percentTerminated: 0, + percentWithRelated: 0, + }); }); it('with n-1 process events', () => { @@ -375,7 +400,7 @@ describe('data generator', () => { const timestamp = new Date().getTime(); const root = generator.generateEvent({ timestamp }); const generations = 2; - const events = [root, ...generator.descendantsTreeGenerator(root, generations)]; + const events = [root, ...generator.descendantsTreeGenerator(root, { generations })]; const rootNode = buildResolverTree(events); const visitedEvents = countResolverEvents(rootNode, generations); expect(visitedEvents).toEqual(events.length); diff --git a/x-pack/plugins/security_solution/common/endpoint/generate_data.ts b/x-pack/plugins/security_solution/common/endpoint/generate_data.ts index 5af34b6a694e8..25b943d2e3af1 100644 --- a/x-pack/plugins/security_solution/common/endpoint/generate_data.ts +++ b/x-pack/plugins/security_solution/common/endpoint/generate_data.ts @@ -17,6 +17,7 @@ import { EndpointStatus, } from './types'; import { factory as policyFactory } from './models/policy_config'; +import { parentEntityId } from './models/event'; export type Event = AlertEvent | EndpointEvent; /** @@ -38,6 +39,8 @@ interface EventOptions { eventCategory?: string | string[]; processName?: string; ancestry?: string[]; + useAncestryArray?: boolean; + ancestryArrayLimit?: number; pid?: number; parentPid?: number; extensions?: object; @@ -258,6 +261,11 @@ export interface Tree { * Map of entity_id to node */ children: Map; + /** + * An array of levels of the children, that doesn't include the origin or any ancestors + * childrenLevels[0] are the direct children of the origin node. The next level would be those children's descendants + */ + childrenLevels: Array>; /** * Map of entity_id to node */ @@ -281,12 +289,35 @@ export interface TreeOptions { percentWithRelated?: number; percentTerminated?: number; alwaysGenMaxChildrenPerNode?: boolean; + useAncestryArray?: boolean; + ancestryArraySize?: number; +} + +type TreeOptionDefaults = Required; + +/** + * This function provides defaults for fields that are not specified in the options + * + * @param options tree options for defining the structure of the tree + */ +export function getTreeOptionsWithDef(options?: TreeOptions): TreeOptionDefaults { + return { + ancestors: options?.ancestors ?? 3, + generations: options?.generations ?? 2, + children: options?.children ?? 2, + relatedEvents: options?.relatedEvents ?? 5, + relatedAlerts: options?.relatedAlerts ?? 3, + percentWithRelated: options?.percentWithRelated ?? 30, + percentTerminated: options?.percentTerminated ?? 100, + alwaysGenMaxChildrenPerNode: options?.alwaysGenMaxChildrenPerNode ?? false, + useAncestryArray: options?.useAncestryArray ?? true, + ancestryArraySize: options?.ancestryArraySize ?? ANCESTRY_LIMIT, + }; } export class EndpointDocGenerator { commonInfo: HostInfo; random: seedrandom.prng; - constructor(seed: string | seedrandom.prng = Math.random().toString()) { if (typeof seed === 'string') { this.random = seedrandom(seed); @@ -492,6 +523,11 @@ export class EndpointDocGenerator { * @param options - Allows event field values to be specified */ public generateEvent(options: EventOptions = {}): EndpointEvent { + let ancestry: string[] | undefined; + if (options?.useAncestryArray === true || options?.ancestry !== undefined) { + ancestry = options.ancestry?.slice(0, options.ancestryArrayLimit ?? ANCESTRY_LIMIT) ?? []; + } + const processName = options.processName ? options.processName : randomProcessName(); const detailRecordForEventType = options.extensions || @@ -552,7 +588,9 @@ export class EndpointDocGenerator { name: processName, // simulate a finite ancestry array size, the endpoint limits the ancestry array to 20 entries we'll use // 2 so that the backend can handle that case - Ext: { ancestry: options.ancestry?.slice(0, ANCESTRY_LIMIT) || [] }, + Ext: { + ancestry, + }, }, user: { domain: this.randomString(10), @@ -570,6 +608,7 @@ export class EndpointDocGenerator { * @returns a Tree structure that makes accessing specific events easier */ public generateTree(options: TreeOptions = {}): Tree { + const optionsWithDef = getTreeOptionsWithDef(options); const addEventToMap = (nodeMap: Map, event: Event) => { const nodeId = event.process.entity_id; // if a node already exists for the entity_id we'll use that one, otherwise let's create a new empty node @@ -593,13 +632,46 @@ export class EndpointDocGenerator { return nodeMap.set(nodeId, node); }; - const ancestry = this.createAlertEventAncestry( - options.ancestors, - options.relatedEvents, - options.relatedAlerts, - options.percentWithRelated, - options.percentTerminated - ); + const groupNodesByParent = (children: Map) => { + const nodesByParent: Map> = new Map(); + for (const node of children.values()) { + const parentID = parentEntityId(node.lifecycle[0]); + if (parentID) { + let groupedNodes = nodesByParent.get(parentID); + + if (!groupedNodes) { + groupedNodes = new Map(); + nodesByParent.set(parentID, groupedNodes); + } + groupedNodes.set(node.id, node); + } + } + + return nodesByParent; + }; + + const createLevels = ( + childrenByParent: Map>, + levels: Array>, + currentNodes: Map | undefined + ): Array> => { + if (!currentNodes || currentNodes.size === 0) { + return levels; + } + levels.push(currentNodes); + const nextLevel: Map = new Map(); + for (const node of currentNodes.values()) { + const children = childrenByParent.get(node.id); + if (children) { + for (const child of children.values()) { + nextLevel.set(child.id, child); + } + } + } + return createLevels(childrenByParent, levels, nextLevel); + }; + + const ancestry = this.createAlertEventAncestry(optionsWithDef); // create a mapping of entity_id -> {lifecycle, related events, and related alerts} const ancestryNodes: Map = ancestry.reduce(addEventToMap, new Map()); @@ -610,26 +682,18 @@ export class EndpointDocGenerator { throw Error(`could not find origin while building tree: ${alert.process.entity_id}`); } - const children = Array.from( - this.descendantsTreeGenerator( - alert, - options.generations, - options.children, - options.relatedEvents, - options.relatedAlerts, - options.percentWithRelated, - options.percentTerminated, - options.alwaysGenMaxChildrenPerNode - ) - ); + const children = Array.from(this.descendantsTreeGenerator(alert, optionsWithDef)); const childrenNodes: Map = children.reduce(addEventToMap, new Map()); + const childrenByParent = groupNodesByParent(childrenNodes); + const levels = createLevels(childrenByParent, [], childrenByParent.get(origin.id)); return { children: childrenNodes, ancestry: ancestryNodes, allEvents: [...ancestry, ...children], origin, + childrenLevels: levels, }; } @@ -647,8 +711,9 @@ export class EndpointDocGenerator { * @param alwaysGenMaxChildrenPerNode - flag to always return the max children per node instead of it being a random number of children */ public *alertsGenerator(numAlerts: number, options: TreeOptions = {}) { + const opts = getTreeOptionsWithDef(options); for (let i = 0; i < numAlerts; i++) { - yield* this.fullResolverTreeGenerator(options); + yield* this.fullResolverTreeGenerator(opts); } } @@ -667,27 +732,14 @@ export class EndpointDocGenerator { * @param alwaysGenMaxChildrenPerNode - flag to always return the max children per node instead of it being a random number of children */ public *fullResolverTreeGenerator(options: TreeOptions = {}) { - const ancestry = this.createAlertEventAncestry( - options.ancestors, - options.relatedEvents, - options.relatedAlerts, - options.percentWithRelated, - options.percentTerminated - ); + const opts = getTreeOptionsWithDef(options); + + const ancestry = this.createAlertEventAncestry(opts); for (let i = 0; i < ancestry.length; i++) { yield ancestry[i]; } // ancestry will always have at least 2 elements, and the last element will be the alert - yield* this.descendantsTreeGenerator( - ancestry[ancestry.length - 1], - options.generations, - options.children, - options.relatedEvents, - options.relatedAlerts, - options.percentWithRelated, - options.percentTerminated, - options.alwaysGenMaxChildrenPerNode - ); + yield* this.descendantsTreeGenerator(ancestry[ancestry.length - 1], opts); } /** @@ -699,16 +751,15 @@ export class EndpointDocGenerator { * @param pctWithRelated - percent of ancestors that will have related events and alerts * @param pctWithTerminated - percent of ancestors that will have termination events */ - public createAlertEventAncestry( - alertAncestors = 3, - relatedEventsPerNode: RelatedEventInfo[] | number = 5, - relatedAlertsPerNode: number = 3, - pctWithRelated = 30, - pctWithTerminated = 100 - ): Event[] { + public createAlertEventAncestry(options: TreeOptions = {}): Event[] { + const opts = getTreeOptionsWithDef(options); + const events = []; const startDate = new Date().getTime(); - const root = this.generateEvent({ timestamp: startDate + 1000 }); + const root = this.generateEvent({ + timestamp: startDate + 1000, + useAncestryArray: opts.useAncestryArray, + }); events.push(root); let ancestor = root; let timestamp = root['@timestamp'] + 1000; @@ -727,7 +778,7 @@ export class EndpointDocGenerator { const addRelatedEvents = (node: Event, secBeforeEvent: number, eventList: Event[]) => { for (const relatedEvent of this.relatedEventsGenerator( node, - relatedEventsPerNode, + opts.relatedEvents, secBeforeEvent )) { eventList.push(relatedEvent); @@ -736,13 +787,13 @@ export class EndpointDocGenerator { // generate related alerts for root const processDuration: number = 6 * 3600; - if (this.randomN(100) < pctWithRelated) { + if (this.randomN(100) < opts.percentWithRelated) { addRelatedEvents(ancestor, processDuration, events); - addRelatedAlerts(ancestor, relatedAlertsPerNode, processDuration, events); + addRelatedAlerts(ancestor, opts.relatedAlerts, processDuration, events); } // generate the termination event for the root - if (this.randomN(100) < pctWithTerminated) { + if (this.randomN(100) < opts.percentTerminated) { const termProcessDuration = this.randomN(1000000); // This lets termination events be up to 1 million seconds after the creation event (~11 days) events.push( this.generateEvent({ @@ -751,23 +802,26 @@ export class EndpointDocGenerator { parentEntityID: root.process.parent?.entity_id, eventCategory: 'process', eventType: 'end', + useAncestryArray: opts.useAncestryArray, }) ); } - for (let i = 0; i < alertAncestors; i++) { + for (let i = 0; i < opts.ancestors; i++) { ancestor = this.generateEvent({ timestamp, parentEntityID: ancestor.process.entity_id, // add the parent to the ancestry array - ancestry: [ancestor.process.entity_id, ...ancestor.process.Ext.ancestry], + ancestry: [ancestor.process.entity_id, ...(ancestor.process.Ext.ancestry ?? [])], + ancestryArrayLimit: opts.ancestryArraySize, + useAncestryArray: opts.useAncestryArray, parentPid: ancestor.process.pid, pid: this.randomN(5000), }); events.push(ancestor); timestamp = timestamp + 1000; - if (this.randomN(100) < pctWithTerminated) { + if (this.randomN(100) < opts.percentTerminated) { const termProcessDuration = this.randomN(1000000); // This lets termination events be up to 1 million seconds after the creation event (~11 days) events.push( this.generateEvent({ @@ -777,18 +831,20 @@ export class EndpointDocGenerator { eventCategory: 'process', eventType: 'end', ancestry: ancestor.process.Ext.ancestry, + ancestryArrayLimit: opts.ancestryArraySize, + useAncestryArray: opts.useAncestryArray, }) ); } // generate related alerts for ancestor - if (this.randomN(100) < pctWithRelated) { + if (this.randomN(100) < opts.percentWithRelated) { addRelatedEvents(ancestor, processDuration, events); - let numAlertsPerNode = relatedAlertsPerNode; + let numAlertsPerNode = opts.relatedAlerts; // if this is the last ancestor, create one less related alert so that we have a uniform amount of related alerts // for each node. The last alert at the end of this function should always be created even if the related alerts // amount is 0 - if (i === alertAncestors - 1) { + if (i === opts.ancestors - 1) { numAlertsPerNode -= 1; } addRelatedAlerts(ancestor, numAlertsPerNode, processDuration, events); @@ -816,19 +872,11 @@ export class EndpointDocGenerator { * @param percentChildrenTerminated - percent of nodes which will have process termination events * @param alwaysGenMaxChildrenPerNode - flag to always return the max children per node instead of it being a random number of children */ - public *descendantsTreeGenerator( - root: Event, - generations = 2, - maxChildrenPerNode = 2, - relatedEventsPerNode: RelatedEventInfo[] | number = 3, - relatedAlertsPerNode: number = 3, - percentNodesWithRelated = 100, - percentChildrenTerminated = 100, - alwaysGenMaxChildrenPerNode = false - ) { - let maxChildren = this.randomN(maxChildrenPerNode + 1); - if (alwaysGenMaxChildrenPerNode) { - maxChildren = maxChildrenPerNode; + public *descendantsTreeGenerator(root: Event, options: TreeOptions = {}) { + const opts = getTreeOptionsWithDef(options); + let maxChildren = this.randomN(opts.children + 1); + if (opts.alwaysGenMaxChildrenPerNode) { + maxChildren = opts.children; } const rootState: NodeState = { @@ -843,7 +891,7 @@ export class EndpointDocGenerator { // If we get to a state node and it has made all the children, move back up a level if ( currentState.childrenCreated === currentState.maxChildren || - lineage.length === generations + 1 + lineage.length === opts.generations + 1 ) { lineage.pop(); // eslint-disable-next-line no-continue @@ -857,13 +905,15 @@ export class EndpointDocGenerator { parentEntityID: currentState.event.process.entity_id, ancestry: [ currentState.event.process.entity_id, - ...currentState.event.process.Ext.ancestry, + ...(currentState.event.process.Ext.ancestry ?? []), ], + ancestryArrayLimit: opts.ancestryArraySize, + useAncestryArray: opts.useAncestryArray, }); - maxChildren = this.randomN(maxChildrenPerNode + 1); - if (alwaysGenMaxChildrenPerNode) { - maxChildren = maxChildrenPerNode; + maxChildren = this.randomN(opts.children + 1); + if (opts.alwaysGenMaxChildrenPerNode) { + maxChildren = opts.children; } lineage.push({ event: child, @@ -872,7 +922,7 @@ export class EndpointDocGenerator { }); yield child; let processDuration: number = 6 * 3600; - if (this.randomN(100) < percentChildrenTerminated) { + if (this.randomN(100) < opts.percentTerminated) { processDuration = this.randomN(1000000); // This lets termination events be up to 1 million seconds after the creation event (~11 days) yield this.generateEvent({ timestamp: timestamp + processDuration * 1000, @@ -881,11 +931,13 @@ export class EndpointDocGenerator { eventCategory: 'process', eventType: 'end', ancestry: child.process.Ext.ancestry, + ancestryArrayLimit: opts.ancestryArraySize, + useAncestryArray: opts.useAncestryArray, }); } - if (this.randomN(100) < percentNodesWithRelated) { - yield* this.relatedEventsGenerator(child, relatedEventsPerNode, processDuration); - yield* this.relatedAlertsGenerator(child, relatedAlertsPerNode, processDuration); + if (this.randomN(100) < opts.percentWithRelated) { + yield* this.relatedEventsGenerator(child, opts.relatedEvents, processDuration); + yield* this.relatedAlertsGenerator(child, opts.relatedAlerts, processDuration); } } } From f0ee76ac4b05a14b6e7eb8242d8614dd1715f0da Mon Sep 17 00:00:00 2001 From: Jonathan Buttner Date: Fri, 26 Jun 2020 17:48:44 -0400 Subject: [PATCH 02/16] Adding optional ancestry array --- x-pack/plugins/security_solution/common/endpoint/types.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/common/endpoint/types.ts b/x-pack/plugins/security_solution/common/endpoint/types.ts index 42f5f4b220da9..2efaeef5719b7 100644 --- a/x-pack/plugins/security_solution/common/endpoint/types.ts +++ b/x-pack/plugins/security_solution/common/endpoint/types.ts @@ -498,7 +498,7 @@ export interface EndpointEvent { * ancestry_array[0] == process.parent.entity_id and ancestry_array[1] == process.parent.parent.entity_id */ Ext: { - ancestry: string[]; + ancestry?: string[]; }; }; user?: { From c95132f907a7aa1397772bda0eb321a721386ffe Mon Sep 17 00:00:00 2001 From: Jonathan Buttner Date: Fri, 26 Jun 2020 17:56:19 -0400 Subject: [PATCH 03/16] Refactor the pagination since the totals are not used anymore --- .../routes/resolver/utils/pagination.test.ts | 36 ++------ .../routes/resolver/utils/pagination.ts | 90 +++++-------------- 2 files changed, 27 insertions(+), 99 deletions(-) diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/pagination.test.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/pagination.test.ts index 74e4e252861e6..4daa45aec2a74 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/pagination.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/pagination.test.ts @@ -18,20 +18,20 @@ describe('Pagination', () => { const root = generator.generateEvent(); const events = Array.from(generator.relatedEventsGenerator(root, 5)); - it('does not build a cursor when all events are present', () => { - expect(PaginationBuilder.buildCursor(0, events)).toBeNull(); + it('does build a cursor when received the same number of events as was requested', () => { + expect(PaginationBuilder.buildCursorRequestLimit(4, events)).not.toBeNull(); }); - it('creates a cursor when not all events are present', () => { - expect(PaginationBuilder.buildCursor(events.length + 1, events)).not.toBeNull(); + it('does not create a cursor when the number of events received is less than the amount requested', () => { + expect(PaginationBuilder.buildCursorRequestLimit(events.length + 1, events)).toBeNull(); }); it('creates a cursor with the right information', () => { - const cursor = PaginationBuilder.buildCursor(events.length + 1, events); + const cursor = PaginationBuilder.buildCursorRequestLimit(events.length, events); expect(cursor).not.toBeNull(); // we are guaranteed that the cursor won't be null from the check above const builder = PaginationBuilder.createBuilder(0, cursor!); - const fields = builder.buildQueryFields(0, '', ''); + const fields = builder.buildQueryFields(''); expect(fields.search_after).toStrictEqual(getSearchAfterInfo(events)); }); }); @@ -39,30 +39,8 @@ describe('Pagination', () => { describe('pagination builder', () => { it('does not include the search after information when no cursor is provided', () => { const builder = PaginationBuilder.createBuilder(100); - const fields = builder.buildQueryFields(1, '', ''); + const fields = builder.buildQueryFields(''); expect(fields).not.toHaveProperty('search_after'); }); - - it('returns no results when the aggregation does not exist in the response', () => { - expect(PaginationBuilder.getTotals()).toStrictEqual({}); - }); - - it('constructs the totals from the aggregation results', () => { - const agg = { - totals: { - buckets: [ - { - key: 'awesome', - doc_count: 5, - }, - { - key: 'soup', - doc_count: 1, - }, - ], - }, - }; - expect(PaginationBuilder.getTotals(agg)).toStrictEqual({ awesome: 5, soup: 1 }); - }); }); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/pagination.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/pagination.ts index 61cb5bdb8f146..2b107ab1b6db4 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/pagination.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/pagination.ts @@ -8,41 +8,11 @@ import { ResolverEvent } from '../../../../../common/endpoint/types'; import { eventId } from '../../../../../common/endpoint/models/event'; import { JsonObject } from '../../../../../../../../src/plugins/kibana_utils/common'; -/** - * Represents a single result bucket of an aggregation - */ -export interface AggBucket { - key: string; - doc_count: number; -} - -interface TotalsAggregation { - totals?: { - buckets?: AggBucket[]; - }; -} - interface PaginationCursor { timestamp: number; eventID: string; } -/** - * The result structure of a query that leverages pagination. This includes totals that can be used to determine if - * additional nodes exist and additional queries need to be made to retrieve the nodes. - */ -export interface PaginatedResults { - /** - * Resulting events returned from the query. - */ - results: ResolverEvent[]; - /** - * Mapping of unique ID to total number of events that exist in ES. The events this references is scoped to the events - * that the query is searching for. - */ - totals: Record; -} - /** * This class handles constructing pagination cursors that resolver can use to return additional events in subsequent * queries. It also constructs an aggregation query to determine the totals for other queries. This class should be used @@ -83,19 +53,28 @@ export class PaginationBuilder { } /** - * Constructs a cursor to use in subsequent queries to retrieve the next set of results. + * Construct a cursor to use in subsequent queries. * - * @param total the total events that exist in ES scoped for a particular query. * @param results the events that were returned by the ES query */ - static buildCursor(total: number, results: ResolverEvent[]): string | null { - if (total > results.length && results.length > 0) { - const lastResult = results[results.length - 1]; - const cursor = { - timestamp: lastResult['@timestamp'], - eventID: eventId(lastResult), - }; - return PaginationBuilder.urlEncodeCursor(cursor); + static buildCursor(results: ResolverEvent[]): string | null { + const lastResult = results[results.length - 1]; + const cursor = { + timestamp: lastResult['@timestamp'], + eventID: eventId(lastResult), + }; + return PaginationBuilder.urlEncodeCursor(cursor); + } + + /** + * Constructs a cursor if the requested limit has not been met. + * + * @param requestLimit the request limit for a query. + * @param results the events that were returned by the ES query + */ + static buildCursorRequestLimit(requestLimit: number, results: ResolverEvent[]): string | null { + if (requestLimit <= results.length && results.length > 0) { + return PaginationBuilder.buildCursor(results); } return null; } @@ -124,45 +103,16 @@ export class PaginationBuilder { /** * Creates an object for adding the pagination fields to a query * - * @param numTerms number of unique IDs that are being search for in this query * @param tiebreaker a unique field to use as the tiebreaker for the search_after - * @param aggregator the field that specifies a unique ID per event (e.g. entity_id) - * @param aggs other aggregations being used with this query * @returns an object containing the pagination information */ - buildQueryFields( - numTerms: number, - tiebreaker: string, - aggregator: string, - aggs: JsonObject = {} - ): JsonObject { + buildQueryFields(tiebreaker: string): JsonObject { const fields: JsonObject = {}; fields.sort = [{ '@timestamp': 'asc' }, { [tiebreaker]: 'asc' }]; - fields.aggs = { ...aggs, totals: { terms: { field: aggregator, size: numTerms } } }; fields.size = this.size; if (this.timestamp && this.eventID) { fields.search_after = [this.timestamp, this.eventID] as Array; } return fields; } - - /** - * Returns the totals found for the specified query - * - * @param aggregations the aggregation field from the ES response - * @returns a mapping of unique ID (e.g. entity_ids) to totals found for those IDs - */ - static getTotals(aggregations?: TotalsAggregation): Record { - if (!aggregations?.totals?.buckets) { - return {}; - } - - return aggregations?.totals?.buckets?.reduce( - (cumulative: Record, bucket: AggBucket) => ({ - ...cumulative, - [bucket.key]: bucket.doc_count, - }), - {} - ); - } } From e4b8a13bcf3c8ec915a45bcde5bb9d876c8d27c3 Mon Sep 17 00:00:00 2001 From: Jonathan Buttner Date: Fri, 26 Jun 2020 17:57:33 -0400 Subject: [PATCH 04/16] Updating the queries to not use aggregations for determining the totals --- .../routes/resolver/queries/alerts.ts | 19 ++++-------- .../endpoint/routes/resolver/queries/base.ts | 23 +++++++++----- .../routes/resolver/queries/children.ts | 30 ++++++++++--------- .../routes/resolver/queries/events.ts | 19 ++++-------- .../routes/resolver/queries/lifecycle.ts | 2 +- .../routes/resolver/queries/multi_searcher.ts | 18 +++++++---- .../endpoint/routes/resolver/queries/stats.ts | 5 ++-- 7 files changed, 59 insertions(+), 57 deletions(-) diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/alerts.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/alerts.ts index 95bc612c58a1b..feb4a404b2359 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/alerts.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/alerts.ts @@ -6,13 +6,13 @@ import { SearchResponse } from 'elasticsearch'; import { ResolverEvent } from '../../../../../common/endpoint/types'; import { ResolverQuery } from './base'; -import { PaginationBuilder, PaginatedResults } from '../utils/pagination'; +import { PaginationBuilder } from '../utils/pagination'; import { JsonObject } from '../../../../../../../../src/plugins/kibana_utils/common'; /** * Builds a query for retrieving alerts for a node. */ -export class AlertsQuery extends ResolverQuery { +export class AlertsQuery extends ResolverQuery { constructor( private readonly pagination: PaginationBuilder, indexPattern: string | string[], @@ -38,11 +38,7 @@ export class AlertsQuery extends ResolverQuery { ], }, }, - ...this.pagination.buildQueryFields( - uniquePIDs.length, - 'endgame.serial_event_id', - 'endgame.unique_pid' - ), + ...this.pagination.buildQueryFields('endgame.serial_event_id'), }; } @@ -60,14 +56,11 @@ export class AlertsQuery extends ResolverQuery { ], }, }, - ...this.pagination.buildQueryFields(entityIDs.length, 'event.id', 'process.entity_id'), + ...this.pagination.buildQueryFields('event.id'), }; } - formatResponse(response: SearchResponse): PaginatedResults { - return { - results: ResolverQuery.getResults(response), - totals: PaginationBuilder.getTotals(response.aggregations), - }; + formatResponse(response: SearchResponse): ResolverEvent[] { + return this.getResults(response); } } diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/base.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/base.ts index 025d3a9420634..0e91dac9e14e1 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/base.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/base.ts @@ -17,7 +17,7 @@ import { MSearchQuery } from './multi_searcher'; * @param T the structured return type of a resolver query. This represents the type that is returned when translating * Elasticsearch's SearchResponse response. */ -export abstract class ResolverQuery implements MSearchQuery { +export abstract class ResolverQuery implements MSearchQuery { /** * * @param indexPattern the index pattern to use in the query for finding indices with documents in ES. @@ -50,7 +50,7 @@ export abstract class ResolverQuery implements MSearchQuery { }; } - protected static getResults(response: SearchResponse): ResolverEvent[] { + protected getResults(response: SearchResponse): R[] { return response.hits.hits.map((hit) => hit._source); } @@ -68,19 +68,26 @@ export abstract class ResolverQuery implements MSearchQuery { } /** - * Searches ES for the specified ids. + * Searches ES for the specified ids and format the response. * * @param client a client for searching ES * @param ids a single more multiple unique node ids (e.g. entity_id or unique_pid) */ - async search(client: IScopedClusterClient, ids: string | string[]): Promise { - const res: SearchResponse = await client.callAsCurrentUser( - 'search', - this.buildSearch(ids) - ); + async searchAndFormat(client: IScopedClusterClient, ids: string | string[]): Promise { + const res: SearchResponse = await this.search(client, ids); return this.formatResponse(res); } + /** + * Searches ES for the specified ids but do not format the response. + * + * @param client a client for searching ES + * @param ids a single more multiple unique node ids (e.g. entity_id or unique_pid) + */ + async search(client: IScopedClusterClient, ids: string | string[]) { + return client.callAsCurrentUser('search', this.buildSearch(ids)); + } + /** * Builds a query to search the legacy data format. * diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/children.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/children.ts index b7b1a16926a15..7fd3808662baa 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/children.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/children.ts @@ -6,13 +6,13 @@ import { SearchResponse } from 'elasticsearch'; import { ResolverEvent } from '../../../../../common/endpoint/types'; import { ResolverQuery } from './base'; -import { PaginationBuilder, PaginatedResults } from '../utils/pagination'; +import { PaginationBuilder } from '../utils/pagination'; import { JsonObject } from '../../../../../../../../src/plugins/kibana_utils/common'; /** * Builds a query for retrieving descendants of a node. */ -export class ChildrenQuery extends ResolverQuery { +export class ChildrenQuery extends ResolverQuery { constructor( private readonly pagination: PaginationBuilder, indexPattern: string | string[], @@ -53,11 +53,7 @@ export class ChildrenQuery extends ResolverQuery { ], }, }, - ...this.pagination.buildQueryFields( - uniquePIDs.length, - 'endgame.serial_event_id', - 'endgame.unique_ppid' - ), + ...this.pagination.buildQueryFields('endgame.serial_event_id'), }; } @@ -67,7 +63,16 @@ export class ChildrenQuery extends ResolverQuery { bool: { filter: [ { - terms: { 'process.parent.entity_id': entityIDs }, + bool: { + should: [ + { + terms: { 'process.parent.entity_id': entityIDs }, + }, + { + terms: { 'process.Ext.ancestry': entityIDs }, + }, + ], + }, }, { term: { 'event.category': 'process' }, @@ -81,14 +86,11 @@ export class ChildrenQuery extends ResolverQuery { ], }, }, - ...this.pagination.buildQueryFields(entityIDs.length, 'event.id', 'process.parent.entity_id'), + ...this.pagination.buildQueryFields('event.id'), }; } - formatResponse(response: SearchResponse): PaginatedResults { - return { - results: ResolverQuery.getResults(response), - totals: PaginationBuilder.getTotals(response.aggregations), - }; + formatResponse(response: SearchResponse): ResolverEvent[] { + return this.getResults(response); } } diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/events.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/events.ts index ec65e30d1d5d4..abc86826e77dd 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/events.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/events.ts @@ -6,13 +6,13 @@ import { SearchResponse } from 'elasticsearch'; import { ResolverEvent } from '../../../../../common/endpoint/types'; import { ResolverQuery } from './base'; -import { PaginationBuilder, PaginatedResults } from '../utils/pagination'; +import { PaginationBuilder } from '../utils/pagination'; import { JsonObject } from '../../../../../../../../src/plugins/kibana_utils/common'; /** * Builds a query for retrieving related events for a node. */ -export class EventsQuery extends ResolverQuery { +export class EventsQuery extends ResolverQuery { constructor( private readonly pagination: PaginationBuilder, indexPattern: string | string[], @@ -45,11 +45,7 @@ export class EventsQuery extends ResolverQuery { ], }, }, - ...this.pagination.buildQueryFields( - uniquePIDs.length, - 'endgame.serial_event_id', - 'endgame.unique_pid' - ), + ...this.pagination.buildQueryFields('endgame.serial_event_id'), }; } @@ -74,14 +70,11 @@ export class EventsQuery extends ResolverQuery { ], }, }, - ...this.pagination.buildQueryFields(entityIDs.length, 'event.id', 'process.entity_id'), + ...this.pagination.buildQueryFields('event.id'), }; } - formatResponse(response: SearchResponse): PaginatedResults { - return { - results: ResolverQuery.getResults(response), - totals: PaginationBuilder.getTotals(response.aggregations), - }; + formatResponse(response: SearchResponse): ResolverEvent[] { + return this.getResults(response); } } diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/lifecycle.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/lifecycle.ts index 93910293b00af..0b5728958e91f 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/lifecycle.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/lifecycle.ts @@ -60,6 +60,6 @@ export class LifecycleQuery extends ResolverQuery { } formatResponse(response: SearchResponse): ResolverEvent[] { - return ResolverQuery.getResults(response); + return this.getResults(response); } } diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/multi_searcher.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/multi_searcher.ts index 4c0e1a5126e7c..d9408bb9db819 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/multi_searcher.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/multi_searcher.ts @@ -5,7 +5,7 @@ */ import { IScopedClusterClient } from 'kibana/server'; -import { MSearchResponse } from 'elasticsearch'; +import { MSearchResponse, SearchResponse } from 'elasticsearch'; import { ResolverEvent } from '../../../../../common/endpoint/types'; import { JsonObject } from '../../../../../../../../src/plugins/kibana_utils/common'; @@ -34,6 +34,10 @@ export interface QueryInfo { * one or many unique identifiers to be searched for in this query */ ids: string | string[]; + /** + * a function to handle the response + */ + handler: (response: SearchResponse) => void; } /** @@ -57,10 +61,10 @@ export class MultiSearcher { throw new Error('No queries provided to MultiSearcher'); } - let searchQuery: JsonObject[] = []; - queries.forEach( - (info) => (searchQuery = [...searchQuery, ...info.query.buildMSearch(info.ids)]) - ); + const searchQuery: JsonObject[] = []; + for (const info of queries) { + searchQuery.push(...info.query.buildMSearch(info.ids)); + } const res: MSearchResponse = await this.client.callAsCurrentUser('msearch', { body: searchQuery, }); @@ -72,6 +76,8 @@ export class MultiSearcher { if (res.responses.length !== queries.length) { throw new Error(`Responses length was: ${res.responses.length} expected ${queries.length}`); } - return res.responses; + for (let i = 0; i < queries.length; i++) { + queries[i].handler(res.responses[i]); + } } } diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/stats.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/stats.ts index a728054bef219..d19099ffa738c 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/stats.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/stats.ts @@ -7,14 +7,15 @@ import { SearchResponse } from 'elasticsearch'; import { ResolverQuery } from './base'; import { ResolverEvent, EventStats } from '../../../../../common/endpoint/types'; import { JsonObject } from '../../../../../../../../src/plugins/kibana_utils/common'; -import { AggBucket } from '../utils/pagination'; export interface StatsResult { alerts: Record; events: Record; } -interface CategoriesAgg extends AggBucket { +interface CategoriesAgg { + key: string; + doc_count: number; /** * The reason categories is optional here is because if no data was returned in the query the categories aggregation * will not be defined on the response (because it's a sub aggregation). From 407db6295e56ea182cd3aa6d58af73d1b7f8ca2c Mon Sep 17 00:00:00 2001 From: Jonathan Buttner Date: Fri, 26 Jun 2020 17:58:25 -0400 Subject: [PATCH 05/16] Refactoring the children helper to handle pagination without totals --- .../resolver/utils/children_helper.test.ts | 221 +++++++++++++----- .../routes/resolver/utils/children_helper.ts | 161 +++++++++---- 2 files changed, 290 insertions(+), 92 deletions(-) diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/children_helper.test.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/children_helper.test.ts index 51c9cef08a466..ca5b5aef0f651 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/children_helper.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/children_helper.test.ts @@ -3,78 +3,195 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -import _ from 'lodash'; - -import { EndpointDocGenerator } from '../../../../../common/endpoint/generate_data'; +import { + EndpointDocGenerator, + Tree, + Event, + TreeNode, +} from '../../../../../common/endpoint/generate_data'; import { ChildrenNodesHelper } from './children_helper'; -import { eventId, entityId, parentEntityId } from '../../../../../common/endpoint/models/event'; -import { ResolverEvent, ResolverChildren } from '../../../../../common/endpoint/types'; - -function findParents(events: ResolverEvent[]): ResolverEvent[] { - const cache = _.groupBy(events, entityId); +import { eventId, isProcessStart } from '../../../../../common/endpoint/models/event'; - const parents: ResolverEvent[] = []; - Object.values(cache).forEach((lifecycle) => { - const parentNode = cache[parentEntityId(lifecycle[0])!]; - if (parentNode) { - parents.push(parentNode[0]); +function getStartEvents(events: Event[]): Event[] { + const startEvents: Event[] = []; + for (const event of events) { + if (isProcessStart(event)) { + startEvents.push(event); } - }); - return parents; + } + return startEvents; } -function findNode(tree: ResolverChildren, id: string) { - return tree.childNodes.find((node) => { - return node.entityID === id; - }); +function getAllChildrenEvents(tree: Tree) { + const children: Event[] = []; + for (const child of tree.children.values()) { + children.push(...child.lifecycle); + } + return children; +} + +function getStartEventsFromLevels(levels: Array>) { + const startEvents: Event[] = []; + for (const level of levels) { + for (const node of level.values()) { + startEvents.push(...getStartEvents(node.lifecycle)); + } + } + + return startEvents; } describe('Children helper', () => { const generator = new EndpointDocGenerator(); - const root = generator.generateEvent(); + + let tree: Tree; + let helper: ChildrenNodesHelper; + let childrenEvents: Event[]; + let childrenStartEvents: Event[]; + beforeEach(() => { + tree = generator.generateTree({ + children: 3, + alwaysGenMaxChildrenPerNode: true, + generations: 3, + percentTerminated: 100, + ancestryArraySize: 2, + }); + helper = new ChildrenNodesHelper(tree.origin.id, tree.children.size); + childrenEvents = getAllChildrenEvents(tree); + childrenStartEvents = getStartEvents(childrenEvents); + }); + + it('returns the correct entity_ids', () => { + helper.addLifecycleEvents(childrenEvents); + expect(helper.getEntityIDs()).toEqual(Array.from(tree.children.keys())); + }); + + it('returns the correct number of nodes', () => { + helper.addLifecycleEvents(childrenEvents); + expect(helper.getNumNodes()).toEqual(tree.children.size); + }); + + it('marks the query nodes as null', () => { + // +1 indicates that we haven't received all the results so it should create a pagination cursor for the + // queried node (aka the origin that we're passing in) + helper = new ChildrenNodesHelper(tree.origin.id, tree.children.size + 1); + + const nextQuery = helper.addStartEvents(new Set([tree.origin.id]), childrenStartEvents); + helper.addStartEvents(nextQuery!, []); + const nodes = helper.getNodes(); + expect(nodes.nextChild).toBeNull(); + for (const node of nodes.childNodes) { + expect(node.nextChild).toBeNull(); + } + }); + + it('returns undefined when the limit is reached', () => { + helper = new ChildrenNodesHelper(tree.origin.id, tree.children.size - 1); + + expect(helper.addStartEvents(new Set([tree.origin.id]), childrenStartEvents)).toBeUndefined(); + }); + + it('handles multiple additions of start events', () => { + // + 1 indicates that we got everything that ES had + helper = new ChildrenNodesHelper(tree.origin.id, childrenStartEvents.length + 1); + + const level1And2 = getStartEventsFromLevels(tree.childrenLevels.slice(0, 2)); + let nextQuery = helper.addStartEvents(new Set([tree.origin.id]), level1And2); + expect(nextQuery?.size).toEqual(tree.childrenLevels[1].size); + for (const node of tree.childrenLevels[1].values()) { + expect(nextQuery?.has(node.id)).toBeTruthy(); + } + + const level3 = getStartEventsFromLevels(tree.childrenLevels.slice(2, 3)); + nextQuery = helper.addStartEvents(nextQuery!, level3); + expect(nextQuery).toBeUndefined(); + const nodes = helper.getNodes(); + expect(nodes.nextChild).toBeNull(); + for (const node of nodes.childNodes) { + expect(node.nextChild).toBeNull(); + } + }); + + it('handles an empty set', () => { + helper = new ChildrenNodesHelper(tree.origin.id, 1); + + const nextQuery = helper.addStartEvents(new Set([tree.origin.id]), []); + expect(nextQuery).toBeUndefined(); + const nodes = helper.getNodes(); + expect(nodes.nextChild).toBeNull(); + expect(nodes.childNodes.length).toEqual(0); + }); + + it('handles an empty set after multiple additions', () => { + // + 1 indicates that we got everything that ES had + helper = new ChildrenNodesHelper(tree.origin.id, childrenStartEvents.length + 1); + + const level1And2 = getStartEventsFromLevels(tree.childrenLevels.slice(0, 2)); + let nextQuery = helper.addStartEvents(new Set([tree.origin.id]), level1And2); + + nextQuery = helper.addStartEvents(nextQuery!, []); + expect(nextQuery).toBeUndefined(); + const nodes = helper.getNodes(); + expect(nodes.nextChild).toBeNull(); + for (const node of nodes.childNodes) { + expect(node.nextChild).toBeNull(); + } + }); + + it('non leaf nodes are set to undefined by default', () => { + // + 1 indicates that we got everything that ES had + helper = new ChildrenNodesHelper(tree.origin.id, childrenStartEvents.length + 1); + const level1And2 = getStartEventsFromLevels(tree.childrenLevels.slice(0, 2)); + helper.addStartEvents(new Set([tree.origin.id]), level1And2); + const nodes = helper.getNodes(); + expect(nodes.nextChild).toBeNull(); + for (const node of nodes.childNodes) { + if (tree.childrenLevels[0].has(node.entityID)) { + expect(node.nextChild).toBeNull(); + } else { + expect(node.nextChild).toBeUndefined(); + } + } + }); + + it('returns the leaf nodes', () => { + helper = new ChildrenNodesHelper(tree.origin.id, tree.children.size + 1); + + const nextQuery = helper.addStartEvents(new Set([tree.origin.id]), childrenStartEvents); + // we're using an ancestry array of 2 so the leaf nodes are at the second level + expect(nextQuery?.size).toEqual(tree.childrenLevels[1].size); + + for (const node of tree.childrenLevels[1].values()) { + expect(nextQuery?.has(node.id)).toBeTruthy(); + } + }); it('builds the children response structure', () => { - const children = Array.from(generator.descendantsTreeGenerator(root, 3, 3, 0, 0, 0, 100, true)); - - // because we requested the generator to always return the max children, there will always be at least 2 parents - const parents = findParents(children); - - // this represents the aggregation returned from elastic search - // each node in the tree should have 3 children, so if these values are greater than 3 there should be - // pagination cursors created for those children - const totals = { - [root.process.entity_id]: 100, - [entityId(parents[0])]: 10, - [entityId(parents[1])]: 0, - }; - - const helper = new ChildrenNodesHelper(root.process.entity_id); - helper.addChildren(totals, children); - const tree = helper.getNodes(); - expect(tree.nextChild).not.toBeNull(); - - let parent = findNode(tree, entityId(parents[0])); - expect(parent?.nextChild).not.toBeNull(); - parent = findNode(tree, entityId(parents[1])); - expect(parent?.nextChild).toBeNull(); - - tree.childNodes.forEach((node) => { + helper.addStartEvents(new Set([tree.origin.id]), childrenStartEvents); + helper.addLifecycleEvents(childrenEvents); + const childrenNodes = helper.getNodes(); + + // since we got all the nodes all the nextChild cursors should be null + for (const node of childrenNodes.childNodes) { + expect(node.nextChild).toBeUndefined(); + } + expect(childrenNodes.nextChild).not.toBeNull(); + + childrenNodes.childNodes.forEach((node) => { node.lifecycle.forEach((event) => { - expect(children.find((child) => child.event.id === eventId(event))).toEqual(event); + expect(childrenEvents.find((child) => child.event.id === eventId(event))).toEqual(event); }); }); }); it('builds the children response structure twice', () => { - const children = Array.from(generator.descendantsTreeGenerator(root, 3, 3, 0, 0, 100)); - const helper = new ChildrenNodesHelper(root.process.entity_id); - helper.addChildren({}, children); + helper.addLifecycleEvents(childrenEvents); helper.getNodes(); - const tree = helper.getNodes(); - tree.childNodes.forEach((node) => { + const childrenNodes = helper.getNodes(); + childrenNodes.childNodes.forEach((node) => { node.lifecycle.forEach((event) => { - expect(children.find((child) => child.event.id === eventId(event))).toEqual(event); + expect(childrenEvents.find((child) => child.event.id === eventId(event))).toEqual(event); }); }); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/children_helper.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/children_helper.ts index e60e5087c30a9..f51587766c3c9 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/children_helper.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/children_helper.ts @@ -8,30 +8,31 @@ import { entityId, parentEntityId, isProcessStart, + getAncestryAsArray, } from '../../../../../common/endpoint/models/event'; import { ResolverChildNode, ResolverEvent, ResolverChildren, } from '../../../../../common/endpoint/types'; -import { PaginationBuilder } from './pagination'; import { createChild } from './node'; +import { PaginationBuilder } from './pagination'; /** * This class helps construct the children structure when building a resolver tree. */ export class ChildrenNodesHelper { - private readonly cache: Map = new Map(); + private readonly entityToNodeCache: Map = new Map(); - constructor(private readonly rootID: string) { - this.cache.set(rootID, createChild(rootID)); + constructor(private readonly rootID: string, private readonly limit: number) { + this.entityToNodeCache.set(rootID, createChild(rootID)); } /** * Constructs a ResolverChildren response based on the children that were previously add. */ getNodes(): ResolverChildren { - const cacheCopy: Map = new Map(this.cache); + const cacheCopy: Map = new Map(this.entityToNodeCache); const rootNode = cacheCopy.get(this.rootID); let rootNextChild = null; @@ -42,56 +43,136 @@ export class ChildrenNodesHelper { cacheCopy.delete(this.rootID); return { childNodes: Array.from(cacheCopy.values()), - nextChild: rootNextChild, + nextChild: rootNextChild || null, }; } /** - * Add children to the cache. - * - * @param totals a map of unique node IDs to total number of child nodes - * @param results events from a children query + * Get the entity_ids of the nodes that are cached. + */ + getEntityIDs(): string[] { + const cacheCopy: Map = new Map(this.entityToNodeCache); + cacheCopy.delete(this.rootID); + return Array.from(cacheCopy.keys()); + } + + /** + * Get the number of nodes that have been cached. */ - addChildren(totals: Record, results: ResolverEvent[]) { - const startEventsCache: Map = new Map(); + getNumNodes(): number { + // -1 because the root node is in the cache too + return this.entityToNodeCache.size - 1; + } - results.forEach((event) => { + /** + * Add lifecycle events (start, end, etc) to the cache. + * + * @param lifecycle an array of resolver lifecycle events for different process nodes returned from ES. + */ + addLifecycleEvents(lifecycle: ResolverEvent[]) { + for (const event of lifecycle) { const entityID = entityId(event); - const parentID = parentEntityId(event); - if (!entityID || !parentID) { - return; + if (entityID) { + const cachedChild = this.getOrCreateChildNode(entityID); + cachedChild.lifecycle.push(event); } + } + } - let cachedChild = this.cache.get(entityID); - if (!cachedChild) { - cachedChild = createChild(entityID); - this.cache.set(entityID, cachedChild); - } - cachedChild.lifecycle.push(event); + /** + * Add the start events for the nodes received from ES. Pagination cursors will be constructed based on the + * request limit and results returned. + * + * @param queriedNodes the entity_ids of the nodes that returned these start events + * @param startEvents an array of start events returned by ES + */ + addStartEvents(queriedNodes: Set, startEvents: ResolverEvent[]): Set | undefined { + let largestAncestryArray = 0; + const nodesToQueryNext: Map> = new Map(); + const nonLeafNodes: Set = new Set(); - if (isProcessStart(event)) { - let startEvents = startEventsCache.get(parentID); - if (startEvents === undefined) { - startEvents = []; - startEventsCache.set(parentID, startEvents); + const isDistantGrandchild = (event: ResolverEvent) => { + const ancestry = getAncestryAsArray(event); + return ancestry.length > 0 && queriedNodes.has(ancestry[ancestry.length - 1]); + }; + + for (const event of startEvents) { + const parentID = parentEntityId(event); + const entityID = entityId(event); + if (parentID && entityID && isProcessStart(event)) { + // don't actually add the start event to the node, because that'll be done in + // a different call + const childNode = this.getOrCreateChildNode(entityID); + + const ancestry = getAncestryAsArray(event); + // This is to handle the following unlikely but possible scenario: + // if an alert was generated by the kernel process (parent process of all other processes) then + // the direct children of that process would only have an ancestry array of [parent_kernel], a single value in the array. + // The children of those children would have two values in their array [direct parent, parent_kernel] + // we need to determine which nodes are the most distant grandchildren of the queriedNodes because those should + // be used for the next query if more nodes should be retrieved. To generally determine the most distant grandchildren + // we can use the last entry in the ancestry array because of its ordering. The problem with that is in the scenario above + // the direct children of parent_kernel will also meet that criteria even though they are not actually the most + // distant grandchildren. To get around that issue we'll bucket all the nodes by the size of their ancestry array + // and then only return the nodes in the largest bucket because those should be the most distant grandchildren + // from the queried nodes that were passed in. + if (ancestry.length > largestAncestryArray) { + largestAncestryArray = ancestry.length; + } + + // a grandchild must have an array of > 0 and have it's last parent be in the set of previously queried nodes + // this is one of the furthest descendants from the queried nodes + if (isDistantGrandchild(event)) { + let levelOfNodes = nodesToQueryNext.get(ancestry.length); + if (!levelOfNodes) { + levelOfNodes = new Set(); + nodesToQueryNext.set(ancestry.length, levelOfNodes); + } + levelOfNodes.add(entityID); + } else { + nonLeafNodes.add(childNode); } - startEvents.push(event); } - }); + } + + // we may not have received all the possible nodes so mark pagination for the query nodes + // we won't know if the non leaf nodes (non query nodes) have additional children so don't mark them + if (this.limit <= this.getNumNodes()) { + this.setPaginationForNodes(queriedNodes, startEvents); + return; + } + + // the non leaf nodes have received all their children so mark them as finished + for (const nonLeaf of nonLeafNodes.values()) { + nonLeaf.nextChild = null; + } - this.addChildrenPagination(startEventsCache, totals); + // we've received all the descendants of the previously queried node that we can get using it's ancestry array + // so mark those nodes as complete + for (const nodeEntityID of queriedNodes.values()) { + const node = this.entityToNodeCache.get(nodeEntityID); + if (node) { + node.nextChild = null; + } + } + return nodesToQueryNext.get(largestAncestryArray); } - private addChildrenPagination( - startEventsCache: Map, - totals: Record - ) { - Object.entries(totals).forEach(([parentID, total]) => { - const parentNode = this.cache.get(parentID); - const childrenStartEvents = startEventsCache.get(parentID); - if (parentNode && childrenStartEvents) { - parentNode.nextChild = PaginationBuilder.buildCursor(total, childrenStartEvents); + private setPaginationForNodes(nodes: Set, startEvents: ResolverEvent[]) { + for (const nodeEntityID of nodes.values()) { + const cachedNode = this.entityToNodeCache.get(nodeEntityID); + if (cachedNode) { + cachedNode.nextChild = PaginationBuilder.buildCursor(startEvents); } - }); + } + } + + private getOrCreateChildNode(entityID: string) { + let cachedChild = this.entityToNodeCache.get(entityID); + if (!cachedChild) { + cachedChild = createChild(entityID); + this.entityToNodeCache.set(entityID, cachedChild); + } + return cachedChild; } } From 8947b4330d6dbe3061c4f78cee1fead156f95914 Mon Sep 17 00:00:00 2001 From: Jonathan Buttner Date: Fri, 26 Jun 2020 17:59:00 -0400 Subject: [PATCH 06/16] Pinning the seed for the resolver tree generator service --- x-pack/test/api_integration/services/resolver.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/x-pack/test/api_integration/services/resolver.ts b/x-pack/test/api_integration/services/resolver.ts index 7a100c37aea91..750d2f702fb84 100644 --- a/x-pack/test/api_integration/services/resolver.ts +++ b/x-pack/test/api_integration/services/resolver.ts @@ -18,6 +18,7 @@ export interface Options extends TreeOptions { * Number of trees to generate. */ numTrees?: number; + seed?: string; } /** @@ -38,8 +39,9 @@ export function ResolverGeneratorProvider({ getService }: FtrProviderContext) { eventsIndex: string = 'logs-endpoint.events.process-default', alertsIndex: string = 'logs-endpoint.alerts-default' ): Promise { + const seed = options.seed || 'resolver-seed'; const allTrees: Tree[] = []; - const generator = new EndpointDocGenerator(); + const generator = new EndpointDocGenerator(seed); const numTrees = options.numTrees ?? 1; for (let j = 0; j < numTrees; j++) { const tree = generator.generateTree(options); From 1287379271d0b6dd41d0bcd841e9bfab7deb6512 Mon Sep 17 00:00:00 2001 From: Jonathan Buttner Date: Fri, 26 Jun 2020 18:00:49 -0400 Subject: [PATCH 07/16] Splitting the fetcher into multiple classes for msearch --- .../resolver/utils/alerts_query_handler.ts | 83 +++++ .../resolver/utils/ancestry_query_handler.ts | 130 +++++++ .../utils/children_lifecycle_query_handler.ts | 71 ++++ .../utils/children_start_query_handler.ts | 109 ++++++ .../resolver/utils/events_query_handler.ts | 81 ++++ .../endpoint/routes/resolver/utils/fetch.ts | 350 +++++++++--------- .../resolver/utils/lifecycle_query_handler.ts | 72 ++++ .../endpoint/routes/resolver/utils/node.ts | 15 +- 8 files changed, 743 insertions(+), 168 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/alerts_query_handler.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/ancestry_query_handler.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/children_lifecycle_query_handler.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/children_start_query_handler.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/events_query_handler.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/lifecycle_query_handler.ts diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/alerts_query_handler.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/alerts_query_handler.ts new file mode 100644 index 0000000000000..79e4acceca13c --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/alerts_query_handler.ts @@ -0,0 +1,83 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { SearchResponse } from 'elasticsearch'; +import { IScopedClusterClient } from 'src/core/server'; +import { ResolverRelatedAlerts, ResolverEvent } from '../../../../../common/endpoint/types'; +import { createRelatedAlerts } from './node'; +import { AlertsQuery } from '../queries/alerts'; +import { PaginationBuilder } from './pagination'; +import { QueryInfo } from '../queries/multi_searcher'; +import { SingleQueryHandler } from './fetch'; + +/** + * Requests related alerts for the given node. + */ +export class RelatedAlertsQueryHandler implements SingleQueryHandler { + private relatedAlerts: ResolverRelatedAlerts | undefined; + private readonly query: AlertsQuery; + constructor( + private readonly limit: number, + private readonly entityID: string, + after: string | undefined, + indexPattern: string, + legacyEndpointID: string | undefined + ) { + this.query = new AlertsQuery( + PaginationBuilder.createBuilder(limit, after), + indexPattern, + legacyEndpointID + ); + } + + private handleResponse = (response: SearchResponse) => { + const results = this.query.formatResponse(response); + this.relatedAlerts = createRelatedAlerts( + this.entityID, + results, + PaginationBuilder.buildCursorRequestLimit(this.limit, results) + ); + }; + + /** + * Builds a QueryInfo object that defines the related alerts to search for and how to handle the response. + * + * This will return undefined onces the results have been retrieved from ES. + */ + nextQuery(): QueryInfo | undefined { + if (this.getResults()) { + return; + } + + return { + query: this.query, + ids: this.entityID, + handler: this.handleResponse, + }; + } + + /** + * Get the results after an msearch. + */ + getResults() { + return this.relatedAlerts; + } + + /** + * Perform a regular search and return the results. + * + * @param client the elasticsearch client + */ + async search(client: IScopedClusterClient) { + const results = this.getResults(); + if (results) { + return results; + } + + this.handleResponse(await this.query.search(client, this.entityID)); + return this.getResults() ?? createRelatedAlerts(this.entityID); + } +} diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/ancestry_query_handler.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/ancestry_query_handler.ts new file mode 100644 index 0000000000000..70987aaa01603 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/ancestry_query_handler.ts @@ -0,0 +1,130 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { SearchResponse } from 'elasticsearch'; +import { IScopedClusterClient } from 'src/core/server'; +import { + parentEntityId, + entityId, + getAncestryAsArray, +} from '../../../../../common/endpoint/models/event'; +import { + ResolverAncestry, + ResolverEvent, + LifecycleNode, +} from '../../../../../common/endpoint/types'; +import { createAncestry, createLifecycle } from './node'; +import { LifecycleQuery } from '../queries/lifecycle'; +import { QueryInfo } from '../queries/multi_searcher'; +import { QueryHandler } from './fetch'; + +/** + * Retrieve the ancestry portion of a resolver tree. + */ +export class AncestryQueryHandler implements QueryHandler { + private readonly ancestry: ResolverAncestry = createAncestry(); + private ancestorsToFind: string[]; + private readonly query: LifecycleQuery; + + constructor( + private levels: number, + indexPattern: string, + legacyEndpointID: string | undefined, + originNode: LifecycleNode | undefined + ) { + this.ancestorsToFind = getAncestryAsArray(originNode?.lifecycle[0]).slice(0, levels); + this.query = new LifecycleQuery(indexPattern, legacyEndpointID); + + // add the origin node to the response if it exists + if (originNode) { + this.ancestry.ancestors.push(originNode); + this.ancestry.nextAncestor = parentEntityId(originNode.lifecycle[0]) || null; + } + } + + private toMapOfNodes(results: ResolverEvent[]) { + return results.reduce((nodes: Map, event: ResolverEvent) => { + const nodeId = entityId(event); + let node = nodes.get(nodeId); + if (!node) { + node = createLifecycle(nodeId, []); + } + + node.lifecycle.push(event); + return nodes.set(nodeId, node); + }, new Map()); + } + + private setNoMore() { + this.ancestry.nextAncestor = null; + this.ancestorsToFind = []; + this.levels = 0; + } + + private handleResponse = (searchResp: SearchResponse) => { + const results = this.query.formatResponse(searchResp); + if (results.length === 0) { + this.setNoMore(); + return; + } + + // bucket the start and end events together for a single node + const ancestryNodes = this.toMapOfNodes(results); + + // the order of this array is going to be weird, it will look like this + // [furthest grandparent...closer grandparent, next recursive call furthest grandparent...closer grandparent] + this.ancestry.ancestors.push(...ancestryNodes.values()); + this.ancestry.nextAncestor = parentEntityId(results[0]) || null; + this.levels = this.levels - ancestryNodes.size; + // the results come back in ascending order on timestamp so the first entry in the + // results should be the further ancestor (most distant grandparent) + this.ancestorsToFind = getAncestryAsArray(results[0]).slice(0, this.levels); + }; + + /** + * Returns whether there are more results to retrieve based on the limit that is passed in and the results that + * have already been received from ES. + */ + hasMore(): boolean { + return this.levels > 0 && this.ancestorsToFind.length > 0; + } + + /** + * Get a query info for retrieving the next set of results. + */ + nextQuery(): QueryInfo | undefined { + if (this.hasMore()) { + return { + query: this.query, + ids: this.ancestorsToFind, + handler: this.handleResponse, + }; + } + } + + /** + * Return the results after using msearch to find them. + */ + getResults() { + return this.ancestry; + } + + /** + * Perform a regular search and return the results. + * + * @param client the elasticsearch client. + */ + async search(client: IScopedClusterClient) { + while (this.hasMore()) { + const info = this.nextQuery(); + if (!info) { + break; + } + this.handleResponse(await this.query.search(client, info.ids)); + } + return this.getResults(); + } +} diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/children_lifecycle_query_handler.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/children_lifecycle_query_handler.ts new file mode 100644 index 0000000000000..f4223cbb2e38b --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/children_lifecycle_query_handler.ts @@ -0,0 +1,71 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { SearchResponse } from 'elasticsearch'; +import { IScopedClusterClient } from 'src/core/server'; +import { ResolverEvent, ResolverChildren } from '../../../../../common/endpoint/types'; +import { LifecycleQuery } from '../queries/lifecycle'; +import { QueryInfo } from '../queries/multi_searcher'; +import { SingleQueryHandler } from './fetch'; +import { ChildrenNodesHelper } from './children_helper'; +import { createChildren } from './node'; + +/** + * Returns the children of a resolver tree. + */ +export class ChildrenLifecycleQueryHandler implements SingleQueryHandler { + private lifecycle: ResolverChildren | undefined; + private readonly query: LifecycleQuery; + constructor( + private readonly childrenHelper: ChildrenNodesHelper, + indexPattern: string, + legacyEndpointID: string | undefined + ) { + this.query = new LifecycleQuery(indexPattern, legacyEndpointID); + } + + private handleResponse = (response: SearchResponse) => { + this.childrenHelper.addLifecycleEvents(this.query.formatResponse(response)); + this.lifecycle = this.childrenHelper.getNodes(); + }; + + /** + * Get the query for msearch. Once the results are set this will return undefined. + */ + nextQuery(): QueryInfo | undefined { + if (this.getResults()) { + return; + } + + return { + query: this.query, + ids: this.childrenHelper.getEntityIDs(), + handler: this.handleResponse, + }; + } + + /** + * Return the results from the search. + */ + getResults(): ResolverChildren | undefined { + return this.lifecycle; + } + + /** + * Perform a regular search and return the results. + * + * @param client the elasticsearch client + */ + async search(client: IScopedClusterClient) { + const results = this.getResults(); + if (results) { + return results; + } + + this.handleResponse(await this.query.search(client, this.childrenHelper.getEntityIDs())); + return this.getResults() || createChildren(); + } +} diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/children_start_query_handler.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/children_start_query_handler.ts new file mode 100644 index 0000000000000..15bcbceb3a342 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/children_start_query_handler.ts @@ -0,0 +1,109 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { SearchResponse } from 'elasticsearch'; +import { IScopedClusterClient } from 'src/core/server'; +import { ResolverEvent } from '../../../../../common/endpoint/types'; +import { ChildrenQuery } from '../queries/children'; +import { QueryInfo } from '../queries/multi_searcher'; +import { QueryHandler } from './fetch'; +import { ChildrenNodesHelper } from './children_helper'; +import { PaginationBuilder } from './pagination'; + +/** + * Retrieve the start lifecycle events for the children of a resolver tree. + * + * If using msearch you should loop over hasMore() because the results are limited to the size of the ancestry array. + */ +export class ChildrenStartQueryHandler implements QueryHandler { + private readonly childrenHelper: ChildrenNodesHelper; + private limitLeft: number; + private query: ChildrenQuery; + private nodesToQuery: Set; + + constructor( + private readonly limit: number, + entityID: string, + after: string | undefined, + private readonly indexPattern: string, + private readonly legacyEndpointID: string | undefined + ) { + this.query = new ChildrenQuery( + PaginationBuilder.createBuilder(limit, after), + indexPattern, + legacyEndpointID + ); + this.childrenHelper = new ChildrenNodesHelper(entityID, this.limit); + this.limitLeft = this.limit; + this.nodesToQuery = new Set([entityID]); + } + + private setNoMore() { + this.nodesToQuery = new Set(); + this.limitLeft = 0; + } + + private handleResponse = (response: SearchResponse) => { + const results = this.query.formatResponse(response); + this.nodesToQuery = this.childrenHelper.addStartEvents(this.nodesToQuery, results) ?? new Set(); + + if (results.length === 0) { + this.setNoMore(); + return; + } + + this.limitLeft = this.limit - this.childrenHelper.getNumNodes(); + this.query = new ChildrenQuery( + PaginationBuilder.createBuilder(this.limitLeft), + this.indexPattern, + this.legacyEndpointID + ); + }; + + /** + * Check if there are more results to retrieve based on the limit that was passed in. + */ + hasMore(): boolean { + return this.limitLeft > 0 && this.nodesToQuery.size > 0; + } + + /** + * Get a query to retrieve the next set of results. + */ + nextQuery(): QueryInfo | undefined { + if (this.hasMore()) { + return { + query: this.query, + // This should never be undefined because the check above + ids: Array.from(this.nodesToQuery.values()), + handler: this.handleResponse, + }; + } + } + + /** + * Get the cached results from the ES responses. + */ + getResults(): ChildrenNodesHelper { + return this.childrenHelper; + } + + /** + * Perform a regular search and return the helper. + * + * @param client the elasticsearch client + */ + async search(client: IScopedClusterClient) { + while (this.hasMore()) { + const info = this.nextQuery(); + if (!info) { + break; + } + this.handleResponse(await this.query.search(client, info.ids)); + } + return this.getResults(); + } +} diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/events_query_handler.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/events_query_handler.ts new file mode 100644 index 0000000000000..54d5c878033c5 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/events_query_handler.ts @@ -0,0 +1,81 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { SearchResponse } from 'elasticsearch'; +import { IScopedClusterClient } from 'src/core/server'; +import { ResolverRelatedEvents, ResolverEvent } from '../../../../../common/endpoint/types'; +import { createRelatedEvents } from './node'; +import { EventsQuery } from '../queries/events'; +import { PaginationBuilder } from './pagination'; +import { QueryInfo } from '../queries/multi_searcher'; +import { SingleQueryHandler } from './fetch'; + +/** + * This retrieves the related events for the origin node of a resolver tree. + */ +export class RelatedEventsQueryHandler implements SingleQueryHandler { + private relatedEvents: ResolverRelatedEvents | undefined; + private readonly query: EventsQuery; + constructor( + private readonly limit: number, + private readonly entityID: string, + after: string | undefined, + indexPattern: string, + legacyEndpointID: string | undefined + ) { + this.query = new EventsQuery( + PaginationBuilder.createBuilder(limit, after), + indexPattern, + legacyEndpointID + ); + } + + private handleResponse = (response: SearchResponse) => { + const results = this.query.formatResponse(response); + this.relatedEvents = createRelatedEvents( + this.entityID, + results, + PaginationBuilder.buildCursorRequestLimit(this.limit, results) + ); + }; + + /** + * Get a query to use in a msearch. + */ + nextQuery(): QueryInfo | undefined { + if (this.getResults()) { + return; + } + + return { + query: this.query, + ids: this.entityID, + handler: this.handleResponse, + }; + } + + /** + * Get the results after an msearch. + */ + getResults() { + return this.relatedEvents; + } + + /** + * Perform a normal search and return the related events results. + * + * @param client the elasticsearch client + */ + async search(client: IScopedClusterClient) { + const results = this.getResults(); + if (results) { + return results; + } + + this.handleResponse(await this.query.search(client, this.entityID)); + return this.getResults() ?? createRelatedEvents(this.entityID); + } +} diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/fetch.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/fetch.ts index 0af2fca7106be..c38127e600eb4 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/fetch.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/fetch.ts @@ -11,22 +11,61 @@ import { ResolverAncestry, ResolverRelatedAlerts, ResolverLifecycleNode, - ResolverEvent, } from '../../../../../common/endpoint/types'; -import { - entityId, - ancestryArray, - parentEntityId, -} from '../../../../../common/endpoint/models/event'; -import { PaginationBuilder } from './pagination'; import { Tree } from './tree'; import { LifecycleQuery } from '../queries/lifecycle'; -import { ChildrenQuery } from '../queries/children'; -import { EventsQuery } from '../queries/events'; import { StatsQuery } from '../queries/stats'; -import { createAncestry, createRelatedEvents, createLifecycle, createRelatedAlerts } from './node'; -import { ChildrenNodesHelper } from './children_helper'; -import { AlertsQuery } from '../queries/alerts'; +import { createLifecycle } from './node'; +import { MultiSearcher, QueryInfo } from '../queries/multi_searcher'; +import { AncestryQueryHandler } from './ancestry_query_handler'; +import { RelatedEventsQueryHandler } from './events_query_handler'; +import { RelatedAlertsQueryHandler } from './alerts_query_handler'; +import { ChildrenStartQueryHandler } from './children_start_query_handler'; +import { ChildrenLifecycleQueryHandler } from './children_lifecycle_query_handler'; +import { LifecycleQueryHandler } from './lifecycle_query_handler'; + +/** + * The query parameters passed in from the request. These define the limits for the ES requests for retrieving the + * resolver tree. + */ +export interface TreeOptions { + children: number; + ancestors: number; + events: number; + alerts: number; + afterAlert?: string; + afterEvent?: string; + afterChild?: string; +} + +interface QueryBuilder { + nextQuery(): QueryInfo | undefined; +} + +/** + * This interface defines the contract for a query handler that will only be used once in an msearch call. + */ +export interface SingleQueryHandler extends QueryBuilder { + /** + * This method returns the results if the query has been used in an msearch call or undefined if not. + */ + getResults(): T | undefined; + /** + * Do a regular search instead of msearch. + * @param client the elasticsearch client + */ + search(client: IScopedClusterClient): Promise; +} + +/** + * This interface defines the contract for a query handler that can be used multiple times by msearch. + */ +export interface QueryHandler extends SingleQueryHandler { + /** + * Returns whether additional msearch are required to retrieve the rest of the expected data from ES. + */ + hasMore(): boolean; +} /** * Handles retrieving nodes of a resolver tree. @@ -52,46 +91,138 @@ export class Fetcher { private readonly endpointID?: string ) {} + /** + * This method retrieves the resolver tree starting from the `id` during construction of the class. + * + * @param options the options for retrieving the structure of the tree. + */ + public async tree(options: TreeOptions) { + const addQueryToList = (queryHandler: QueryBuilder, queries: QueryInfo[]) => { + const queryInfo = queryHandler.nextQuery(); + if (queryInfo !== undefined) { + queries.push(queryInfo); + } + }; + + const originHandler = new LifecycleQueryHandler( + this.id, + this.eventsIndexPattern, + this.endpointID + ); + + const eventsHandler = new RelatedEventsQueryHandler( + options.events, + this.id, + options.afterEvent, + this.eventsIndexPattern, + this.endpointID + ); + + const alertsHandler = new RelatedAlertsQueryHandler( + options.alerts, + this.id, + options.afterAlert, + this.alertsIndexPattern, + this.endpointID + ); + + // we need to get the start events first because the API request defines how many nodes to return and we don't want + // to count or limit ourselves based on the other lifecycle events (end, etc) + const childrenHandler = new ChildrenStartQueryHandler( + options.children, + this.id, + options.afterChild, + this.eventsIndexPattern, + this.endpointID + ); + + const msearch = new MultiSearcher(this.client); + + let queries: QueryInfo[] = []; + addQueryToList(eventsHandler, queries); + addQueryToList(alertsHandler, queries); + addQueryToList(childrenHandler, queries); + addQueryToList(originHandler, queries); + + // get the related events, related alerts, the first pass of children start events, and the origin node + // the origin node is needed so we can get the ancestry array for the additional ancestor calls + await msearch.search(queries); + + const ancestryHandler = new AncestryQueryHandler( + options.ancestors, + this.eventsIndexPattern, + this.endpointID, + originHandler.getResults() + ); + + // get the remaining ancestors and children start events + while (ancestryHandler.hasMore() || childrenHandler.hasMore()) { + queries = []; + addQueryToList(ancestryHandler, queries); + addQueryToList(childrenHandler, queries); + await msearch.search(queries); + } + + const childrenTotalsHelper = childrenHandler.getResults(); + + const childrenLifecycleHandler = new ChildrenLifecycleQueryHandler( + childrenTotalsHelper, + this.eventsIndexPattern, + this.endpointID + ); + + // now that we have all the start events get the full lifecycle nodes + childrenLifecycleHandler.search(this.client); + + const tree = new Tree(this.id, { + ancestry: ancestryHandler.getResults(), + relatedEvents: eventsHandler.getResults(), + relatedAlerts: alertsHandler.getResults(), + children: childrenLifecycleHandler.getResults(), + }); + + // add the stats to the tree + return this.stats(tree); + } + /** * Retrieves the ancestor nodes for the resolver tree. * * @param limit upper limit of ancestors to retrieve */ public async ancestors(limit: number): Promise { - const ancestryInfo = createAncestry(); const originNode = await this.getNode(this.id); - if (originNode) { - ancestryInfo.ancestors.push(originNode); - // If the request is only for the origin node then set next to its parent - ancestryInfo.nextAncestor = parentEntityId(originNode.lifecycle[0]) || null; - await this.doAncestors( - // limit the ancestors we're looking for to the number of levels - // the array could be up to length 20 but that could change - Fetcher.getAncestryAsArray(originNode.lifecycle[0]).slice(0, limit), - limit, - ancestryInfo - ); - } - return ancestryInfo; + const ancestryHandler = new AncestryQueryHandler( + limit, + this.eventsIndexPattern, + this.endpointID, + originNode + ); + return ancestryHandler.search(this.client); } /** * Retrieves the children nodes for the resolver tree. * * @param limit the number of children to retrieve for a single level - * @param generations number of levels to return * @param after a cursor to use as the starting point for retrieving children */ - public async children( - limit: number, - generations: number, - after?: string - ): Promise { - const helper = new ChildrenNodesHelper(this.id); - - await this.doChildren(helper, [this.id], limit, generations, after); + public async children(limit: number, after?: string): Promise { + const childrenHandler = new ChildrenStartQueryHandler( + limit, + this.id, + after, + this.eventsIndexPattern, + this.endpointID + ); + const helper = await childrenHandler.search(this.client); + const childrenLifecycleHandler = new ChildrenLifecycleQueryHandler( + helper, + this.eventsIndexPattern, + this.endpointID + ); - return helper.getNodes(); + return childrenLifecycleHandler.search(this.client); } /** @@ -101,7 +232,15 @@ export class Fetcher { * @param after a cursor to use as the starting point for retrieving related events */ public async events(limit: number, after?: string): Promise { - return this.doEvents(limit, after); + const eventsHandler = new RelatedEventsQueryHandler( + limit, + this.id, + after, + this.eventsIndexPattern, + this.endpointID + ); + + return eventsHandler.search(this.client); } /** @@ -111,26 +250,15 @@ export class Fetcher { * @param after a cursor to use as the starting point for retrieving alerts */ public async alerts(limit: number, after?: string): Promise { - const query = new AlertsQuery( - PaginationBuilder.createBuilder(limit, after), + const alertsHandler = new RelatedAlertsQueryHandler( + limit, + this.id, + after, this.alertsIndexPattern, this.endpointID ); - const { totals, results } = await query.search(this.client, this.id); - if (results.length === 0) { - // return an empty set of results - return createRelatedAlerts(this.id); - } - if (!totals[this.id]) { - throw new Error(`Could not find the totals for related events entity_id: ${this.id}`); - } - - return createRelatedAlerts( - this.id, - results, - PaginationBuilder.buildCursor(totals[this.id], results) - ); + return alertsHandler.search(this.client); } /** @@ -145,7 +273,7 @@ export class Fetcher { private async getNode(entityID: string): Promise { const query = new LifecycleQuery(this.eventsIndexPattern, this.endpointID); - const results = await query.search(this.client, entityID); + const results = await query.searchAndFormat(this.client, entityID); if (results.length === 0) { return; } @@ -153,125 +281,13 @@ export class Fetcher { return createLifecycle(entityID, results); } - private static getAncestryAsArray(event: ResolverEvent): string[] { - const ancestors = ancestryArray(event); - if (ancestors) { - return ancestors; - } - - const parentID = parentEntityId(event); - if (parentID) { - return [parentID]; - } - - return []; - } - - private async doAncestors( - ancestors: string[], - levels: number, - ancestorInfo: ResolverAncestry - ): Promise { - if (levels <= 0) { - return; - } - - const query = new LifecycleQuery(this.eventsIndexPattern, this.endpointID); - const results = await query.search(this.client, ancestors); - - if (results.length === 0) { - ancestorInfo.nextAncestor = null; - return; - } - - // bucket the start and end events together for a single node - const ancestryNodes = results.reduce( - (nodes: Map, ancestorEvent: ResolverEvent) => { - const nodeId = entityId(ancestorEvent); - let node = nodes.get(nodeId); - if (!node) { - node = createLifecycle(nodeId, []); - } - - node.lifecycle.push(ancestorEvent); - return nodes.set(nodeId, node); - }, - new Map() - ); - - // the order of this array is going to be weird, it will look like this - // [furthest grandparent...closer grandparent, next recursive call furthest grandparent...closer grandparent] - ancestorInfo.ancestors.push(...ancestryNodes.values()); - ancestorInfo.nextAncestor = parentEntityId(results[0]) || null; - const levelsLeft = levels - ancestryNodes.size; - // the results come back in ascending order on timestamp so the first entry in the - // results should be the further ancestor (most distant grandparent) - const next = Fetcher.getAncestryAsArray(results[0]).slice(0, levelsLeft); - // the ancestry array currently only holds up to 20 values but we can't rely on that so keep recursing - await this.doAncestors(next, levelsLeft, ancestorInfo); - } - - private async doEvents(limit: number, after?: string) { - const query = new EventsQuery( - PaginationBuilder.createBuilder(limit, after), - this.eventsIndexPattern, - this.endpointID - ); - - const { totals, results } = await query.search(this.client, this.id); - if (results.length === 0) { - // return an empty set of results - return createRelatedEvents(this.id); - } - if (!totals[this.id]) { - throw new Error(`Could not find the totals for related events entity_id: ${this.id}`); - } - - return createRelatedEvents( - this.id, - results, - PaginationBuilder.buildCursor(totals[this.id], results) - ); - } - - private async doChildren( - cache: ChildrenNodesHelper, - ids: string[], - limit: number, - levels: number, - after?: string - ) { - if (levels === 0 || ids.length === 0) { - return; - } - - const childrenQuery = new ChildrenQuery( - PaginationBuilder.createBuilder(limit, after), - this.eventsIndexPattern, - this.endpointID - ); - const lifecycleQuery = new LifecycleQuery(this.eventsIndexPattern, this.endpointID); - - const { totals, results } = await childrenQuery.search(this.client, ids); - if (results.length === 0) { - return; - } - - const childIDs = results.map(entityId); - const children = await lifecycleQuery.search(this.client, childIDs); - - cache.addChildren(totals, children); - - await this.doChildren(cache, childIDs, limit, levels - 1); - } - private async doStats(tree: Tree) { const statsQuery = new StatsQuery( [this.eventsIndexPattern, this.alertsIndexPattern], this.endpointID ); const ids = tree.ids(); - const res = await statsQuery.search(this.client, ids); + const res = await statsQuery.searchAndFormat(this.client, ids); const alerts = res.alerts; const events = res.events; ids.forEach((id) => { diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/lifecycle_query_handler.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/lifecycle_query_handler.ts new file mode 100644 index 0000000000000..165a502a9a4fc --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/lifecycle_query_handler.ts @@ -0,0 +1,72 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { SearchResponse } from 'elasticsearch'; +import { IScopedClusterClient } from 'src/core/server'; +import { ResolverEvent, LifecycleNode } from '../../../../../common/endpoint/types'; +import { LifecycleQuery } from '../queries/lifecycle'; +import { QueryInfo } from '../queries/multi_searcher'; +import { SingleQueryHandler } from './fetch'; +import { createLifecycle } from './node'; + +/** + * Retrieve the lifecycle events for a node. + */ +export class LifecycleQueryHandler implements SingleQueryHandler { + private lifecycle: LifecycleNode | undefined; + private readonly query: LifecycleQuery; + constructor( + private readonly entityID: string, + indexPattern: string, + legacyEndpointID: string | undefined + ) { + this.query = new LifecycleQuery(indexPattern, legacyEndpointID); + } + + private handleResponse = (response: SearchResponse) => { + const results = this.query.formatResponse(response); + if (results.length !== 0) { + this.lifecycle = createLifecycle(this.entityID, results); + } + }; + + /** + * Build the query for retrieving the lifecycle events. This will return undefined once the results have been found. + */ + nextQuery(): QueryInfo | undefined { + if (this.getResults()) { + return; + } + + return { + query: this.query, + ids: this.entityID, + handler: this.handleResponse, + }; + } + + /** + * Get the results from the msearch. + */ + getResults(): LifecycleNode | undefined { + return this.lifecycle; + } + + /** + * Do a regular search and return the results. + * + * @param client the elasticsearch client. + */ + async search(client: IScopedClusterClient) { + const results = this.getResults(); + if (results) { + return results; + } + + this.handleResponse(await this.query.search(client, this.entityID)); + return this.getResults() ?? createLifecycle(this.entityID, []); + } +} diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/node.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/node.ts index 57a2ebfcc1792..6717d9a9dbf19 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/node.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/node.ts @@ -12,6 +12,7 @@ import { ResolverTree, ResolverChildNode, ResolverRelatedAlerts, + ResolverChildren, } from '../../../../../common/endpoint/types'; /** @@ -53,7 +54,6 @@ export function createChild(entityID: string): ResolverChildNode { const lifecycle = createLifecycle(entityID, []); return { ...lifecycle, - nextChild: null, }; } @@ -77,6 +77,19 @@ export function createLifecycle( return { entityID, lifecycle }; } +/** + * Creates a resolver children response. + * + * @param nodes the child nodes to add to the ResolverChildren response + * @param nextChild the cursor for the response + */ +export function createChildren( + nodes: ChildNode[] = [], + nextChild: string | null = null +): ResolverChildren { + return { childNodes: nodes, nextChild }; +} + /** * Creates an empty `Tree` response structure that the tree handler would return * From 54f1a3feab8ab9447a2874566d2c86cd7c63aae5 Mon Sep 17 00:00:00 2001 From: Jonathan Buttner Date: Fri, 26 Jun 2020 18:01:42 -0400 Subject: [PATCH 08/16] Updating tests and api for ancestry array and msearch --- .../common/endpoint/models/event.ts | 18 +++++ .../common/endpoint/schema/resolver.ts | 2 - .../common/endpoint/types.ts | 25 +++++-- .../endpoint/routes/resolver/children.ts | 4 +- .../server/endpoint/routes/resolver/tree.ts | 3 +- .../api_integration/apis/endpoint/resolver.ts | 74 +++++++++---------- 6 files changed, 77 insertions(+), 49 deletions(-) diff --git a/x-pack/plugins/security_solution/common/endpoint/models/event.ts b/x-pack/plugins/security_solution/common/endpoint/models/event.ts index 98f4b4336a1c8..86cccff957211 100644 --- a/x-pack/plugins/security_solution/common/endpoint/models/event.ts +++ b/x-pack/plugins/security_solution/common/endpoint/models/event.ts @@ -60,6 +60,24 @@ export function ancestryArray(event: ResolverEvent): string[] | undefined { return event.process.Ext.ancestry; } +export function getAncestryAsArray(event: ResolverEvent | undefined): string[] { + if (!event) { + return []; + } + + const ancestors = ancestryArray(event); + if (ancestors) { + return ancestors; + } + + const parentID = parentEntityId(event); + if (parentID) { + return [parentID]; + } + + return []; +} + /** * @param event The event to get the category for */ diff --git a/x-pack/plugins/security_solution/common/endpoint/schema/resolver.ts b/x-pack/plugins/security_solution/common/endpoint/schema/resolver.ts index f5c3fd519c9c5..c1c15d724170b 100644 --- a/x-pack/plugins/security_solution/common/endpoint/schema/resolver.ts +++ b/x-pack/plugins/security_solution/common/endpoint/schema/resolver.ts @@ -13,7 +13,6 @@ export const validateTree = { params: schema.object({ id: schema.string() }), query: schema.object({ children: schema.number({ defaultValue: 10, min: 0, max: 100 }), - generations: schema.number({ defaultValue: 3, min: 0, max: 3 }), ancestors: schema.number({ defaultValue: 3, min: 0, max: 5 }), events: schema.number({ defaultValue: 100, min: 0, max: 1000 }), alerts: schema.number({ defaultValue: 100, min: 0, max: 1000 }), @@ -66,7 +65,6 @@ export const validateChildren = { params: schema.object({ id: schema.string() }), query: schema.object({ children: schema.number({ defaultValue: 10, min: 1, max: 100 }), - generations: schema.number({ defaultValue: 3, min: 1, max: 3 }), afterChild: schema.maybe(schema.string()), legacyEndpointID: schema.maybe(schema.string()), }), diff --git a/x-pack/plugins/security_solution/common/endpoint/types.ts b/x-pack/plugins/security_solution/common/endpoint/types.ts index 2efaeef5719b7..13d1e5adce616 100644 --- a/x-pack/plugins/security_solution/common/endpoint/types.ts +++ b/x-pack/plugins/security_solution/common/endpoint/types.ts @@ -76,12 +76,18 @@ export interface ResolverNodeStats { */ export interface ResolverChildNode extends ResolverLifecycleNode { /** - * A child node's pagination cursor can be null for a couple reasons: - * 1. At the time of querying it could have no children in ES, in which case it will be marked as - * null because we know it does not have children during this query. - * 2. If the max level was reached we do not know if this node has children or not so we'll mark it as null + * nextChild can have 3 different states: + * + * undefined: This indicates that you should not use this node for additional queries. It does not mean that node does + * not have any more direct children. The node could have more direct children but to determine that, use the + * ResolverChildren node's nextChild. + * + * null: Indicates that we have received all the children of the node. There may be more descendants though. + * + * string: Indicates this is a leaf node and it can be used to continue querying for additional descendants + * using this node's entity_id */ - nextChild: string | null; + nextChild?: string | null; } /** @@ -91,7 +97,14 @@ export interface ResolverChildNode extends ResolverLifecycleNode { export interface ResolverChildren { childNodes: ResolverChildNode[]; /** - * This is the children cursor for the origin of a tree. + * nextChild can have 2 different states: + * + * null: Indicates that we have received all the descendants that can be retrieved using this node. To retrieve more + * nodes in the tree use a cursor provided in one of the returned children. If no other cursor exists then the tree + * is complete. + * + * string: Indicates this node has more descendants that can be retrieved, pass this cursor in while using this node's + * entity_id for the request. */ nextChild: string | null; } diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/children.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/children.ts index 74448a324a4ec..9b8cd9fd3edab 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/children.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/children.ts @@ -18,14 +18,14 @@ export function handleChildren( return async (context, req, res) => { const { params: { id }, - query: { children, generations, afterChild, legacyEndpointID: endpointID }, + query: { children, afterChild, legacyEndpointID: endpointID }, } = req; try { const client = context.core.elasticsearch.legacy.client; const fetcher = new Fetcher(client, id, eventsIndexPattern, alertsIndexPattern, endpointID); return res.ok({ - body: await fetcher.children(children, generations, afterChild), + body: await fetcher.children(children, afterChild), }); } catch (err) { log.warn(err); diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree.ts index 181fb8c3df3f9..33011078ee823 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree.ts @@ -21,7 +21,6 @@ export function handleTree( params: { id }, query: { children, - generations, ancestors, events, alerts, @@ -37,7 +36,7 @@ export function handleTree( const fetcher = new Fetcher(client, id, eventsIndexPattern, alertsIndexPattern, endpointID); const [childrenNodes, ancestry, relatedEvents, relatedAlerts] = await Promise.all([ - fetcher.children(children, generations, afterChild), + fetcher.children(children, afterChild), fetcher.ancestors(ancestors), fetcher.events(events, afterEvent), fetcher.alerts(alerts, afterAlert), diff --git a/x-pack/test/api_integration/apis/endpoint/resolver.ts b/x-pack/test/api_integration/apis/endpoint/resolver.ts index eeca8ee54e32f..89b4ce88b32a7 100644 --- a/x-pack/test/api_integration/apis/endpoint/resolver.ts +++ b/x-pack/test/api_integration/apis/endpoint/resolver.ts @@ -16,6 +16,7 @@ import { LegacyEndpointEvent, ResolverNodeStats, ResolverRelatedAlerts, + ChildNode, } from '../../../../plugins/security_solution/common/endpoint/types'; import { parentEntityId } from '../../../../plugins/security_solution/common/endpoint/models/event'; import { FtrProviderContext } from '../../ftr_provider_context'; @@ -246,6 +247,8 @@ export default function resolverAPIIntegrationTests({ getService }: FtrProviderC percentWithRelated: 100, numTrees: 1, alwaysGenMaxChildrenPerNode: true, + useAncestryArray: true, + ancestryArraySize: 2, }; describe('Resolver', () => { @@ -542,13 +545,10 @@ export default function resolverAPIIntegrationTests({ getService }: FtrProviderC it('returns multiple levels of child process lifecycle events', async () => { const { body }: { body: ResolverChildren } = await supertest - .get( - `/api/endpoint/resolver/93802/children?legacyEndpointID=${endpointID}&generations=1` - ) + .get(`/api/endpoint/resolver/93802/children?legacyEndpointID=${endpointID}&children=10`) .expect(200); + expect(body.childNodes.length).to.eql(10); expect(body.nextChild).to.be(null); - expect(body.childNodes[0].nextChild).to.be(null); - expect(body.childNodes.length).to.eql(8); expect(body.childNodes[0].lifecycle.length).to.eql(1); expect( // for some reason the ts server doesn't think `endgame` exists even though we're using ResolverEvent @@ -615,19 +615,27 @@ export default function resolverAPIIntegrationTests({ getService }: FtrProviderC expect(body.childNodes.length).to.eql(12); // there will be 4 parents, the origin of the tree, and it's 3 children verifyChildren(body.childNodes, tree, 4, 3); + expect(body.nextChild).to.eql(null); }); it('returns a single generation of children', async () => { + // this gets a node should have 3 children which were created in succession so that the timestamps + // are ordered correctly to be retrieved in a single call + const distantChildEntityID = Array.from(tree.childrenLevels[0].values())[0].id; const { body }: { body: ResolverChildren } = await supertest - .get(`/api/endpoint/resolver/${tree.origin.id}/children?generations=1`) + .get(`/api/endpoint/resolver/${distantChildEntityID}/children?children=3`) .expect(200); expect(body.childNodes.length).to.eql(3); verifyChildren(body.childNodes, tree, 1, 3); + expect(body.nextChild).to.not.eql(null); }); - it('paginates the children of the origin node', async () => { + it('paginates the children', async () => { + // this gets a node should have 3 children which were created in succession so that the timestamps + // are ordered correctly to be retrieved in a single call + const distantChildEntityID = Array.from(tree.childrenLevels[0].values())[0].id; let { body }: { body: ResolverChildren } = await supertest - .get(`/api/endpoint/resolver/${tree.origin.id}/children?generations=1&children=1`) + .get(`/api/endpoint/resolver/${distantChildEntityID}/children?children=1`) .expect(200); expect(body.childNodes.length).to.eql(1); verifyChildren(body.childNodes, tree, 1, 1); @@ -635,49 +643,41 @@ export default function resolverAPIIntegrationTests({ getService }: FtrProviderC ({ body } = await supertest .get( - `/api/endpoint/resolver/${tree.origin.id}/children?generations=1&afterChild=${body.nextChild}` + `/api/endpoint/resolver/${distantChildEntityID}/children?children=2&afterChild=${body.nextChild}` ) .expect(200)); expect(body.childNodes.length).to.eql(2); verifyChildren(body.childNodes, tree, 1, 2); - expect(body.childNodes[0].nextChild).to.be(null); - expect(body.childNodes[1].nextChild).to.be(null); - }); - - it('paginates the children of different nodes', async () => { - let { body }: { body: ResolverChildren } = await supertest - .get(`/api/endpoint/resolver/${tree.origin.id}/children?generations=2&children=2`) - .expect(200); - // it should return 4 nodes total, 2 for each level - expect(body.childNodes.length).to.eql(4); - verifyChildren(body.childNodes, tree, 2); expect(body.nextChild).to.not.be(null); - expect(body.childNodes[0].nextChild).to.not.be(null); - // the second child will not have any results returned for it so it should not have pagination set (the first) - // request to get it's children should start at the beginning aka not passing any pagination parameter - expect(body.childNodes[1].nextChild).to.be(null); - const firstChild = body.childNodes[0]; - - // get the 3rd child of the origin of the tree ({ body } = await supertest .get( - `/api/endpoint/resolver/${tree.origin.id}/children?generations=1&children=10&afterChild=${body.nextChild}` + `/api/endpoint/resolver/${distantChildEntityID}/children?children=2&afterChild=${body.nextChild}` ) .expect(200)); - expect(body.childNodes.length).to.be(1); - verifyChildren(body.childNodes, tree, 1, 1); - expect(body.childNodes[0].nextChild).to.be(null); + expect(body.childNodes.length).to.eql(0); + expect(body.nextChild).to.be(null); + }); + + it('gets all children in two queries', async () => { + // should get all the children of the origin + let { body }: { body: ResolverChildren } = await supertest + .get(`/api/endpoint/resolver/${tree.origin.id}/children?children=3`) + .expect(200); + expect(body.childNodes.length).to.eql(3); + verifyChildren(body.childNodes, tree); + expect(body.nextChild).to.not.be(null); + const firstNodes = [...body.childNodes]; - // get the 1 child of the origin of the tree's last child ({ body } = await supertest .get( - `/api/endpoint/resolver/${firstChild.entityID}/children?generations=1&children=10&afterChild=${firstChild.nextChild}` + `/api/endpoint/resolver/${tree.origin.id}/children?children=10&afterChild=${body.nextChild}` ) .expect(200)); - expect(body.childNodes.length).to.be(1); - verifyChildren(body.childNodes, tree, 1, 1); - expect(body.childNodes[0].nextChild).to.be(null); + expect(body.childNodes.length).to.eql(9); + // put all the results together and we should have all the children + verifyChildren([...firstNodes, ...body.childNodes], tree, 4, 3); + expect(body.nextChild).to.be(null); }); }); }); @@ -703,7 +703,7 @@ export default function resolverAPIIntegrationTests({ getService }: FtrProviderC it('returns a tree', async () => { const { body }: { body: ResolverTree } = await supertest .get( - `/api/endpoint/resolver/${tree.origin.id}?children=100&generations=3&ancestors=5&events=4&alerts=4` + `/api/endpoint/resolver/${tree.origin.id}?children=100&ancestors=5&events=5&alerts=5` ) .expect(200); From 92005dd8efd5b36dec30cb09dee49f9c45a095cd Mon Sep 17 00:00:00 2001 From: Jonathan Buttner Date: Fri, 26 Jun 2020 18:25:44 -0400 Subject: [PATCH 09/16] Adding more comments and fixing type errors --- .../server/endpoint/routes/resolver/queries/base.ts | 6 ++++-- .../server/endpoint/routes/resolver/queries/stats.ts | 5 ++++- .../routes/resolver/utils/ancestry_query_handler.ts | 6 +++--- .../endpoint/routes/resolver/utils/children_helper.ts | 8 ++++---- .../routes/resolver/utils/lifecycle_query_handler.ts | 8 ++++---- .../server/endpoint/routes/resolver/utils/node.ts | 2 +- .../server/endpoint/routes/resolver/utils/tree.test.ts | 2 +- 7 files changed, 21 insertions(+), 16 deletions(-) diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/base.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/base.ts index 0e91dac9e14e1..6a2ff81c2f2cb 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/base.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/base.ts @@ -14,8 +14,10 @@ import { MSearchQuery } from './multi_searcher'; /** * ResolverQuery provides the base structure for queries to retrieve events when building a resolver graph. * - * @param T the structured return type of a resolver query. This represents the type that is returned when translating - * Elasticsearch's SearchResponse response. + * @param T the structured return type of a resolver query. This represents the final return type of the query after handling + * any aggregations. + * @param R the is the type after transforming ES's response. Making this definable let's us set whether it is a resolver event + * or something else. */ export abstract class ResolverQuery implements MSearchQuery { /** diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/stats.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/stats.ts index d19099ffa738c..b8fa409e2ca21 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/stats.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/stats.ts @@ -13,9 +13,12 @@ export interface StatsResult { events: Record; } -interface CategoriesAgg { +interface AggBucket { key: string; doc_count: number; +} + +interface CategoriesAgg extends AggBucket { /** * The reason categories is optional here is because if no data was returned in the query the categories aggregation * will not be defined on the response (because it's a sub aggregation). diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/ancestry_query_handler.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/ancestry_query_handler.ts index 70987aaa01603..555a02fe7cd7a 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/ancestry_query_handler.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/ancestry_query_handler.ts @@ -14,7 +14,7 @@ import { import { ResolverAncestry, ResolverEvent, - LifecycleNode, + ResolverLifecycleNode, } from '../../../../../common/endpoint/types'; import { createAncestry, createLifecycle } from './node'; import { LifecycleQuery } from '../queries/lifecycle'; @@ -33,7 +33,7 @@ export class AncestryQueryHandler implements QueryHandler { private levels: number, indexPattern: string, legacyEndpointID: string | undefined, - originNode: LifecycleNode | undefined + originNode: ResolverLifecycleNode | undefined ) { this.ancestorsToFind = getAncestryAsArray(originNode?.lifecycle[0]).slice(0, levels); this.query = new LifecycleQuery(indexPattern, legacyEndpointID); @@ -46,7 +46,7 @@ export class AncestryQueryHandler implements QueryHandler { } private toMapOfNodes(results: ResolverEvent[]) { - return results.reduce((nodes: Map, event: ResolverEvent) => { + return results.reduce((nodes: Map, event: ResolverEvent) => { const nodeId = entityId(event); let node = nodes.get(nodeId); if (!node) { diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/children_helper.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/children_helper.ts index f51587766c3c9..01e356682ac47 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/children_helper.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/children_helper.ts @@ -37,13 +37,13 @@ export class ChildrenNodesHelper { let rootNextChild = null; if (rootNode) { - rootNextChild = rootNode.nextChild; + rootNextChild = rootNode.nextChild ?? null; } cacheCopy.delete(this.rootID); return { childNodes: Array.from(cacheCopy.values()), - nextChild: rootNextChild || null, + nextChild: rootNextChild, }; } @@ -51,7 +51,7 @@ export class ChildrenNodesHelper { * Get the entity_ids of the nodes that are cached. */ getEntityIDs(): string[] { - const cacheCopy: Map = new Map(this.entityToNodeCache); + const cacheCopy: Map = new Map(this.entityToNodeCache); cacheCopy.delete(this.rootID); return Array.from(cacheCopy.keys()); } @@ -89,7 +89,7 @@ export class ChildrenNodesHelper { addStartEvents(queriedNodes: Set, startEvents: ResolverEvent[]): Set | undefined { let largestAncestryArray = 0; const nodesToQueryNext: Map> = new Map(); - const nonLeafNodes: Set = new Set(); + const nonLeafNodes: Set = new Set(); const isDistantGrandchild = (event: ResolverEvent) => { const ancestry = getAncestryAsArray(event); diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/lifecycle_query_handler.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/lifecycle_query_handler.ts index 165a502a9a4fc..e0e0eb001944b 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/lifecycle_query_handler.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/lifecycle_query_handler.ts @@ -6,7 +6,7 @@ import { SearchResponse } from 'elasticsearch'; import { IScopedClusterClient } from 'src/core/server'; -import { ResolverEvent, LifecycleNode } from '../../../../../common/endpoint/types'; +import { ResolverEvent, ResolverLifecycleNode } from '../../../../../common/endpoint/types'; import { LifecycleQuery } from '../queries/lifecycle'; import { QueryInfo } from '../queries/multi_searcher'; import { SingleQueryHandler } from './fetch'; @@ -15,8 +15,8 @@ import { createLifecycle } from './node'; /** * Retrieve the lifecycle events for a node. */ -export class LifecycleQueryHandler implements SingleQueryHandler { - private lifecycle: LifecycleNode | undefined; +export class LifecycleQueryHandler implements SingleQueryHandler { + private lifecycle: ResolverLifecycleNode | undefined; private readonly query: LifecycleQuery; constructor( private readonly entityID: string, @@ -51,7 +51,7 @@ export class LifecycleQueryHandler implements SingleQueryHandler /** * Get the results from the msearch. */ - getResults(): LifecycleNode | undefined { + getResults(): ResolverLifecycleNode | undefined { return this.lifecycle; } diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/node.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/node.ts index 6717d9a9dbf19..98180885faf05 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/node.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/node.ts @@ -84,7 +84,7 @@ export function createLifecycle( * @param nextChild the cursor for the response */ export function createChildren( - nodes: ChildNode[] = [], + nodes: ResolverChildNode[] = [], nextChild: string | null = null ): ResolverChildren { return { childNodes: nodes, nextChild }; diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/tree.test.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/tree.test.ts index eb80c840783ef..21db11f3affd3 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/tree.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/tree.test.ts @@ -20,7 +20,7 @@ describe('Tree', () => { // transform the generator's array of events into the format expected by the tree class const ancestorInfo: ResolverAncestry = { ancestors: generator - .createAlertEventAncestry(5, 0, 0) + .createAlertEventAncestry({ ancestors: 5, percentTerminated: 0, percentWithRelated: 0 }) .filter((event) => { return event.event.kind === 'event'; }) From 6cdf961649f5785a8563d5b24573e5da4e57e3cf Mon Sep 17 00:00:00 2001 From: Jonathan Buttner Date: Fri, 26 Jun 2020 18:32:31 -0400 Subject: [PATCH 10/16] Fixing resolver test import --- x-pack/test/api_integration/apis/endpoint/resolver.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/test/api_integration/apis/endpoint/resolver.ts b/x-pack/test/api_integration/apis/endpoint/resolver.ts index 89b4ce88b32a7..54dacadcdd26b 100644 --- a/x-pack/test/api_integration/apis/endpoint/resolver.ts +++ b/x-pack/test/api_integration/apis/endpoint/resolver.ts @@ -16,7 +16,7 @@ import { LegacyEndpointEvent, ResolverNodeStats, ResolverRelatedAlerts, - ChildNode, + ResolverChildNode, } from '../../../../plugins/security_solution/common/endpoint/types'; import { parentEntityId } from '../../../../plugins/security_solution/common/endpoint/models/event'; import { FtrProviderContext } from '../../ftr_provider_context'; From 40f553739d73ddd8b82d3ef316aeacea2169502d Mon Sep 17 00:00:00 2001 From: Jonathan Buttner Date: Mon, 29 Jun 2020 09:15:57 -0400 Subject: [PATCH 11/16] Fixing tests and type errors --- .../resolver/utils/children_helper.test.ts | 21 +++++++++++++++++-- .../routes/resolver/utils/tree.test.ts | 2 +- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/children_helper.test.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/children_helper.test.ts index 51c9cef08a466..1d55cb7cfd735 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/children_helper.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/children_helper.test.ts @@ -34,7 +34,16 @@ describe('Children helper', () => { const root = generator.generateEvent(); it('builds the children response structure', () => { - const children = Array.from(generator.descendantsTreeGenerator(root, 3, 3, 0, 0, 0, 100, true)); + const children = Array.from( + generator.descendantsTreeGenerator(root, { + generations: 3, + children: 3, + relatedEvents: 0, + relatedAlerts: 0, + percentTerminated: 100, + alwaysGenMaxChildrenPerNode: true, + }) + ); // because we requested the generator to always return the max children, there will always be at least 2 parents const parents = findParents(children); @@ -66,7 +75,15 @@ describe('Children helper', () => { }); it('builds the children response structure twice', () => { - const children = Array.from(generator.descendantsTreeGenerator(root, 3, 3, 0, 0, 100)); + const children = Array.from( + generator.descendantsTreeGenerator(root, { + generations: 3, + children: 3, + relatedEvents: 0, + relatedAlerts: 0, + percentTerminated: 100, + }) + ); const helper = new ChildrenNodesHelper(root.process.entity_id); helper.addChildren({}, children); helper.getNodes(); diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/tree.test.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/tree.test.ts index eb80c840783ef..21db11f3affd3 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/tree.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/tree.test.ts @@ -20,7 +20,7 @@ describe('Tree', () => { // transform the generator's array of events into the format expected by the tree class const ancestorInfo: ResolverAncestry = { ancestors: generator - .createAlertEventAncestry(5, 0, 0) + .createAlertEventAncestry({ ancestors: 5, percentTerminated: 0, percentWithRelated: 0 }) .filter((event) => { return event.event.kind === 'event'; }) From 1eac63fe971acda8d3d665808f7bd83854830362 Mon Sep 17 00:00:00 2001 From: Jonathan Buttner Date: Mon, 29 Jun 2020 09:24:28 -0400 Subject: [PATCH 12/16] Fixing type errors and tests --- .../server/endpoint/routes/resolver/queries/children.test.ts | 2 +- x-pack/test/api_integration/apis/endpoint/resolver.ts | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/children.test.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/children.test.ts index a4d4cd546ef60..8175764b3a0a2 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/children.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/children.test.ts @@ -25,7 +25,7 @@ describe('Children query', () => { // eslint-disable-next-line @typescript-eslint/no-explicit-any const msearch: any = query.buildMSearch(['1234', '5678']); expect(msearch[0].index).toBe('index-pattern'); - expect(msearch[1].query.bool.filter[0]).toStrictEqual({ + expect(msearch[1].query.bool.filter[0].bool.should[0]).toStrictEqual({ terms: { 'process.parent.entity_id': ['1234', '5678'] }, }); }); diff --git a/x-pack/test/api_integration/apis/endpoint/resolver.ts b/x-pack/test/api_integration/apis/endpoint/resolver.ts index 54dacadcdd26b..d333a39714907 100644 --- a/x-pack/test/api_integration/apis/endpoint/resolver.ts +++ b/x-pack/test/api_integration/apis/endpoint/resolver.ts @@ -16,7 +16,6 @@ import { LegacyEndpointEvent, ResolverNodeStats, ResolverRelatedAlerts, - ResolverChildNode, } from '../../../../plugins/security_solution/common/endpoint/types'; import { parentEntityId } from '../../../../plugins/security_solution/common/endpoint/models/event'; import { FtrProviderContext } from '../../ftr_provider_context'; From 99acb58c7c41a383ae42cf4da4383cd38e7ee626 Mon Sep 17 00:00:00 2001 From: Jonathan Buttner Date: Mon, 29 Jun 2020 12:04:32 -0400 Subject: [PATCH 13/16] Removing useAncestry field --- .../common/endpoint/generate_data.ts | 19 +++++++------------ .../scripts/endpoint/resolver_generator.ts | 8 ++++++++ 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/x-pack/plugins/security_solution/common/endpoint/generate_data.ts b/x-pack/plugins/security_solution/common/endpoint/generate_data.ts index 25b943d2e3af1..b969146b970ce 100644 --- a/x-pack/plugins/security_solution/common/endpoint/generate_data.ts +++ b/x-pack/plugins/security_solution/common/endpoint/generate_data.ts @@ -39,7 +39,6 @@ interface EventOptions { eventCategory?: string | string[]; processName?: string; ancestry?: string[]; - useAncestryArray?: boolean; ancestryArrayLimit?: number; pid?: number; parentPid?: number; @@ -289,7 +288,6 @@ export interface TreeOptions { percentWithRelated?: number; percentTerminated?: number; alwaysGenMaxChildrenPerNode?: boolean; - useAncestryArray?: boolean; ancestryArraySize?: number; } @@ -310,7 +308,6 @@ export function getTreeOptionsWithDef(options?: TreeOptions): TreeOptionDefaults percentWithRelated: options?.percentWithRelated ?? 30, percentTerminated: options?.percentTerminated ?? 100, alwaysGenMaxChildrenPerNode: options?.alwaysGenMaxChildrenPerNode ?? false, - useAncestryArray: options?.useAncestryArray ?? true, ancestryArraySize: options?.ancestryArraySize ?? ANCESTRY_LIMIT, }; } @@ -523,9 +520,13 @@ export class EndpointDocGenerator { * @param options - Allows event field values to be specified */ public generateEvent(options: EventOptions = {}): EndpointEvent { - let ancestry: string[] | undefined; - if (options?.useAncestryArray === true || options?.ancestry !== undefined) { - ancestry = options.ancestry?.slice(0, options.ancestryArrayLimit ?? ANCESTRY_LIMIT) ?? []; + // this will default to an empty array for the ancestry field if options.ancestry isn't included + let ancestry: string[] | undefined = + options.ancestry?.slice(0, options?.ancestryArrayLimit ?? ANCESTRY_LIMIT) ?? []; + + // to disable the ancestry array set ancestryArrayLimit to 0 + if (options?.ancestryArrayLimit !== undefined && options.ancestryArrayLimit <= 0) { + ancestry = undefined; } const processName = options.processName ? options.processName : randomProcessName(); @@ -758,7 +759,6 @@ export class EndpointDocGenerator { const startDate = new Date().getTime(); const root = this.generateEvent({ timestamp: startDate + 1000, - useAncestryArray: opts.useAncestryArray, }); events.push(root); let ancestor = root; @@ -802,7 +802,6 @@ export class EndpointDocGenerator { parentEntityID: root.process.parent?.entity_id, eventCategory: 'process', eventType: 'end', - useAncestryArray: opts.useAncestryArray, }) ); } @@ -814,7 +813,6 @@ export class EndpointDocGenerator { // add the parent to the ancestry array ancestry: [ancestor.process.entity_id, ...(ancestor.process.Ext.ancestry ?? [])], ancestryArrayLimit: opts.ancestryArraySize, - useAncestryArray: opts.useAncestryArray, parentPid: ancestor.process.pid, pid: this.randomN(5000), }); @@ -832,7 +830,6 @@ export class EndpointDocGenerator { eventType: 'end', ancestry: ancestor.process.Ext.ancestry, ancestryArrayLimit: opts.ancestryArraySize, - useAncestryArray: opts.useAncestryArray, }) ); } @@ -908,7 +905,6 @@ export class EndpointDocGenerator { ...(currentState.event.process.Ext.ancestry ?? []), ], ancestryArrayLimit: opts.ancestryArraySize, - useAncestryArray: opts.useAncestryArray, }); maxChildren = this.randomN(opts.children + 1); @@ -932,7 +928,6 @@ export class EndpointDocGenerator { eventType: 'end', ancestry: child.process.Ext.ancestry, ancestryArrayLimit: opts.ancestryArraySize, - useAncestryArray: opts.useAncestryArray, }); } if (this.randomN(100) < opts.percentWithRelated) { diff --git a/x-pack/plugins/security_solution/scripts/endpoint/resolver_generator.ts b/x-pack/plugins/security_solution/scripts/endpoint/resolver_generator.ts index 53fa59060550f..cfe1c741ef3f1 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/resolver_generator.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/resolver_generator.ts @@ -11,6 +11,7 @@ import fetch from 'node-fetch'; import { Client, ClientOptions } from '@elastic/elasticsearch'; import { ResponseError } from '@elastic/elasticsearch/lib/errors'; import { indexHostsAndAlerts } from '../../common/endpoint/index_data'; +import { ANCESTRY_LIMIT } from '../../common/endpoint/generate_data'; main(); @@ -122,6 +123,12 @@ async function main() { type: 'number', default: 3, }, + ancestryArraySize: { + alias: 'ancSize', + describe: 'the upper bound size of the ancestry array, 0 will mark the field as undefined', + type: 'number', + default: ANCESTRY_LIMIT, + }, generations: { alias: 'gen', describe: 'number of child generations to create', @@ -229,6 +236,7 @@ async function main() { percentWithRelated: argv.percentWithRelated, percentTerminated: argv.percentTerminated, alwaysGenMaxChildrenPerNode: argv.maxChildrenPerNode, + ancestryArraySize: argv.ancestryArraySize, } ); console.log(`Creating and indexing documents took: ${new Date().getTime() - startTime}ms`); From 3f9f3eb86ccde7e135b6a871a4e9ed129403aa06 Mon Sep 17 00:00:00 2001 From: Jonathan Buttner Date: Mon, 29 Jun 2020 12:06:36 -0400 Subject: [PATCH 14/16] Fixing test --- .../security_solution/common/endpoint/generate_data.test.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/x-pack/plugins/security_solution/common/endpoint/generate_data.test.ts b/x-pack/plugins/security_solution/common/endpoint/generate_data.test.ts index 1bfd1bd56d508..f6025283577f6 100644 --- a/x-pack/plugins/security_solution/common/endpoint/generate_data.test.ts +++ b/x-pack/plugins/security_solution/common/endpoint/generate_data.test.ts @@ -113,7 +113,6 @@ describe('data generator', () => { percentWithRelated: 100, relatedEvents: 0, relatedAlerts: 0, - useAncestryArray: true, ancestryArraySize: ANCESTRY_LIMIT, }); tree.ancestry.delete(tree.origin.id); @@ -152,7 +151,6 @@ describe('data generator', () => { { category: RelatedEventCategory.Network, count: 1 }, ], relatedAlerts, - useAncestryArray: true, ancestryArraySize: ANCESTRY_LIMIT, }); }); From caa8f8f2328b0a8954a77251c436c1fc7e9f432a Mon Sep 17 00:00:00 2001 From: Jonathan Buttner Date: Mon, 29 Jun 2020 12:18:27 -0400 Subject: [PATCH 15/16] Removing useAncestry field from tests --- x-pack/test/api_integration/apis/endpoint/resolver.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/test/api_integration/apis/endpoint/resolver.ts b/x-pack/test/api_integration/apis/endpoint/resolver.ts index d333a39714907..ace32111005f4 100644 --- a/x-pack/test/api_integration/apis/endpoint/resolver.ts +++ b/x-pack/test/api_integration/apis/endpoint/resolver.ts @@ -246,7 +246,6 @@ export default function resolverAPIIntegrationTests({ getService }: FtrProviderC percentWithRelated: 100, numTrees: 1, alwaysGenMaxChildrenPerNode: true, - useAncestryArray: true, ancestryArraySize: 2, }; From 544e1f300eda3414505bea92db60ba32c9f32fb9 Mon Sep 17 00:00:00 2001 From: Jonathan Buttner Date: Mon, 29 Jun 2020 13:32:35 -0400 Subject: [PATCH 16/16] An empty array will be returned because that's how ES will do it too --- .../common/endpoint/generate_data.test.ts | 24 +++++++++++++++++++ .../common/endpoint/generate_data.ts | 12 +++------- .../common/endpoint/types.ts | 2 +- 3 files changed, 28 insertions(+), 10 deletions(-) diff --git a/x-pack/plugins/security_solution/common/endpoint/generate_data.test.ts b/x-pack/plugins/security_solution/common/endpoint/generate_data.test.ts index f6025283577f6..f64462f71a87b 100644 --- a/x-pack/plugins/security_solution/common/endpoint/generate_data.test.ts +++ b/x-pack/plugins/security_solution/common/endpoint/generate_data.test.ts @@ -101,6 +101,30 @@ describe('data generator', () => { expect(processEvent.process.name).not.toBeNull(); }); + describe('creates events with an empty ancestry array', () => { + let tree: Tree; + beforeEach(() => { + tree = generator.generateTree({ + alwaysGenMaxChildrenPerNode: true, + ancestors: 3, + children: 3, + generations: 3, + percentTerminated: 100, + percentWithRelated: 100, + relatedEvents: 0, + relatedAlerts: 0, + ancestryArraySize: 0, + }); + tree.ancestry.delete(tree.origin.id); + }); + + it('creates all events with an empty ancestry array', () => { + for (const event of tree.allEvents) { + expect(event.process.Ext.ancestry.length).toEqual(0); + } + }); + }); + describe('creates an origin alert when no related alerts are requested', () => { let tree: Tree; beforeEach(() => { diff --git a/x-pack/plugins/security_solution/common/endpoint/generate_data.ts b/x-pack/plugins/security_solution/common/endpoint/generate_data.ts index b969146b970ce..aa3a7b80f1157 100644 --- a/x-pack/plugins/security_solution/common/endpoint/generate_data.ts +++ b/x-pack/plugins/security_solution/common/endpoint/generate_data.ts @@ -390,6 +390,7 @@ export class EndpointDocGenerator { * @param ts - Timestamp to put in the event * @param entityID - entityID of the originating process * @param parentEntityID - optional entityID of the parent process, if it exists + * @param ancestryArray - an array of ancestors for the generated alert */ public generateAlert( ts = new Date().getTime(), @@ -455,9 +456,7 @@ export class EndpointDocGenerator { sha256: 'fake sha256', }, Ext: { - // simulate a finite ancestry array size, the endpoint limits the ancestry array to 20 entries we'll use - // 2 so that the backend can handle that case - ancestry: ancestryArray.slice(0, ANCESTRY_LIMIT), + ancestry: ancestryArray, code_signature: [ { trusted: false, @@ -521,14 +520,9 @@ export class EndpointDocGenerator { */ public generateEvent(options: EventOptions = {}): EndpointEvent { // this will default to an empty array for the ancestry field if options.ancestry isn't included - let ancestry: string[] | undefined = + const ancestry: string[] = options.ancestry?.slice(0, options?.ancestryArrayLimit ?? ANCESTRY_LIMIT) ?? []; - // to disable the ancestry array set ancestryArrayLimit to 0 - if (options?.ancestryArrayLimit !== undefined && options.ancestryArrayLimit <= 0) { - ancestry = undefined; - } - const processName = options.processName ? options.processName : randomProcessName(); const detailRecordForEventType = options.extensions || diff --git a/x-pack/plugins/security_solution/common/endpoint/types.ts b/x-pack/plugins/security_solution/common/endpoint/types.ts index 2efaeef5719b7..42f5f4b220da9 100644 --- a/x-pack/plugins/security_solution/common/endpoint/types.ts +++ b/x-pack/plugins/security_solution/common/endpoint/types.ts @@ -498,7 +498,7 @@ export interface EndpointEvent { * ancestry_array[0] == process.parent.entity_id and ancestry_array[1] == process.parent.parent.entity_id */ Ext: { - ancestry?: string[]; + ancestry: string[]; }; }; user?: {