diff --git a/x-pack/plugins/security_solution/common/endpoint/models/event.ts b/x-pack/plugins/security_solution/common/endpoint/models/event.ts index 98f4b4336a1c8..86cccff957211 100644 --- a/x-pack/plugins/security_solution/common/endpoint/models/event.ts +++ b/x-pack/plugins/security_solution/common/endpoint/models/event.ts @@ -60,6 +60,24 @@ export function ancestryArray(event: ResolverEvent): string[] | undefined { return event.process.Ext.ancestry; } +export function getAncestryAsArray(event: ResolverEvent | undefined): string[] { + if (!event) { + return []; + } + + const ancestors = ancestryArray(event); + if (ancestors) { + return ancestors; + } + + const parentID = parentEntityId(event); + if (parentID) { + return [parentID]; + } + + return []; +} + /** * @param event The event to get the category for */ diff --git a/x-pack/plugins/security_solution/common/endpoint/schema/resolver.ts b/x-pack/plugins/security_solution/common/endpoint/schema/resolver.ts index 398e2710b3253..42cbc2327fc28 100644 --- a/x-pack/plugins/security_solution/common/endpoint/schema/resolver.ts +++ b/x-pack/plugins/security_solution/common/endpoint/schema/resolver.ts @@ -13,7 +13,6 @@ export const validateTree = { params: schema.object({ id: schema.string() }), query: schema.object({ children: schema.number({ defaultValue: 10, min: 0, max: 100 }), - generations: schema.number({ defaultValue: 3, min: 0, max: 3 }), ancestors: schema.number({ defaultValue: 3, min: 0, max: 5 }), events: schema.number({ defaultValue: 100, min: 0, max: 1000 }), alerts: schema.number({ defaultValue: 100, min: 0, max: 1000 }), @@ -66,7 +65,6 @@ export const validateChildren = { params: schema.object({ id: schema.string() }), query: schema.object({ children: schema.number({ defaultValue: 10, min: 1, max: 100 }), - generations: schema.number({ defaultValue: 3, min: 1, max: 3 }), afterChild: schema.maybe(schema.string()), legacyEndpointID: schema.maybe(schema.string()), }), diff --git a/x-pack/plugins/security_solution/common/endpoint/types.ts b/x-pack/plugins/security_solution/common/endpoint/types.ts index 72839a8370495..3ce160cc9a99d 100644 --- a/x-pack/plugins/security_solution/common/endpoint/types.ts +++ b/x-pack/plugins/security_solution/common/endpoint/types.ts @@ -76,12 +76,18 @@ export interface ResolverNodeStats { */ export interface ResolverChildNode extends ResolverLifecycleNode { /** - * A child node's pagination cursor can be null for a couple reasons: - * 1. At the time of querying it could have no children in ES, in which case it will be marked as - * null because we know it does not have children during this query. - * 2. If the max level was reached we do not know if this node has children or not so we'll mark it as null + * nextChild can have 3 different states: + * + * undefined: This indicates that you should not use this node for additional queries. It does not mean that node does + * not have any more direct children. The node could have more direct children but to determine that, use the + * ResolverChildren node's nextChild. + * + * null: Indicates that we have received all the children of the node. There may be more descendants though. + * + * string: Indicates this is a leaf node and it can be used to continue querying for additional descendants + * using this node's entity_id */ - nextChild: string | null; + nextChild?: string | null; } /** @@ -91,7 +97,14 @@ export interface ResolverChildNode extends ResolverLifecycleNode { export interface ResolverChildren { childNodes: ResolverChildNode[]; /** - * This is the children cursor for the origin of a tree. + * nextChild can have 2 different states: + * + * null: Indicates that we have received all the descendants that can be retrieved using this node. To retrieve more + * nodes in the tree use a cursor provided in one of the returned children. If no other cursor exists then the tree + * is complete. + * + * string: Indicates this node has more descendants that can be retrieved, pass this cursor in while using this node's + * entity_id for the request. */ nextChild: string | null; } diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/children.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/children.ts index 74448a324a4ec..9b8cd9fd3edab 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/children.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/children.ts @@ -18,14 +18,14 @@ export function handleChildren( return async (context, req, res) => { const { params: { id }, - query: { children, generations, afterChild, legacyEndpointID: endpointID }, + query: { children, afterChild, legacyEndpointID: endpointID }, } = req; try { const client = context.core.elasticsearch.legacy.client; const fetcher = new Fetcher(client, id, eventsIndexPattern, alertsIndexPattern, endpointID); return res.ok({ - body: await fetcher.children(children, generations, afterChild), + body: await fetcher.children(children, afterChild), }); } catch (err) { log.warn(err); diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/alerts.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/alerts.ts index 95bc612c58a1b..feb4a404b2359 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/alerts.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/alerts.ts @@ -6,13 +6,13 @@ import { SearchResponse } from 'elasticsearch'; import { ResolverEvent } from '../../../../../common/endpoint/types'; import { ResolverQuery } from './base'; -import { PaginationBuilder, PaginatedResults } from '../utils/pagination'; +import { PaginationBuilder } from '../utils/pagination'; import { JsonObject } from '../../../../../../../../src/plugins/kibana_utils/common'; /** * Builds a query for retrieving alerts for a node. */ -export class AlertsQuery extends ResolverQuery { +export class AlertsQuery extends ResolverQuery { constructor( private readonly pagination: PaginationBuilder, indexPattern: string | string[], @@ -38,11 +38,7 @@ export class AlertsQuery extends ResolverQuery { ], }, }, - ...this.pagination.buildQueryFields( - uniquePIDs.length, - 'endgame.serial_event_id', - 'endgame.unique_pid' - ), + ...this.pagination.buildQueryFields('endgame.serial_event_id'), }; } @@ -60,14 +56,11 @@ export class AlertsQuery extends ResolverQuery { ], }, }, - ...this.pagination.buildQueryFields(entityIDs.length, 'event.id', 'process.entity_id'), + ...this.pagination.buildQueryFields('event.id'), }; } - formatResponse(response: SearchResponse): PaginatedResults { - return { - results: ResolverQuery.getResults(response), - totals: PaginationBuilder.getTotals(response.aggregations), - }; + formatResponse(response: SearchResponse): ResolverEvent[] { + return this.getResults(response); } } diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/base.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/base.ts index 35f8cad01e672..1b6a8f2f83387 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/base.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/base.ts @@ -14,10 +14,12 @@ import { MSearchQuery } from './multi_searcher'; /** * ResolverQuery provides the base structure for queries to retrieve events when building a resolver graph. * - * @param T the structured return type of a resolver query. This represents the type that is returned when translating - * Elasticsearch's SearchResponse response. + * @param T the structured return type of a resolver query. This represents the final return type of the query after handling + * any aggregations. + * @param R the is the type after transforming ES's response. Making this definable let's us set whether it is a resolver event + * or something else. */ -export abstract class ResolverQuery implements MSearchQuery { +export abstract class ResolverQuery implements MSearchQuery { /** * * @param indexPattern the index pattern to use in the query for finding indices with documents in ES. @@ -50,7 +52,7 @@ export abstract class ResolverQuery implements MSearchQuery { }; } - protected static getResults(response: SearchResponse): ResolverEvent[] { + protected getResults(response: SearchResponse): R[] { return response.hits.hits.map((hit) => hit._source); } @@ -68,19 +70,26 @@ export abstract class ResolverQuery implements MSearchQuery { } /** - * Searches ES for the specified ids. + * Searches ES for the specified ids and format the response. * * @param client a client for searching ES * @param ids a single more multiple unique node ids (e.g. entity_id or unique_pid) */ - async search(client: ILegacyScopedClusterClient, ids: string | string[]): Promise { - const res: SearchResponse = await client.callAsCurrentUser( - 'search', - this.buildSearch(ids) - ); + async searchAndFormat(client: ILegacyScopedClusterClient, ids: string | string[]): Promise { + const res: SearchResponse = await this.search(client, ids); return this.formatResponse(res); } + /** + * Searches ES for the specified ids but do not format the response. + * + * @param client a client for searching ES + * @param ids a single more multiple unique node ids (e.g. entity_id or unique_pid) + */ + async search(client: ILegacyScopedClusterClient, ids: string | string[]) { + return client.callAsCurrentUser('search', this.buildSearch(ids)); + } + /** * Builds a query to search the legacy data format. * diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/children.test.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/children.test.ts index a4d4cd546ef60..8175764b3a0a2 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/children.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/children.test.ts @@ -25,7 +25,7 @@ describe('Children query', () => { // eslint-disable-next-line @typescript-eslint/no-explicit-any const msearch: any = query.buildMSearch(['1234', '5678']); expect(msearch[0].index).toBe('index-pattern'); - expect(msearch[1].query.bool.filter[0]).toStrictEqual({ + expect(msearch[1].query.bool.filter[0].bool.should[0]).toStrictEqual({ terms: { 'process.parent.entity_id': ['1234', '5678'] }, }); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/children.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/children.ts index b7b1a16926a15..7fd3808662baa 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/children.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/children.ts @@ -6,13 +6,13 @@ import { SearchResponse } from 'elasticsearch'; import { ResolverEvent } from '../../../../../common/endpoint/types'; import { ResolverQuery } from './base'; -import { PaginationBuilder, PaginatedResults } from '../utils/pagination'; +import { PaginationBuilder } from '../utils/pagination'; import { JsonObject } from '../../../../../../../../src/plugins/kibana_utils/common'; /** * Builds a query for retrieving descendants of a node. */ -export class ChildrenQuery extends ResolverQuery { +export class ChildrenQuery extends ResolverQuery { constructor( private readonly pagination: PaginationBuilder, indexPattern: string | string[], @@ -53,11 +53,7 @@ export class ChildrenQuery extends ResolverQuery { ], }, }, - ...this.pagination.buildQueryFields( - uniquePIDs.length, - 'endgame.serial_event_id', - 'endgame.unique_ppid' - ), + ...this.pagination.buildQueryFields('endgame.serial_event_id'), }; } @@ -67,7 +63,16 @@ export class ChildrenQuery extends ResolverQuery { bool: { filter: [ { - terms: { 'process.parent.entity_id': entityIDs }, + bool: { + should: [ + { + terms: { 'process.parent.entity_id': entityIDs }, + }, + { + terms: { 'process.Ext.ancestry': entityIDs }, + }, + ], + }, }, { term: { 'event.category': 'process' }, @@ -81,14 +86,11 @@ export class ChildrenQuery extends ResolverQuery { ], }, }, - ...this.pagination.buildQueryFields(entityIDs.length, 'event.id', 'process.parent.entity_id'), + ...this.pagination.buildQueryFields('event.id'), }; } - formatResponse(response: SearchResponse): PaginatedResults { - return { - results: ResolverQuery.getResults(response), - totals: PaginationBuilder.getTotals(response.aggregations), - }; + formatResponse(response: SearchResponse): ResolverEvent[] { + return this.getResults(response); } } diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/events.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/events.ts index ec65e30d1d5d4..abc86826e77dd 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/events.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/events.ts @@ -6,13 +6,13 @@ import { SearchResponse } from 'elasticsearch'; import { ResolverEvent } from '../../../../../common/endpoint/types'; import { ResolverQuery } from './base'; -import { PaginationBuilder, PaginatedResults } from '../utils/pagination'; +import { PaginationBuilder } from '../utils/pagination'; import { JsonObject } from '../../../../../../../../src/plugins/kibana_utils/common'; /** * Builds a query for retrieving related events for a node. */ -export class EventsQuery extends ResolverQuery { +export class EventsQuery extends ResolverQuery { constructor( private readonly pagination: PaginationBuilder, indexPattern: string | string[], @@ -45,11 +45,7 @@ export class EventsQuery extends ResolverQuery { ], }, }, - ...this.pagination.buildQueryFields( - uniquePIDs.length, - 'endgame.serial_event_id', - 'endgame.unique_pid' - ), + ...this.pagination.buildQueryFields('endgame.serial_event_id'), }; } @@ -74,14 +70,11 @@ export class EventsQuery extends ResolverQuery { ], }, }, - ...this.pagination.buildQueryFields(entityIDs.length, 'event.id', 'process.entity_id'), + ...this.pagination.buildQueryFields('event.id'), }; } - formatResponse(response: SearchResponse): PaginatedResults { - return { - results: ResolverQuery.getResults(response), - totals: PaginationBuilder.getTotals(response.aggregations), - }; + formatResponse(response: SearchResponse): ResolverEvent[] { + return this.getResults(response); } } diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/lifecycle.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/lifecycle.ts index 93910293b00af..0b5728958e91f 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/lifecycle.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/lifecycle.ts @@ -60,6 +60,6 @@ export class LifecycleQuery extends ResolverQuery { } formatResponse(response: SearchResponse): ResolverEvent[] { - return ResolverQuery.getResults(response); + return this.getResults(response); } } diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/multi_searcher.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/multi_searcher.ts index f873ab3019f64..02dbd92d9252b 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/multi_searcher.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/multi_searcher.ts @@ -5,7 +5,7 @@ */ import { ILegacyScopedClusterClient } from 'kibana/server'; -import { MSearchResponse } from 'elasticsearch'; +import { MSearchResponse, SearchResponse } from 'elasticsearch'; import { ResolverEvent } from '../../../../../common/endpoint/types'; import { JsonObject } from '../../../../../../../../src/plugins/kibana_utils/common'; @@ -34,6 +34,10 @@ export interface QueryInfo { * one or many unique identifiers to be searched for in this query */ ids: string | string[]; + /** + * a function to handle the response + */ + handler: (response: SearchResponse) => void; } /** @@ -57,10 +61,10 @@ export class MultiSearcher { throw new Error('No queries provided to MultiSearcher'); } - let searchQuery: JsonObject[] = []; - queries.forEach( - (info) => (searchQuery = [...searchQuery, ...info.query.buildMSearch(info.ids)]) - ); + const searchQuery: JsonObject[] = []; + for (const info of queries) { + searchQuery.push(...info.query.buildMSearch(info.ids)); + } const res: MSearchResponse = await this.client.callAsCurrentUser('msearch', { body: searchQuery, }); @@ -72,6 +76,8 @@ export class MultiSearcher { if (res.responses.length !== queries.length) { throw new Error(`Responses length was: ${res.responses.length} expected ${queries.length}`); } - return res.responses; + for (let i = 0; i < queries.length; i++) { + queries[i].handler(res.responses[i]); + } } } diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/stats.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/stats.ts index a728054bef219..b8fa409e2ca21 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/stats.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/stats.ts @@ -7,13 +7,17 @@ import { SearchResponse } from 'elasticsearch'; import { ResolverQuery } from './base'; import { ResolverEvent, EventStats } from '../../../../../common/endpoint/types'; import { JsonObject } from '../../../../../../../../src/plugins/kibana_utils/common'; -import { AggBucket } from '../utils/pagination'; export interface StatsResult { alerts: Record; events: Record; } +interface AggBucket { + key: string; + doc_count: number; +} + interface CategoriesAgg extends AggBucket { /** * The reason categories is optional here is because if no data was returned in the query the categories aggregation diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree.ts index 181fb8c3df3f9..33011078ee823 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree.ts @@ -21,7 +21,6 @@ export function handleTree( params: { id }, query: { children, - generations, ancestors, events, alerts, @@ -37,7 +36,7 @@ export function handleTree( const fetcher = new Fetcher(client, id, eventsIndexPattern, alertsIndexPattern, endpointID); const [childrenNodes, ancestry, relatedEvents, relatedAlerts] = await Promise.all([ - fetcher.children(children, generations, afterChild), + fetcher.children(children, afterChild), fetcher.ancestors(ancestors), fetcher.events(events, afterEvent), fetcher.alerts(alerts, afterAlert), diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/alerts_query_handler.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/alerts_query_handler.ts new file mode 100644 index 0000000000000..ae17cf4c3a562 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/alerts_query_handler.ts @@ -0,0 +1,83 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { SearchResponse } from 'elasticsearch'; +import { ILegacyScopedClusterClient } from 'kibana/server'; +import { ResolverRelatedAlerts, ResolverEvent } from '../../../../../common/endpoint/types'; +import { createRelatedAlerts } from './node'; +import { AlertsQuery } from '../queries/alerts'; +import { PaginationBuilder } from './pagination'; +import { QueryInfo } from '../queries/multi_searcher'; +import { SingleQueryHandler } from './fetch'; + +/** + * Requests related alerts for the given node. + */ +export class RelatedAlertsQueryHandler implements SingleQueryHandler { + private relatedAlerts: ResolverRelatedAlerts | undefined; + private readonly query: AlertsQuery; + constructor( + private readonly limit: number, + private readonly entityID: string, + after: string | undefined, + indexPattern: string, + legacyEndpointID: string | undefined + ) { + this.query = new AlertsQuery( + PaginationBuilder.createBuilder(limit, after), + indexPattern, + legacyEndpointID + ); + } + + private handleResponse = (response: SearchResponse) => { + const results = this.query.formatResponse(response); + this.relatedAlerts = createRelatedAlerts( + this.entityID, + results, + PaginationBuilder.buildCursorRequestLimit(this.limit, results) + ); + }; + + /** + * Builds a QueryInfo object that defines the related alerts to search for and how to handle the response. + * + * This will return undefined onces the results have been retrieved from ES. + */ + nextQuery(): QueryInfo | undefined { + if (this.getResults()) { + return; + } + + return { + query: this.query, + ids: this.entityID, + handler: this.handleResponse, + }; + } + + /** + * Get the results after an msearch. + */ + getResults() { + return this.relatedAlerts; + } + + /** + * Perform a regular search and return the results. + * + * @param client the elasticsearch client + */ + async search(client: ILegacyScopedClusterClient) { + const results = this.getResults(); + if (results) { + return results; + } + + this.handleResponse(await this.query.search(client, this.entityID)); + return this.getResults() ?? createRelatedAlerts(this.entityID); + } +} diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/ancestry_query_handler.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/ancestry_query_handler.ts new file mode 100644 index 0000000000000..9bf16dac791d7 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/ancestry_query_handler.ts @@ -0,0 +1,130 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { SearchResponse } from 'elasticsearch'; +import { ILegacyScopedClusterClient } from 'kibana/server'; +import { + parentEntityId, + entityId, + getAncestryAsArray, +} from '../../../../../common/endpoint/models/event'; +import { + ResolverAncestry, + ResolverEvent, + ResolverLifecycleNode, +} from '../../../../../common/endpoint/types'; +import { createAncestry, createLifecycle } from './node'; +import { LifecycleQuery } from '../queries/lifecycle'; +import { QueryInfo } from '../queries/multi_searcher'; +import { QueryHandler } from './fetch'; + +/** + * Retrieve the ancestry portion of a resolver tree. + */ +export class AncestryQueryHandler implements QueryHandler { + private readonly ancestry: ResolverAncestry = createAncestry(); + private ancestorsToFind: string[]; + private readonly query: LifecycleQuery; + + constructor( + private levels: number, + indexPattern: string, + legacyEndpointID: string | undefined, + originNode: ResolverLifecycleNode | undefined + ) { + this.ancestorsToFind = getAncestryAsArray(originNode?.lifecycle[0]).slice(0, levels); + this.query = new LifecycleQuery(indexPattern, legacyEndpointID); + + // add the origin node to the response if it exists + if (originNode) { + this.ancestry.ancestors.push(originNode); + this.ancestry.nextAncestor = parentEntityId(originNode.lifecycle[0]) || null; + } + } + + private toMapOfNodes(results: ResolverEvent[]) { + return results.reduce((nodes: Map, event: ResolverEvent) => { + const nodeId = entityId(event); + let node = nodes.get(nodeId); + if (!node) { + node = createLifecycle(nodeId, []); + } + + node.lifecycle.push(event); + return nodes.set(nodeId, node); + }, new Map()); + } + + private setNoMore() { + this.ancestry.nextAncestor = null; + this.ancestorsToFind = []; + this.levels = 0; + } + + private handleResponse = (searchResp: SearchResponse) => { + const results = this.query.formatResponse(searchResp); + if (results.length === 0) { + this.setNoMore(); + return; + } + + // bucket the start and end events together for a single node + const ancestryNodes = this.toMapOfNodes(results); + + // the order of this array is going to be weird, it will look like this + // [furthest grandparent...closer grandparent, next recursive call furthest grandparent...closer grandparent] + this.ancestry.ancestors.push(...ancestryNodes.values()); + this.ancestry.nextAncestor = parentEntityId(results[0]) || null; + this.levels = this.levels - ancestryNodes.size; + // the results come back in ascending order on timestamp so the first entry in the + // results should be the further ancestor (most distant grandparent) + this.ancestorsToFind = getAncestryAsArray(results[0]).slice(0, this.levels); + }; + + /** + * Returns whether there are more results to retrieve based on the limit that is passed in and the results that + * have already been received from ES. + */ + hasMore(): boolean { + return this.levels > 0 && this.ancestorsToFind.length > 0; + } + + /** + * Get a query info for retrieving the next set of results. + */ + nextQuery(): QueryInfo | undefined { + if (this.hasMore()) { + return { + query: this.query, + ids: this.ancestorsToFind, + handler: this.handleResponse, + }; + } + } + + /** + * Return the results after using msearch to find them. + */ + getResults() { + return this.ancestry; + } + + /** + * Perform a regular search and return the results. + * + * @param client the elasticsearch client. + */ + async search(client: ILegacyScopedClusterClient) { + while (this.hasMore()) { + const info = this.nextQuery(); + if (!info) { + break; + } + this.handleResponse(await this.query.search(client, info.ids)); + } + return this.getResults(); + } +} diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/children_helper.test.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/children_helper.test.ts index 1d55cb7cfd735..ca5b5aef0f651 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/children_helper.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/children_helper.test.ts @@ -3,95 +3,195 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -import _ from 'lodash'; - -import { EndpointDocGenerator } from '../../../../../common/endpoint/generate_data'; +import { + EndpointDocGenerator, + Tree, + Event, + TreeNode, +} from '../../../../../common/endpoint/generate_data'; import { ChildrenNodesHelper } from './children_helper'; -import { eventId, entityId, parentEntityId } from '../../../../../common/endpoint/models/event'; -import { ResolverEvent, ResolverChildren } from '../../../../../common/endpoint/types'; - -function findParents(events: ResolverEvent[]): ResolverEvent[] { - const cache = _.groupBy(events, entityId); +import { eventId, isProcessStart } from '../../../../../common/endpoint/models/event'; - const parents: ResolverEvent[] = []; - Object.values(cache).forEach((lifecycle) => { - const parentNode = cache[parentEntityId(lifecycle[0])!]; - if (parentNode) { - parents.push(parentNode[0]); +function getStartEvents(events: Event[]): Event[] { + const startEvents: Event[] = []; + for (const event of events) { + if (isProcessStart(event)) { + startEvents.push(event); } - }); - return parents; + } + return startEvents; } -function findNode(tree: ResolverChildren, id: string) { - return tree.childNodes.find((node) => { - return node.entityID === id; - }); +function getAllChildrenEvents(tree: Tree) { + const children: Event[] = []; + for (const child of tree.children.values()) { + children.push(...child.lifecycle); + } + return children; +} + +function getStartEventsFromLevels(levels: Array>) { + const startEvents: Event[] = []; + for (const level of levels) { + for (const node of level.values()) { + startEvents.push(...getStartEvents(node.lifecycle)); + } + } + + return startEvents; } describe('Children helper', () => { const generator = new EndpointDocGenerator(); - const root = generator.generateEvent(); + + let tree: Tree; + let helper: ChildrenNodesHelper; + let childrenEvents: Event[]; + let childrenStartEvents: Event[]; + beforeEach(() => { + tree = generator.generateTree({ + children: 3, + alwaysGenMaxChildrenPerNode: true, + generations: 3, + percentTerminated: 100, + ancestryArraySize: 2, + }); + helper = new ChildrenNodesHelper(tree.origin.id, tree.children.size); + childrenEvents = getAllChildrenEvents(tree); + childrenStartEvents = getStartEvents(childrenEvents); + }); + + it('returns the correct entity_ids', () => { + helper.addLifecycleEvents(childrenEvents); + expect(helper.getEntityIDs()).toEqual(Array.from(tree.children.keys())); + }); + + it('returns the correct number of nodes', () => { + helper.addLifecycleEvents(childrenEvents); + expect(helper.getNumNodes()).toEqual(tree.children.size); + }); + + it('marks the query nodes as null', () => { + // +1 indicates that we haven't received all the results so it should create a pagination cursor for the + // queried node (aka the origin that we're passing in) + helper = new ChildrenNodesHelper(tree.origin.id, tree.children.size + 1); + + const nextQuery = helper.addStartEvents(new Set([tree.origin.id]), childrenStartEvents); + helper.addStartEvents(nextQuery!, []); + const nodes = helper.getNodes(); + expect(nodes.nextChild).toBeNull(); + for (const node of nodes.childNodes) { + expect(node.nextChild).toBeNull(); + } + }); + + it('returns undefined when the limit is reached', () => { + helper = new ChildrenNodesHelper(tree.origin.id, tree.children.size - 1); + + expect(helper.addStartEvents(new Set([tree.origin.id]), childrenStartEvents)).toBeUndefined(); + }); + + it('handles multiple additions of start events', () => { + // + 1 indicates that we got everything that ES had + helper = new ChildrenNodesHelper(tree.origin.id, childrenStartEvents.length + 1); + + const level1And2 = getStartEventsFromLevels(tree.childrenLevels.slice(0, 2)); + let nextQuery = helper.addStartEvents(new Set([tree.origin.id]), level1And2); + expect(nextQuery?.size).toEqual(tree.childrenLevels[1].size); + for (const node of tree.childrenLevels[1].values()) { + expect(nextQuery?.has(node.id)).toBeTruthy(); + } + + const level3 = getStartEventsFromLevels(tree.childrenLevels.slice(2, 3)); + nextQuery = helper.addStartEvents(nextQuery!, level3); + expect(nextQuery).toBeUndefined(); + const nodes = helper.getNodes(); + expect(nodes.nextChild).toBeNull(); + for (const node of nodes.childNodes) { + expect(node.nextChild).toBeNull(); + } + }); + + it('handles an empty set', () => { + helper = new ChildrenNodesHelper(tree.origin.id, 1); + + const nextQuery = helper.addStartEvents(new Set([tree.origin.id]), []); + expect(nextQuery).toBeUndefined(); + const nodes = helper.getNodes(); + expect(nodes.nextChild).toBeNull(); + expect(nodes.childNodes.length).toEqual(0); + }); + + it('handles an empty set after multiple additions', () => { + // + 1 indicates that we got everything that ES had + helper = new ChildrenNodesHelper(tree.origin.id, childrenStartEvents.length + 1); + + const level1And2 = getStartEventsFromLevels(tree.childrenLevels.slice(0, 2)); + let nextQuery = helper.addStartEvents(new Set([tree.origin.id]), level1And2); + + nextQuery = helper.addStartEvents(nextQuery!, []); + expect(nextQuery).toBeUndefined(); + const nodes = helper.getNodes(); + expect(nodes.nextChild).toBeNull(); + for (const node of nodes.childNodes) { + expect(node.nextChild).toBeNull(); + } + }); + + it('non leaf nodes are set to undefined by default', () => { + // + 1 indicates that we got everything that ES had + helper = new ChildrenNodesHelper(tree.origin.id, childrenStartEvents.length + 1); + const level1And2 = getStartEventsFromLevels(tree.childrenLevels.slice(0, 2)); + helper.addStartEvents(new Set([tree.origin.id]), level1And2); + const nodes = helper.getNodes(); + expect(nodes.nextChild).toBeNull(); + for (const node of nodes.childNodes) { + if (tree.childrenLevels[0].has(node.entityID)) { + expect(node.nextChild).toBeNull(); + } else { + expect(node.nextChild).toBeUndefined(); + } + } + }); + + it('returns the leaf nodes', () => { + helper = new ChildrenNodesHelper(tree.origin.id, tree.children.size + 1); + + const nextQuery = helper.addStartEvents(new Set([tree.origin.id]), childrenStartEvents); + // we're using an ancestry array of 2 so the leaf nodes are at the second level + expect(nextQuery?.size).toEqual(tree.childrenLevels[1].size); + + for (const node of tree.childrenLevels[1].values()) { + expect(nextQuery?.has(node.id)).toBeTruthy(); + } + }); it('builds the children response structure', () => { - const children = Array.from( - generator.descendantsTreeGenerator(root, { - generations: 3, - children: 3, - relatedEvents: 0, - relatedAlerts: 0, - percentTerminated: 100, - alwaysGenMaxChildrenPerNode: true, - }) - ); - - // because we requested the generator to always return the max children, there will always be at least 2 parents - const parents = findParents(children); - - // this represents the aggregation returned from elastic search - // each node in the tree should have 3 children, so if these values are greater than 3 there should be - // pagination cursors created for those children - const totals = { - [root.process.entity_id]: 100, - [entityId(parents[0])]: 10, - [entityId(parents[1])]: 0, - }; - - const helper = new ChildrenNodesHelper(root.process.entity_id); - helper.addChildren(totals, children); - const tree = helper.getNodes(); - expect(tree.nextChild).not.toBeNull(); - - let parent = findNode(tree, entityId(parents[0])); - expect(parent?.nextChild).not.toBeNull(); - parent = findNode(tree, entityId(parents[1])); - expect(parent?.nextChild).toBeNull(); - - tree.childNodes.forEach((node) => { + helper.addStartEvents(new Set([tree.origin.id]), childrenStartEvents); + helper.addLifecycleEvents(childrenEvents); + const childrenNodes = helper.getNodes(); + + // since we got all the nodes all the nextChild cursors should be null + for (const node of childrenNodes.childNodes) { + expect(node.nextChild).toBeUndefined(); + } + expect(childrenNodes.nextChild).not.toBeNull(); + + childrenNodes.childNodes.forEach((node) => { node.lifecycle.forEach((event) => { - expect(children.find((child) => child.event.id === eventId(event))).toEqual(event); + expect(childrenEvents.find((child) => child.event.id === eventId(event))).toEqual(event); }); }); }); it('builds the children response structure twice', () => { - const children = Array.from( - generator.descendantsTreeGenerator(root, { - generations: 3, - children: 3, - relatedEvents: 0, - relatedAlerts: 0, - percentTerminated: 100, - }) - ); - const helper = new ChildrenNodesHelper(root.process.entity_id); - helper.addChildren({}, children); + helper.addLifecycleEvents(childrenEvents); helper.getNodes(); - const tree = helper.getNodes(); - tree.childNodes.forEach((node) => { + const childrenNodes = helper.getNodes(); + childrenNodes.childNodes.forEach((node) => { node.lifecycle.forEach((event) => { - expect(children.find((child) => child.event.id === eventId(event))).toEqual(event); + expect(childrenEvents.find((child) => child.event.id === eventId(event))).toEqual(event); }); }); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/children_helper.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/children_helper.ts index e60e5087c30a9..01e356682ac47 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/children_helper.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/children_helper.ts @@ -8,35 +8,36 @@ import { entityId, parentEntityId, isProcessStart, + getAncestryAsArray, } from '../../../../../common/endpoint/models/event'; import { ResolverChildNode, ResolverEvent, ResolverChildren, } from '../../../../../common/endpoint/types'; -import { PaginationBuilder } from './pagination'; import { createChild } from './node'; +import { PaginationBuilder } from './pagination'; /** * This class helps construct the children structure when building a resolver tree. */ export class ChildrenNodesHelper { - private readonly cache: Map = new Map(); + private readonly entityToNodeCache: Map = new Map(); - constructor(private readonly rootID: string) { - this.cache.set(rootID, createChild(rootID)); + constructor(private readonly rootID: string, private readonly limit: number) { + this.entityToNodeCache.set(rootID, createChild(rootID)); } /** * Constructs a ResolverChildren response based on the children that were previously add. */ getNodes(): ResolverChildren { - const cacheCopy: Map = new Map(this.cache); + const cacheCopy: Map = new Map(this.entityToNodeCache); const rootNode = cacheCopy.get(this.rootID); let rootNextChild = null; if (rootNode) { - rootNextChild = rootNode.nextChild; + rootNextChild = rootNode.nextChild ?? null; } cacheCopy.delete(this.rootID); @@ -47,51 +48,131 @@ export class ChildrenNodesHelper { } /** - * Add children to the cache. - * - * @param totals a map of unique node IDs to total number of child nodes - * @param results events from a children query + * Get the entity_ids of the nodes that are cached. + */ + getEntityIDs(): string[] { + const cacheCopy: Map = new Map(this.entityToNodeCache); + cacheCopy.delete(this.rootID); + return Array.from(cacheCopy.keys()); + } + + /** + * Get the number of nodes that have been cached. */ - addChildren(totals: Record, results: ResolverEvent[]) { - const startEventsCache: Map = new Map(); + getNumNodes(): number { + // -1 because the root node is in the cache too + return this.entityToNodeCache.size - 1; + } - results.forEach((event) => { + /** + * Add lifecycle events (start, end, etc) to the cache. + * + * @param lifecycle an array of resolver lifecycle events for different process nodes returned from ES. + */ + addLifecycleEvents(lifecycle: ResolverEvent[]) { + for (const event of lifecycle) { const entityID = entityId(event); - const parentID = parentEntityId(event); - if (!entityID || !parentID) { - return; + if (entityID) { + const cachedChild = this.getOrCreateChildNode(entityID); + cachedChild.lifecycle.push(event); } + } + } - let cachedChild = this.cache.get(entityID); - if (!cachedChild) { - cachedChild = createChild(entityID); - this.cache.set(entityID, cachedChild); - } - cachedChild.lifecycle.push(event); + /** + * Add the start events for the nodes received from ES. Pagination cursors will be constructed based on the + * request limit and results returned. + * + * @param queriedNodes the entity_ids of the nodes that returned these start events + * @param startEvents an array of start events returned by ES + */ + addStartEvents(queriedNodes: Set, startEvents: ResolverEvent[]): Set | undefined { + let largestAncestryArray = 0; + const nodesToQueryNext: Map> = new Map(); + const nonLeafNodes: Set = new Set(); - if (isProcessStart(event)) { - let startEvents = startEventsCache.get(parentID); - if (startEvents === undefined) { - startEvents = []; - startEventsCache.set(parentID, startEvents); + const isDistantGrandchild = (event: ResolverEvent) => { + const ancestry = getAncestryAsArray(event); + return ancestry.length > 0 && queriedNodes.has(ancestry[ancestry.length - 1]); + }; + + for (const event of startEvents) { + const parentID = parentEntityId(event); + const entityID = entityId(event); + if (parentID && entityID && isProcessStart(event)) { + // don't actually add the start event to the node, because that'll be done in + // a different call + const childNode = this.getOrCreateChildNode(entityID); + + const ancestry = getAncestryAsArray(event); + // This is to handle the following unlikely but possible scenario: + // if an alert was generated by the kernel process (parent process of all other processes) then + // the direct children of that process would only have an ancestry array of [parent_kernel], a single value in the array. + // The children of those children would have two values in their array [direct parent, parent_kernel] + // we need to determine which nodes are the most distant grandchildren of the queriedNodes because those should + // be used for the next query if more nodes should be retrieved. To generally determine the most distant grandchildren + // we can use the last entry in the ancestry array because of its ordering. The problem with that is in the scenario above + // the direct children of parent_kernel will also meet that criteria even though they are not actually the most + // distant grandchildren. To get around that issue we'll bucket all the nodes by the size of their ancestry array + // and then only return the nodes in the largest bucket because those should be the most distant grandchildren + // from the queried nodes that were passed in. + if (ancestry.length > largestAncestryArray) { + largestAncestryArray = ancestry.length; + } + + // a grandchild must have an array of > 0 and have it's last parent be in the set of previously queried nodes + // this is one of the furthest descendants from the queried nodes + if (isDistantGrandchild(event)) { + let levelOfNodes = nodesToQueryNext.get(ancestry.length); + if (!levelOfNodes) { + levelOfNodes = new Set(); + nodesToQueryNext.set(ancestry.length, levelOfNodes); + } + levelOfNodes.add(entityID); + } else { + nonLeafNodes.add(childNode); } - startEvents.push(event); } - }); + } + + // we may not have received all the possible nodes so mark pagination for the query nodes + // we won't know if the non leaf nodes (non query nodes) have additional children so don't mark them + if (this.limit <= this.getNumNodes()) { + this.setPaginationForNodes(queriedNodes, startEvents); + return; + } + + // the non leaf nodes have received all their children so mark them as finished + for (const nonLeaf of nonLeafNodes.values()) { + nonLeaf.nextChild = null; + } - this.addChildrenPagination(startEventsCache, totals); + // we've received all the descendants of the previously queried node that we can get using it's ancestry array + // so mark those nodes as complete + for (const nodeEntityID of queriedNodes.values()) { + const node = this.entityToNodeCache.get(nodeEntityID); + if (node) { + node.nextChild = null; + } + } + return nodesToQueryNext.get(largestAncestryArray); } - private addChildrenPagination( - startEventsCache: Map, - totals: Record - ) { - Object.entries(totals).forEach(([parentID, total]) => { - const parentNode = this.cache.get(parentID); - const childrenStartEvents = startEventsCache.get(parentID); - if (parentNode && childrenStartEvents) { - parentNode.nextChild = PaginationBuilder.buildCursor(total, childrenStartEvents); + private setPaginationForNodes(nodes: Set, startEvents: ResolverEvent[]) { + for (const nodeEntityID of nodes.values()) { + const cachedNode = this.entityToNodeCache.get(nodeEntityID); + if (cachedNode) { + cachedNode.nextChild = PaginationBuilder.buildCursor(startEvents); } - }); + } + } + + private getOrCreateChildNode(entityID: string) { + let cachedChild = this.entityToNodeCache.get(entityID); + if (!cachedChild) { + cachedChild = createChild(entityID); + this.entityToNodeCache.set(entityID, cachedChild); + } + return cachedChild; } } diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/children_lifecycle_query_handler.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/children_lifecycle_query_handler.ts new file mode 100644 index 0000000000000..8aaf809405d63 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/children_lifecycle_query_handler.ts @@ -0,0 +1,71 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { SearchResponse } from 'elasticsearch'; +import { ILegacyScopedClusterClient } from 'kibana/server'; +import { ResolverEvent, ResolverChildren } from '../../../../../common/endpoint/types'; +import { LifecycleQuery } from '../queries/lifecycle'; +import { QueryInfo } from '../queries/multi_searcher'; +import { SingleQueryHandler } from './fetch'; +import { ChildrenNodesHelper } from './children_helper'; +import { createChildren } from './node'; + +/** + * Returns the children of a resolver tree. + */ +export class ChildrenLifecycleQueryHandler implements SingleQueryHandler { + private lifecycle: ResolverChildren | undefined; + private readonly query: LifecycleQuery; + constructor( + private readonly childrenHelper: ChildrenNodesHelper, + indexPattern: string, + legacyEndpointID: string | undefined + ) { + this.query = new LifecycleQuery(indexPattern, legacyEndpointID); + } + + private handleResponse = (response: SearchResponse) => { + this.childrenHelper.addLifecycleEvents(this.query.formatResponse(response)); + this.lifecycle = this.childrenHelper.getNodes(); + }; + + /** + * Get the query for msearch. Once the results are set this will return undefined. + */ + nextQuery(): QueryInfo | undefined { + if (this.getResults()) { + return; + } + + return { + query: this.query, + ids: this.childrenHelper.getEntityIDs(), + handler: this.handleResponse, + }; + } + + /** + * Return the results from the search. + */ + getResults(): ResolverChildren | undefined { + return this.lifecycle; + } + + /** + * Perform a regular search and return the results. + * + * @param client the elasticsearch client + */ + async search(client: ILegacyScopedClusterClient) { + const results = this.getResults(); + if (results) { + return results; + } + + this.handleResponse(await this.query.search(client, this.childrenHelper.getEntityIDs())); + return this.getResults() || createChildren(); + } +} diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/children_start_query_handler.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/children_start_query_handler.ts new file mode 100644 index 0000000000000..1c74184720793 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/children_start_query_handler.ts @@ -0,0 +1,109 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { SearchResponse } from 'elasticsearch'; +import { ILegacyScopedClusterClient } from 'kibana/server'; +import { ResolverEvent } from '../../../../../common/endpoint/types'; +import { ChildrenQuery } from '../queries/children'; +import { QueryInfo } from '../queries/multi_searcher'; +import { QueryHandler } from './fetch'; +import { ChildrenNodesHelper } from './children_helper'; +import { PaginationBuilder } from './pagination'; + +/** + * Retrieve the start lifecycle events for the children of a resolver tree. + * + * If using msearch you should loop over hasMore() because the results are limited to the size of the ancestry array. + */ +export class ChildrenStartQueryHandler implements QueryHandler { + private readonly childrenHelper: ChildrenNodesHelper; + private limitLeft: number; + private query: ChildrenQuery; + private nodesToQuery: Set; + + constructor( + private readonly limit: number, + entityID: string, + after: string | undefined, + private readonly indexPattern: string, + private readonly legacyEndpointID: string | undefined + ) { + this.query = new ChildrenQuery( + PaginationBuilder.createBuilder(limit, after), + indexPattern, + legacyEndpointID + ); + this.childrenHelper = new ChildrenNodesHelper(entityID, this.limit); + this.limitLeft = this.limit; + this.nodesToQuery = new Set([entityID]); + } + + private setNoMore() { + this.nodesToQuery = new Set(); + this.limitLeft = 0; + } + + private handleResponse = (response: SearchResponse) => { + const results = this.query.formatResponse(response); + this.nodesToQuery = this.childrenHelper.addStartEvents(this.nodesToQuery, results) ?? new Set(); + + if (results.length === 0) { + this.setNoMore(); + return; + } + + this.limitLeft = this.limit - this.childrenHelper.getNumNodes(); + this.query = new ChildrenQuery( + PaginationBuilder.createBuilder(this.limitLeft), + this.indexPattern, + this.legacyEndpointID + ); + }; + + /** + * Check if there are more results to retrieve based on the limit that was passed in. + */ + hasMore(): boolean { + return this.limitLeft > 0 && this.nodesToQuery.size > 0; + } + + /** + * Get a query to retrieve the next set of results. + */ + nextQuery(): QueryInfo | undefined { + if (this.hasMore()) { + return { + query: this.query, + // This should never be undefined because the check above + ids: Array.from(this.nodesToQuery.values()), + handler: this.handleResponse, + }; + } + } + + /** + * Get the cached results from the ES responses. + */ + getResults(): ChildrenNodesHelper { + return this.childrenHelper; + } + + /** + * Perform a regular search and return the helper. + * + * @param client the elasticsearch client + */ + async search(client: ILegacyScopedClusterClient) { + while (this.hasMore()) { + const info = this.nextQuery(); + if (!info) { + break; + } + this.handleResponse(await this.query.search(client, info.ids)); + } + return this.getResults(); + } +} diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/events_query_handler.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/events_query_handler.ts new file mode 100644 index 0000000000000..849dbc25fe4db --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/events_query_handler.ts @@ -0,0 +1,81 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { SearchResponse } from 'elasticsearch'; +import { ILegacyScopedClusterClient } from 'kibana/server'; +import { ResolverRelatedEvents, ResolverEvent } from '../../../../../common/endpoint/types'; +import { createRelatedEvents } from './node'; +import { EventsQuery } from '../queries/events'; +import { PaginationBuilder } from './pagination'; +import { QueryInfo } from '../queries/multi_searcher'; +import { SingleQueryHandler } from './fetch'; + +/** + * This retrieves the related events for the origin node of a resolver tree. + */ +export class RelatedEventsQueryHandler implements SingleQueryHandler { + private relatedEvents: ResolverRelatedEvents | undefined; + private readonly query: EventsQuery; + constructor( + private readonly limit: number, + private readonly entityID: string, + after: string | undefined, + indexPattern: string, + legacyEndpointID: string | undefined + ) { + this.query = new EventsQuery( + PaginationBuilder.createBuilder(limit, after), + indexPattern, + legacyEndpointID + ); + } + + private handleResponse = (response: SearchResponse) => { + const results = this.query.formatResponse(response); + this.relatedEvents = createRelatedEvents( + this.entityID, + results, + PaginationBuilder.buildCursorRequestLimit(this.limit, results) + ); + }; + + /** + * Get a query to use in a msearch. + */ + nextQuery(): QueryInfo | undefined { + if (this.getResults()) { + return; + } + + return { + query: this.query, + ids: this.entityID, + handler: this.handleResponse, + }; + } + + /** + * Get the results after an msearch. + */ + getResults() { + return this.relatedEvents; + } + + /** + * Perform a normal search and return the related events results. + * + * @param client the elasticsearch client + */ + async search(client: ILegacyScopedClusterClient) { + const results = this.getResults(); + if (results) { + return results; + } + + this.handleResponse(await this.query.search(client, this.entityID)); + return this.getResults() ?? createRelatedEvents(this.entityID); + } +} diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/fetch.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/fetch.ts index 1a532c54c7d5d..feb165c308a91 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/fetch.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/fetch.ts @@ -11,22 +11,61 @@ import { ResolverAncestry, ResolverRelatedAlerts, ResolverLifecycleNode, - ResolverEvent, } from '../../../../../common/endpoint/types'; -import { - entityId, - ancestryArray, - parentEntityId, -} from '../../../../../common/endpoint/models/event'; -import { PaginationBuilder } from './pagination'; import { Tree } from './tree'; import { LifecycleQuery } from '../queries/lifecycle'; -import { ChildrenQuery } from '../queries/children'; -import { EventsQuery } from '../queries/events'; import { StatsQuery } from '../queries/stats'; -import { createAncestry, createRelatedEvents, createLifecycle, createRelatedAlerts } from './node'; -import { ChildrenNodesHelper } from './children_helper'; -import { AlertsQuery } from '../queries/alerts'; +import { createLifecycle } from './node'; +import { MultiSearcher, QueryInfo } from '../queries/multi_searcher'; +import { AncestryQueryHandler } from './ancestry_query_handler'; +import { RelatedEventsQueryHandler } from './events_query_handler'; +import { RelatedAlertsQueryHandler } from './alerts_query_handler'; +import { ChildrenStartQueryHandler } from './children_start_query_handler'; +import { ChildrenLifecycleQueryHandler } from './children_lifecycle_query_handler'; +import { LifecycleQueryHandler } from './lifecycle_query_handler'; + +/** + * The query parameters passed in from the request. These define the limits for the ES requests for retrieving the + * resolver tree. + */ +export interface TreeOptions { + children: number; + ancestors: number; + events: number; + alerts: number; + afterAlert?: string; + afterEvent?: string; + afterChild?: string; +} + +interface QueryBuilder { + nextQuery(): QueryInfo | undefined; +} + +/** + * This interface defines the contract for a query handler that will only be used once in an msearch call. + */ +export interface SingleQueryHandler extends QueryBuilder { + /** + * This method returns the results if the query has been used in an msearch call or undefined if not. + */ + getResults(): T | undefined; + /** + * Do a regular search instead of msearch. + * @param client the elasticsearch client + */ + search(client: ILegacyScopedClusterClient): Promise; +} + +/** + * This interface defines the contract for a query handler that can be used multiple times by msearch. + */ +export interface QueryHandler extends SingleQueryHandler { + /** + * Returns whether additional msearch are required to retrieve the rest of the expected data from ES. + */ + hasMore(): boolean; +} /** * Handles retrieving nodes of a resolver tree. @@ -52,46 +91,138 @@ export class Fetcher { private readonly endpointID?: string ) {} + /** + * This method retrieves the resolver tree starting from the `id` during construction of the class. + * + * @param options the options for retrieving the structure of the tree. + */ + public async tree(options: TreeOptions) { + const addQueryToList = (queryHandler: QueryBuilder, queries: QueryInfo[]) => { + const queryInfo = queryHandler.nextQuery(); + if (queryInfo !== undefined) { + queries.push(queryInfo); + } + }; + + const originHandler = new LifecycleQueryHandler( + this.id, + this.eventsIndexPattern, + this.endpointID + ); + + const eventsHandler = new RelatedEventsQueryHandler( + options.events, + this.id, + options.afterEvent, + this.eventsIndexPattern, + this.endpointID + ); + + const alertsHandler = new RelatedAlertsQueryHandler( + options.alerts, + this.id, + options.afterAlert, + this.alertsIndexPattern, + this.endpointID + ); + + // we need to get the start events first because the API request defines how many nodes to return and we don't want + // to count or limit ourselves based on the other lifecycle events (end, etc) + const childrenHandler = new ChildrenStartQueryHandler( + options.children, + this.id, + options.afterChild, + this.eventsIndexPattern, + this.endpointID + ); + + const msearch = new MultiSearcher(this.client); + + let queries: QueryInfo[] = []; + addQueryToList(eventsHandler, queries); + addQueryToList(alertsHandler, queries); + addQueryToList(childrenHandler, queries); + addQueryToList(originHandler, queries); + + // get the related events, related alerts, the first pass of children start events, and the origin node + // the origin node is needed so we can get the ancestry array for the additional ancestor calls + await msearch.search(queries); + + const ancestryHandler = new AncestryQueryHandler( + options.ancestors, + this.eventsIndexPattern, + this.endpointID, + originHandler.getResults() + ); + + // get the remaining ancestors and children start events + while (ancestryHandler.hasMore() || childrenHandler.hasMore()) { + queries = []; + addQueryToList(ancestryHandler, queries); + addQueryToList(childrenHandler, queries); + await msearch.search(queries); + } + + const childrenTotalsHelper = childrenHandler.getResults(); + + const childrenLifecycleHandler = new ChildrenLifecycleQueryHandler( + childrenTotalsHelper, + this.eventsIndexPattern, + this.endpointID + ); + + // now that we have all the start events get the full lifecycle nodes + childrenLifecycleHandler.search(this.client); + + const tree = new Tree(this.id, { + ancestry: ancestryHandler.getResults(), + relatedEvents: eventsHandler.getResults(), + relatedAlerts: alertsHandler.getResults(), + children: childrenLifecycleHandler.getResults(), + }); + + // add the stats to the tree + return this.stats(tree); + } + /** * Retrieves the ancestor nodes for the resolver tree. * * @param limit upper limit of ancestors to retrieve */ public async ancestors(limit: number): Promise { - const ancestryInfo = createAncestry(); const originNode = await this.getNode(this.id); - if (originNode) { - ancestryInfo.ancestors.push(originNode); - // If the request is only for the origin node then set next to its parent - ancestryInfo.nextAncestor = parentEntityId(originNode.lifecycle[0]) || null; - await this.doAncestors( - // limit the ancestors we're looking for to the number of levels - // the array could be up to length 20 but that could change - Fetcher.getAncestryAsArray(originNode.lifecycle[0]).slice(0, limit), - limit, - ancestryInfo - ); - } - return ancestryInfo; + const ancestryHandler = new AncestryQueryHandler( + limit, + this.eventsIndexPattern, + this.endpointID, + originNode + ); + return ancestryHandler.search(this.client); } /** * Retrieves the children nodes for the resolver tree. * * @param limit the number of children to retrieve for a single level - * @param generations number of levels to return * @param after a cursor to use as the starting point for retrieving children */ - public async children( - limit: number, - generations: number, - after?: string - ): Promise { - const helper = new ChildrenNodesHelper(this.id); - - await this.doChildren(helper, [this.id], limit, generations, after); + public async children(limit: number, after?: string): Promise { + const childrenHandler = new ChildrenStartQueryHandler( + limit, + this.id, + after, + this.eventsIndexPattern, + this.endpointID + ); + const helper = await childrenHandler.search(this.client); + const childrenLifecycleHandler = new ChildrenLifecycleQueryHandler( + helper, + this.eventsIndexPattern, + this.endpointID + ); - return helper.getNodes(); + return childrenLifecycleHandler.search(this.client); } /** @@ -101,7 +232,15 @@ export class Fetcher { * @param after a cursor to use as the starting point for retrieving related events */ public async events(limit: number, after?: string): Promise { - return this.doEvents(limit, after); + const eventsHandler = new RelatedEventsQueryHandler( + limit, + this.id, + after, + this.eventsIndexPattern, + this.endpointID + ); + + return eventsHandler.search(this.client); } /** @@ -111,26 +250,15 @@ export class Fetcher { * @param after a cursor to use as the starting point for retrieving alerts */ public async alerts(limit: number, after?: string): Promise { - const query = new AlertsQuery( - PaginationBuilder.createBuilder(limit, after), + const alertsHandler = new RelatedAlertsQueryHandler( + limit, + this.id, + after, this.alertsIndexPattern, this.endpointID ); - const { totals, results } = await query.search(this.client, this.id); - if (results.length === 0) { - // return an empty set of results - return createRelatedAlerts(this.id); - } - if (!totals[this.id]) { - throw new Error(`Could not find the totals for related events entity_id: ${this.id}`); - } - - return createRelatedAlerts( - this.id, - results, - PaginationBuilder.buildCursor(totals[this.id], results) - ); + return alertsHandler.search(this.client); } /** @@ -145,7 +273,7 @@ export class Fetcher { private async getNode(entityID: string): Promise { const query = new LifecycleQuery(this.eventsIndexPattern, this.endpointID); - const results = await query.search(this.client, entityID); + const results = await query.searchAndFormat(this.client, entityID); if (results.length === 0) { return; } @@ -153,125 +281,13 @@ export class Fetcher { return createLifecycle(entityID, results); } - private static getAncestryAsArray(event: ResolverEvent): string[] { - const ancestors = ancestryArray(event); - if (ancestors) { - return ancestors; - } - - const parentID = parentEntityId(event); - if (parentID) { - return [parentID]; - } - - return []; - } - - private async doAncestors( - ancestors: string[], - levels: number, - ancestorInfo: ResolverAncestry - ): Promise { - if (levels <= 0) { - return; - } - - const query = new LifecycleQuery(this.eventsIndexPattern, this.endpointID); - const results = await query.search(this.client, ancestors); - - if (results.length === 0) { - ancestorInfo.nextAncestor = null; - return; - } - - // bucket the start and end events together for a single node - const ancestryNodes = results.reduce( - (nodes: Map, ancestorEvent: ResolverEvent) => { - const nodeId = entityId(ancestorEvent); - let node = nodes.get(nodeId); - if (!node) { - node = createLifecycle(nodeId, []); - } - - node.lifecycle.push(ancestorEvent); - return nodes.set(nodeId, node); - }, - new Map() - ); - - // the order of this array is going to be weird, it will look like this - // [furthest grandparent...closer grandparent, next recursive call furthest grandparent...closer grandparent] - ancestorInfo.ancestors.push(...ancestryNodes.values()); - ancestorInfo.nextAncestor = parentEntityId(results[0]) || null; - const levelsLeft = levels - ancestryNodes.size; - // the results come back in ascending order on timestamp so the first entry in the - // results should be the further ancestor (most distant grandparent) - const next = Fetcher.getAncestryAsArray(results[0]).slice(0, levelsLeft); - // the ancestry array currently only holds up to 20 values but we can't rely on that so keep recursing - await this.doAncestors(next, levelsLeft, ancestorInfo); - } - - private async doEvents(limit: number, after?: string) { - const query = new EventsQuery( - PaginationBuilder.createBuilder(limit, after), - this.eventsIndexPattern, - this.endpointID - ); - - const { totals, results } = await query.search(this.client, this.id); - if (results.length === 0) { - // return an empty set of results - return createRelatedEvents(this.id); - } - if (!totals[this.id]) { - throw new Error(`Could not find the totals for related events entity_id: ${this.id}`); - } - - return createRelatedEvents( - this.id, - results, - PaginationBuilder.buildCursor(totals[this.id], results) - ); - } - - private async doChildren( - cache: ChildrenNodesHelper, - ids: string[], - limit: number, - levels: number, - after?: string - ) { - if (levels === 0 || ids.length === 0) { - return; - } - - const childrenQuery = new ChildrenQuery( - PaginationBuilder.createBuilder(limit, after), - this.eventsIndexPattern, - this.endpointID - ); - const lifecycleQuery = new LifecycleQuery(this.eventsIndexPattern, this.endpointID); - - const { totals, results } = await childrenQuery.search(this.client, ids); - if (results.length === 0) { - return; - } - - const childIDs = results.map(entityId); - const children = await lifecycleQuery.search(this.client, childIDs); - - cache.addChildren(totals, children); - - await this.doChildren(cache, childIDs, limit, levels - 1); - } - private async doStats(tree: Tree) { const statsQuery = new StatsQuery( [this.eventsIndexPattern, this.alertsIndexPattern], this.endpointID ); const ids = tree.ids(); - const res = await statsQuery.search(this.client, ids); + const res = await statsQuery.searchAndFormat(this.client, ids); const alerts = res.alerts; const events = res.events; ids.forEach((id) => { diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/lifecycle_query_handler.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/lifecycle_query_handler.ts new file mode 100644 index 0000000000000..ab0501e099490 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/lifecycle_query_handler.ts @@ -0,0 +1,72 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { SearchResponse } from 'elasticsearch'; +import { ILegacyScopedClusterClient } from 'kibana/server'; +import { ResolverEvent, ResolverLifecycleNode } from '../../../../../common/endpoint/types'; +import { LifecycleQuery } from '../queries/lifecycle'; +import { QueryInfo } from '../queries/multi_searcher'; +import { SingleQueryHandler } from './fetch'; +import { createLifecycle } from './node'; + +/** + * Retrieve the lifecycle events for a node. + */ +export class LifecycleQueryHandler implements SingleQueryHandler { + private lifecycle: ResolverLifecycleNode | undefined; + private readonly query: LifecycleQuery; + constructor( + private readonly entityID: string, + indexPattern: string, + legacyEndpointID: string | undefined + ) { + this.query = new LifecycleQuery(indexPattern, legacyEndpointID); + } + + private handleResponse = (response: SearchResponse) => { + const results = this.query.formatResponse(response); + if (results.length !== 0) { + this.lifecycle = createLifecycle(this.entityID, results); + } + }; + + /** + * Build the query for retrieving the lifecycle events. This will return undefined once the results have been found. + */ + nextQuery(): QueryInfo | undefined { + if (this.getResults()) { + return; + } + + return { + query: this.query, + ids: this.entityID, + handler: this.handleResponse, + }; + } + + /** + * Get the results from the msearch. + */ + getResults(): ResolverLifecycleNode | undefined { + return this.lifecycle; + } + + /** + * Do a regular search and return the results. + * + * @param client the elasticsearch client. + */ + async search(client: ILegacyScopedClusterClient) { + const results = this.getResults(); + if (results) { + return results; + } + + this.handleResponse(await this.query.search(client, this.entityID)); + return this.getResults() ?? createLifecycle(this.entityID, []); + } +} diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/node.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/node.ts index 57a2ebfcc1792..98180885faf05 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/node.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/node.ts @@ -12,6 +12,7 @@ import { ResolverTree, ResolverChildNode, ResolverRelatedAlerts, + ResolverChildren, } from '../../../../../common/endpoint/types'; /** @@ -53,7 +54,6 @@ export function createChild(entityID: string): ResolverChildNode { const lifecycle = createLifecycle(entityID, []); return { ...lifecycle, - nextChild: null, }; } @@ -77,6 +77,19 @@ export function createLifecycle( return { entityID, lifecycle }; } +/** + * Creates a resolver children response. + * + * @param nodes the child nodes to add to the ResolverChildren response + * @param nextChild the cursor for the response + */ +export function createChildren( + nodes: ResolverChildNode[] = [], + nextChild: string | null = null +): ResolverChildren { + return { childNodes: nodes, nextChild }; +} + /** * Creates an empty `Tree` response structure that the tree handler would return * diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/pagination.test.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/pagination.test.ts index 74e4e252861e6..4daa45aec2a74 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/pagination.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/pagination.test.ts @@ -18,20 +18,20 @@ describe('Pagination', () => { const root = generator.generateEvent(); const events = Array.from(generator.relatedEventsGenerator(root, 5)); - it('does not build a cursor when all events are present', () => { - expect(PaginationBuilder.buildCursor(0, events)).toBeNull(); + it('does build a cursor when received the same number of events as was requested', () => { + expect(PaginationBuilder.buildCursorRequestLimit(4, events)).not.toBeNull(); }); - it('creates a cursor when not all events are present', () => { - expect(PaginationBuilder.buildCursor(events.length + 1, events)).not.toBeNull(); + it('does not create a cursor when the number of events received is less than the amount requested', () => { + expect(PaginationBuilder.buildCursorRequestLimit(events.length + 1, events)).toBeNull(); }); it('creates a cursor with the right information', () => { - const cursor = PaginationBuilder.buildCursor(events.length + 1, events); + const cursor = PaginationBuilder.buildCursorRequestLimit(events.length, events); expect(cursor).not.toBeNull(); // we are guaranteed that the cursor won't be null from the check above const builder = PaginationBuilder.createBuilder(0, cursor!); - const fields = builder.buildQueryFields(0, '', ''); + const fields = builder.buildQueryFields(''); expect(fields.search_after).toStrictEqual(getSearchAfterInfo(events)); }); }); @@ -39,30 +39,8 @@ describe('Pagination', () => { describe('pagination builder', () => { it('does not include the search after information when no cursor is provided', () => { const builder = PaginationBuilder.createBuilder(100); - const fields = builder.buildQueryFields(1, '', ''); + const fields = builder.buildQueryFields(''); expect(fields).not.toHaveProperty('search_after'); }); - - it('returns no results when the aggregation does not exist in the response', () => { - expect(PaginationBuilder.getTotals()).toStrictEqual({}); - }); - - it('constructs the totals from the aggregation results', () => { - const agg = { - totals: { - buckets: [ - { - key: 'awesome', - doc_count: 5, - }, - { - key: 'soup', - doc_count: 1, - }, - ], - }, - }; - expect(PaginationBuilder.getTotals(agg)).toStrictEqual({ awesome: 5, soup: 1 }); - }); }); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/pagination.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/pagination.ts index 61cb5bdb8f146..2b107ab1b6db4 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/pagination.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/pagination.ts @@ -8,41 +8,11 @@ import { ResolverEvent } from '../../../../../common/endpoint/types'; import { eventId } from '../../../../../common/endpoint/models/event'; import { JsonObject } from '../../../../../../../../src/plugins/kibana_utils/common'; -/** - * Represents a single result bucket of an aggregation - */ -export interface AggBucket { - key: string; - doc_count: number; -} - -interface TotalsAggregation { - totals?: { - buckets?: AggBucket[]; - }; -} - interface PaginationCursor { timestamp: number; eventID: string; } -/** - * The result structure of a query that leverages pagination. This includes totals that can be used to determine if - * additional nodes exist and additional queries need to be made to retrieve the nodes. - */ -export interface PaginatedResults { - /** - * Resulting events returned from the query. - */ - results: ResolverEvent[]; - /** - * Mapping of unique ID to total number of events that exist in ES. The events this references is scoped to the events - * that the query is searching for. - */ - totals: Record; -} - /** * This class handles constructing pagination cursors that resolver can use to return additional events in subsequent * queries. It also constructs an aggregation query to determine the totals for other queries. This class should be used @@ -83,19 +53,28 @@ export class PaginationBuilder { } /** - * Constructs a cursor to use in subsequent queries to retrieve the next set of results. + * Construct a cursor to use in subsequent queries. * - * @param total the total events that exist in ES scoped for a particular query. * @param results the events that were returned by the ES query */ - static buildCursor(total: number, results: ResolverEvent[]): string | null { - if (total > results.length && results.length > 0) { - const lastResult = results[results.length - 1]; - const cursor = { - timestamp: lastResult['@timestamp'], - eventID: eventId(lastResult), - }; - return PaginationBuilder.urlEncodeCursor(cursor); + static buildCursor(results: ResolverEvent[]): string | null { + const lastResult = results[results.length - 1]; + const cursor = { + timestamp: lastResult['@timestamp'], + eventID: eventId(lastResult), + }; + return PaginationBuilder.urlEncodeCursor(cursor); + } + + /** + * Constructs a cursor if the requested limit has not been met. + * + * @param requestLimit the request limit for a query. + * @param results the events that were returned by the ES query + */ + static buildCursorRequestLimit(requestLimit: number, results: ResolverEvent[]): string | null { + if (requestLimit <= results.length && results.length > 0) { + return PaginationBuilder.buildCursor(results); } return null; } @@ -124,45 +103,16 @@ export class PaginationBuilder { /** * Creates an object for adding the pagination fields to a query * - * @param numTerms number of unique IDs that are being search for in this query * @param tiebreaker a unique field to use as the tiebreaker for the search_after - * @param aggregator the field that specifies a unique ID per event (e.g. entity_id) - * @param aggs other aggregations being used with this query * @returns an object containing the pagination information */ - buildQueryFields( - numTerms: number, - tiebreaker: string, - aggregator: string, - aggs: JsonObject = {} - ): JsonObject { + buildQueryFields(tiebreaker: string): JsonObject { const fields: JsonObject = {}; fields.sort = [{ '@timestamp': 'asc' }, { [tiebreaker]: 'asc' }]; - fields.aggs = { ...aggs, totals: { terms: { field: aggregator, size: numTerms } } }; fields.size = this.size; if (this.timestamp && this.eventID) { fields.search_after = [this.timestamp, this.eventID] as Array; } return fields; } - - /** - * Returns the totals found for the specified query - * - * @param aggregations the aggregation field from the ES response - * @returns a mapping of unique ID (e.g. entity_ids) to totals found for those IDs - */ - static getTotals(aggregations?: TotalsAggregation): Record { - if (!aggregations?.totals?.buckets) { - return {}; - } - - return aggregations?.totals?.buckets?.reduce( - (cumulative: Record, bucket: AggBucket) => ({ - ...cumulative, - [bucket.key]: bucket.doc_count, - }), - {} - ); - } } diff --git a/x-pack/test/api_integration/apis/endpoint/resolver.ts b/x-pack/test/api_integration/apis/endpoint/resolver.ts index eeca8ee54e32f..ace32111005f4 100644 --- a/x-pack/test/api_integration/apis/endpoint/resolver.ts +++ b/x-pack/test/api_integration/apis/endpoint/resolver.ts @@ -246,6 +246,7 @@ export default function resolverAPIIntegrationTests({ getService }: FtrProviderC percentWithRelated: 100, numTrees: 1, alwaysGenMaxChildrenPerNode: true, + ancestryArraySize: 2, }; describe('Resolver', () => { @@ -542,13 +543,10 @@ export default function resolverAPIIntegrationTests({ getService }: FtrProviderC it('returns multiple levels of child process lifecycle events', async () => { const { body }: { body: ResolverChildren } = await supertest - .get( - `/api/endpoint/resolver/93802/children?legacyEndpointID=${endpointID}&generations=1` - ) + .get(`/api/endpoint/resolver/93802/children?legacyEndpointID=${endpointID}&children=10`) .expect(200); + expect(body.childNodes.length).to.eql(10); expect(body.nextChild).to.be(null); - expect(body.childNodes[0].nextChild).to.be(null); - expect(body.childNodes.length).to.eql(8); expect(body.childNodes[0].lifecycle.length).to.eql(1); expect( // for some reason the ts server doesn't think `endgame` exists even though we're using ResolverEvent @@ -615,19 +613,27 @@ export default function resolverAPIIntegrationTests({ getService }: FtrProviderC expect(body.childNodes.length).to.eql(12); // there will be 4 parents, the origin of the tree, and it's 3 children verifyChildren(body.childNodes, tree, 4, 3); + expect(body.nextChild).to.eql(null); }); it('returns a single generation of children', async () => { + // this gets a node should have 3 children which were created in succession so that the timestamps + // are ordered correctly to be retrieved in a single call + const distantChildEntityID = Array.from(tree.childrenLevels[0].values())[0].id; const { body }: { body: ResolverChildren } = await supertest - .get(`/api/endpoint/resolver/${tree.origin.id}/children?generations=1`) + .get(`/api/endpoint/resolver/${distantChildEntityID}/children?children=3`) .expect(200); expect(body.childNodes.length).to.eql(3); verifyChildren(body.childNodes, tree, 1, 3); + expect(body.nextChild).to.not.eql(null); }); - it('paginates the children of the origin node', async () => { + it('paginates the children', async () => { + // this gets a node should have 3 children which were created in succession so that the timestamps + // are ordered correctly to be retrieved in a single call + const distantChildEntityID = Array.from(tree.childrenLevels[0].values())[0].id; let { body }: { body: ResolverChildren } = await supertest - .get(`/api/endpoint/resolver/${tree.origin.id}/children?generations=1&children=1`) + .get(`/api/endpoint/resolver/${distantChildEntityID}/children?children=1`) .expect(200); expect(body.childNodes.length).to.eql(1); verifyChildren(body.childNodes, tree, 1, 1); @@ -635,49 +641,41 @@ export default function resolverAPIIntegrationTests({ getService }: FtrProviderC ({ body } = await supertest .get( - `/api/endpoint/resolver/${tree.origin.id}/children?generations=1&afterChild=${body.nextChild}` + `/api/endpoint/resolver/${distantChildEntityID}/children?children=2&afterChild=${body.nextChild}` ) .expect(200)); expect(body.childNodes.length).to.eql(2); verifyChildren(body.childNodes, tree, 1, 2); - expect(body.childNodes[0].nextChild).to.be(null); - expect(body.childNodes[1].nextChild).to.be(null); - }); - - it('paginates the children of different nodes', async () => { - let { body }: { body: ResolverChildren } = await supertest - .get(`/api/endpoint/resolver/${tree.origin.id}/children?generations=2&children=2`) - .expect(200); - // it should return 4 nodes total, 2 for each level - expect(body.childNodes.length).to.eql(4); - verifyChildren(body.childNodes, tree, 2); expect(body.nextChild).to.not.be(null); - expect(body.childNodes[0].nextChild).to.not.be(null); - // the second child will not have any results returned for it so it should not have pagination set (the first) - // request to get it's children should start at the beginning aka not passing any pagination parameter - expect(body.childNodes[1].nextChild).to.be(null); - const firstChild = body.childNodes[0]; - - // get the 3rd child of the origin of the tree ({ body } = await supertest .get( - `/api/endpoint/resolver/${tree.origin.id}/children?generations=1&children=10&afterChild=${body.nextChild}` + `/api/endpoint/resolver/${distantChildEntityID}/children?children=2&afterChild=${body.nextChild}` ) .expect(200)); - expect(body.childNodes.length).to.be(1); - verifyChildren(body.childNodes, tree, 1, 1); - expect(body.childNodes[0].nextChild).to.be(null); + expect(body.childNodes.length).to.eql(0); + expect(body.nextChild).to.be(null); + }); + + it('gets all children in two queries', async () => { + // should get all the children of the origin + let { body }: { body: ResolverChildren } = await supertest + .get(`/api/endpoint/resolver/${tree.origin.id}/children?children=3`) + .expect(200); + expect(body.childNodes.length).to.eql(3); + verifyChildren(body.childNodes, tree); + expect(body.nextChild).to.not.be(null); + const firstNodes = [...body.childNodes]; - // get the 1 child of the origin of the tree's last child ({ body } = await supertest .get( - `/api/endpoint/resolver/${firstChild.entityID}/children?generations=1&children=10&afterChild=${firstChild.nextChild}` + `/api/endpoint/resolver/${tree.origin.id}/children?children=10&afterChild=${body.nextChild}` ) .expect(200)); - expect(body.childNodes.length).to.be(1); - verifyChildren(body.childNodes, tree, 1, 1); - expect(body.childNodes[0].nextChild).to.be(null); + expect(body.childNodes.length).to.eql(9); + // put all the results together and we should have all the children + verifyChildren([...firstNodes, ...body.childNodes], tree, 4, 3); + expect(body.nextChild).to.be(null); }); }); }); @@ -703,7 +701,7 @@ export default function resolverAPIIntegrationTests({ getService }: FtrProviderC it('returns a tree', async () => { const { body }: { body: ResolverTree } = await supertest .get( - `/api/endpoint/resolver/${tree.origin.id}?children=100&generations=3&ancestors=5&events=4&alerts=4` + `/api/endpoint/resolver/${tree.origin.id}?children=100&ancestors=5&events=5&alerts=5` ) .expect(200); diff --git a/x-pack/test/api_integration/services/resolver.ts b/x-pack/test/api_integration/services/resolver.ts index 7a100c37aea91..750d2f702fb84 100644 --- a/x-pack/test/api_integration/services/resolver.ts +++ b/x-pack/test/api_integration/services/resolver.ts @@ -18,6 +18,7 @@ export interface Options extends TreeOptions { * Number of trees to generate. */ numTrees?: number; + seed?: string; } /** @@ -38,8 +39,9 @@ export function ResolverGeneratorProvider({ getService }: FtrProviderContext) { eventsIndex: string = 'logs-endpoint.events.process-default', alertsIndex: string = 'logs-endpoint.alerts-default' ): Promise { + const seed = options.seed || 'resolver-seed'; const allTrees: Tree[] = []; - const generator = new EndpointDocGenerator(); + const generator = new EndpointDocGenerator(seed); const numTrees = options.numTrees ?? 1; for (let j = 0; j < numTrees; j++) { const tree = generator.generateTree(options);