[Security_Solution][Endpoint] Leveraging msearch and ancestry array f…

…or resolver (#70134) (#70583)

* Refactor generator for ancestry support

* Adding optional ancestry array

* Refactor the pagination since the totals are not used anymore

* Updating the queries to not use aggregations for determining the totals

* Refactoring the children helper to handle pagination without totals

* Pinning the seed for the resolver tree generator service

* Splitting the fetcher into multiple classes for msearch

* Updating tests and api for ancestry array and msearch

* Adding more comments and fixing type errors

* Fixing resolver test import

* Fixing tests and type errors

* Fixing type errors and tests

* Removing useAncestry field

* Fixing test

* Removing useAncestry field from tests

* An empty array will be returned because that's how ES will do it too

x-pack/plugins/security_solution/common/endpoint/models/event.ts

@@ -60,6 +60,24 @@ export function ancestryArray(event: ResolverEvent): string[] | undefined {
   return event.process.Ext.ancestry;
 }
 
+export function getAncestryAsArray(event: ResolverEvent | undefined): string[] {
+  if (!event) {
+    return [];
+  }
+
+  const ancestors = ancestryArray(event);
+  if (ancestors) {
+    return ancestors;
+  }
+
+  const parentID = parentEntityId(event);
+  if (parentID) {
+    return [parentID];
+  }
+
+  return [];
+}
+
 /**
  * @param event The event to get the category for
  */

x-pack/plugins/security_solution/common/endpoint/schema/resolver.ts

@@ -13,7 +13,6 @@ export const validateTree = {
   params: schema.object({ id: schema.string() }),
   query: schema.object({
     children: schema.number({ defaultValue: 10, min: 0, max: 100 }),
-    generations: schema.number({ defaultValue: 3, min: 0, max: 3 }),
     ancestors: schema.number({ defaultValue: 3, min: 0, max: 5 }),
     events: schema.number({ defaultValue: 100, min: 0, max: 1000 }),
     alerts: schema.number({ defaultValue: 100, min: 0, max: 1000 }),
@@ -66,7 +65,6 @@ export const validateChildren = {
   params: schema.object({ id: schema.string() }),
   query: schema.object({
     children: schema.number({ defaultValue: 10, min: 1, max: 100 }),
-    generations: schema.number({ defaultValue: 3, min: 1, max: 3 }),
     afterChild: schema.maybe(schema.string()),
     legacyEndpointID: schema.maybe(schema.string()),
   }),

x-pack/plugins/security_solution/common/endpoint/types.ts

@@ -76,12 +76,18 @@ export interface ResolverNodeStats {
  */
 export interface ResolverChildNode extends ResolverLifecycleNode {
   /**
-   * A child node's pagination cursor can be null for a couple reasons:
-   * 1. At the time of querying it could have no children in ES, in which case it will be marked as
-   *  null because we know it does not have children during this query.
-   * 2. If the max level was reached we do not know if this node has children or not so we'll mark it as null
+   * nextChild can have 3 different states:
+   *
+   * undefined: This indicates that you should not use this node for additional queries. It does not mean that node does
+   * not have any more direct children. The node could have more direct children but to determine that, use the
+   * ResolverChildren node's nextChild.
+   *
+   * null: Indicates that we have received all the children of the node. There may be more descendants though.
+   *
+   * string: Indicates this is a leaf node and it can be used to continue querying for additional descendants
+   * using this node's entity_id
    */
-  nextChild: string | null;
+  nextChild?: string | null;
 }
 
 /**
@@ -91,7 +97,14 @@ export interface ResolverChildNode extends ResolverLifecycleNode {
 export interface ResolverChildren {
   childNodes: ResolverChildNode[];
   /**
-   * This is the children cursor for the origin of a tree.
+   * nextChild can have 2 different states:
+   *
+   * null: Indicates that we have received all the descendants that can be retrieved using this node. To retrieve more
+   * nodes in the tree use a cursor provided in one of the returned children. If no other cursor exists then the tree
+   * is complete.
+   *
+   * string: Indicates this node has more descendants that can be retrieved, pass this cursor in while using this node's
+   * entity_id for the request.
    */
   nextChild: string | null;
 }

x-pack/plugins/security_solution/server/endpoint/routes/resolver/children.ts

@@ -18,14 +18,14 @@ export function handleChildren(
   return async (context, req, res) => {
     const {
       params: { id },
-      query: { children, generations, afterChild, legacyEndpointID: endpointID },
+      query: { children, afterChild, legacyEndpointID: endpointID },
     } = req;
     try {
       const client = context.core.elasticsearch.legacy.client;
       const fetcher = new Fetcher(client, id, eventsIndexPattern, alertsIndexPattern, endpointID);
 
       return res.ok({
-        body: await fetcher.children(children, generations, afterChild),
+        body: await fetcher.children(children, afterChild),
       });
     } catch (err) {
       log.warn(err);

x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/alerts.ts

@@ -6,13 +6,13 @@
 import { SearchResponse } from 'elasticsearch';
 import { ResolverEvent } from '../../../../../common/endpoint/types';
 import { ResolverQuery } from './base';
-import { PaginationBuilder, PaginatedResults } from '../utils/pagination';
+import { PaginationBuilder } from '../utils/pagination';
 import { JsonObject } from '../../../../../../../../src/plugins/kibana_utils/common';
 
 /**
  * Builds a query for retrieving alerts for a node.
  */
-export class AlertsQuery extends ResolverQuery<PaginatedResults> {
+export class AlertsQuery extends ResolverQuery<ResolverEvent[]> {
   constructor(
     private readonly pagination: PaginationBuilder,
     indexPattern: string | string[],
@@ -38,11 +38,7 @@ export class AlertsQuery extends ResolverQuery<PaginatedResults> {
           ],
         },
       },
-      ...this.pagination.buildQueryFields(
-        uniquePIDs.length,
-        'endgame.serial_event_id',
-        'endgame.unique_pid'
-      ),
+      ...this.pagination.buildQueryFields('endgame.serial_event_id'),
     };
   }
 
@@ -60,14 +56,11 @@ export class AlertsQuery extends ResolverQuery<PaginatedResults> {
           ],
         },
       },
-      ...this.pagination.buildQueryFields(entityIDs.length, 'event.id', 'process.entity_id'),
+      ...this.pagination.buildQueryFields('event.id'),
     };
   }
 
-  formatResponse(response: SearchResponse<ResolverEvent>): PaginatedResults {
-    return {
-      results: ResolverQuery.getResults(response),
-      totals: PaginationBuilder.getTotals(response.aggregations),
-    };
+  formatResponse(response: SearchResponse<ResolverEvent>): ResolverEvent[] {
+    return this.getResults(response);
   }
 }

x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/base.ts

@@ -14,10 +14,12 @@ import { MSearchQuery } from './multi_searcher';
 /**
  * ResolverQuery provides the base structure for queries to retrieve events when building a resolver graph.
  *
- * @param T the structured return type of a resolver query. This represents the type that is returned when translating
- * Elasticsearch's SearchResponse<ResolverEvent> response.
+ * @param T the structured return type of a resolver query. This represents the final return type of the query after handling
+ * any aggregations.
+ * @param R the is the type after transforming ES's response. Making this definable let's us set whether it is a resolver event
+ * or something else.
  */
-export abstract class ResolverQuery<T> implements MSearchQuery {
+export abstract class ResolverQuery<T, R = ResolverEvent> implements MSearchQuery {
   /**
    *
    * @param indexPattern the index pattern to use in the query for finding indices with documents in ES.
@@ -50,7 +52,7 @@ export abstract class ResolverQuery<T> implements MSearchQuery {
     };
   }
 
-  protected static getResults(response: SearchResponse<ResolverEvent>): ResolverEvent[] {
+  protected getResults(response: SearchResponse<R>): R[] {
     return response.hits.hits.map((hit) => hit._source);
   }
 
@@ -68,19 +70,26 @@ export abstract class ResolverQuery<T> implements MSearchQuery {
   }
 
   /**
-   * Searches ES for the specified ids.
+   * Searches ES for the specified ids and format the response.
    *
    * @param client a client for searching ES
    * @param ids a single more multiple unique node ids (e.g. entity_id or unique_pid)
    */
-  async search(client: ILegacyScopedClusterClient, ids: string | string[]): Promise<T> {
-    const res: SearchResponse<ResolverEvent> = await client.callAsCurrentUser(
-      'search',
-      this.buildSearch(ids)
-    );
+  async searchAndFormat(client: ILegacyScopedClusterClient, ids: string | string[]): Promise<T> {
+    const res: SearchResponse<ResolverEvent> = await this.search(client, ids);
     return this.formatResponse(res);
   }
 
+  /**
+   * Searches ES for the specified ids but do not format the response.
+   *
+   * @param client a client for searching ES
+   * @param ids a single more multiple unique node ids (e.g. entity_id or unique_pid)
+   */
+  async search(client: ILegacyScopedClusterClient, ids: string | string[]) {
+    return client.callAsCurrentUser('search', this.buildSearch(ids));
+  }
+
   /**
    * Builds a query to search the legacy data format.
    *

x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/children.test.ts

@@ -25,7 +25,7 @@ describe('Children query', () => {
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
     const msearch: any = query.buildMSearch(['1234', '5678']);
     expect(msearch[0].index).toBe('index-pattern');
-    expect(msearch[1].query.bool.filter[0]).toStrictEqual({
+    expect(msearch[1].query.bool.filter[0].bool.should[0]).toStrictEqual({
       terms: { 'process.parent.entity_id': ['1234', '5678'] },
     });
   });

x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/children.ts

@@ -6,13 +6,13 @@
 import { SearchResponse } from 'elasticsearch';
 import { ResolverEvent } from '../../../../../common/endpoint/types';
 import { ResolverQuery } from './base';
-import { PaginationBuilder, PaginatedResults } from '../utils/pagination';
+import { PaginationBuilder } from '../utils/pagination';
 import { JsonObject } from '../../../../../../../../src/plugins/kibana_utils/common';
 
 /**
  * Builds a query for retrieving descendants of a node.
  */
-export class ChildrenQuery extends ResolverQuery<PaginatedResults> {
+export class ChildrenQuery extends ResolverQuery<ResolverEvent[]> {
   constructor(
     private readonly pagination: PaginationBuilder,
     indexPattern: string | string[],
@@ -53,11 +53,7 @@ export class ChildrenQuery extends ResolverQuery<PaginatedResults> {
           ],
         },
       },
-      ...this.pagination.buildQueryFields(
-        uniquePIDs.length,
-        'endgame.serial_event_id',
-        'endgame.unique_ppid'
-      ),
+      ...this.pagination.buildQueryFields('endgame.serial_event_id'),
     };
   }
 
@@ -67,7 +63,16 @@ export class ChildrenQuery extends ResolverQuery<PaginatedResults> {
         bool: {
           filter: [
             {
-              terms: { 'process.parent.entity_id': entityIDs },
+              bool: {
+                should: [
+                  {
+                    terms: { 'process.parent.entity_id': entityIDs },
+                  },
+                  {
+                    terms: { 'process.Ext.ancestry': entityIDs },
+                  },
+                ],
+              },
             },
             {
               term: { 'event.category': 'process' },
@@ -81,14 +86,11 @@ export class ChildrenQuery extends ResolverQuery<PaginatedResults> {
           ],
         },
       },
-      ...this.pagination.buildQueryFields(entityIDs.length, 'event.id', 'process.parent.entity_id'),
+      ...this.pagination.buildQueryFields('event.id'),
     };
   }
 
-  formatResponse(response: SearchResponse<ResolverEvent>): PaginatedResults {
-    return {
-      results: ResolverQuery.getResults(response),
-      totals: PaginationBuilder.getTotals(response.aggregations),
-    };
+  formatResponse(response: SearchResponse<ResolverEvent>): ResolverEvent[] {
+    return this.getResults(response);
   }
 }

x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/events.ts

@@ -6,13 +6,13 @@
 import { SearchResponse } from 'elasticsearch';
 import { ResolverEvent } from '../../../../../common/endpoint/types';
 import { ResolverQuery } from './base';
-import { PaginationBuilder, PaginatedResults } from '../utils/pagination';
+import { PaginationBuilder } from '../utils/pagination';
 import { JsonObject } from '../../../../../../../../src/plugins/kibana_utils/common';
 
 /**
  * Builds a query for retrieving related events for a node.
  */
-export class EventsQuery extends ResolverQuery<PaginatedResults> {
+export class EventsQuery extends ResolverQuery<ResolverEvent[]> {
   constructor(
     private readonly pagination: PaginationBuilder,
     indexPattern: string | string[],
@@ -45,11 +45,7 @@ export class EventsQuery extends ResolverQuery<PaginatedResults> {
           ],
         },
       },
-      ...this.pagination.buildQueryFields(
-        uniquePIDs.length,
-        'endgame.serial_event_id',
-        'endgame.unique_pid'
-      ),
+      ...this.pagination.buildQueryFields('endgame.serial_event_id'),
     };
   }
 
@@ -74,14 +70,11 @@ export class EventsQuery extends ResolverQuery<PaginatedResults> {
           ],
         },
       },
-      ...this.pagination.buildQueryFields(entityIDs.length, 'event.id', 'process.entity_id'),
+      ...this.pagination.buildQueryFields('event.id'),
     };
   }
 
-  formatResponse(response: SearchResponse<ResolverEvent>): PaginatedResults {
-    return {
-      results: ResolverQuery.getResults(response),
-      totals: PaginationBuilder.getTotals(response.aggregations),
-    };
+  formatResponse(response: SearchResponse<ResolverEvent>): ResolverEvent[] {
+    return this.getResults(response);
   }
 }

x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/lifecycle.ts

@@ -60,6 +60,6 @@ export class LifecycleQuery extends ResolverQuery<ResolverEvent[]> {
   }
 
   formatResponse(response: SearchResponse<ResolverEvent>): ResolverEvent[] {
-    return ResolverQuery.getResults(response);
+    return this.getResults(response);
   }
 }

x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/multi_searcher.ts

@@ -5,7 +5,7 @@
  */
 
 import { ILegacyScopedClusterClient } from 'kibana/server';
-import { MSearchResponse } from 'elasticsearch';
+import { MSearchResponse, SearchResponse } from 'elasticsearch';
 import { ResolverEvent } from '../../../../../common/endpoint/types';
 import { JsonObject } from '../../../../../../../../src/plugins/kibana_utils/common';
 
@@ -34,6 +34,10 @@ export interface QueryInfo {
    * one or many unique identifiers to be searched for in this query
    */
   ids: string | string[];
+  /**
+   * a function to handle the response
+   */
+  handler: (response: SearchResponse<ResolverEvent>) => void;
 }
 
 /**
@@ -57,10 +61,10 @@ export class MultiSearcher {
       throw new Error('No queries provided to MultiSearcher');
     }
 
-    let searchQuery: JsonObject[] = [];
-    queries.forEach(
-      (info) => (searchQuery = [...searchQuery, ...info.query.buildMSearch(info.ids)])
-    );
+    const searchQuery: JsonObject[] = [];
+    for (const info of queries) {
+      searchQuery.push(...info.query.buildMSearch(info.ids));
+    }
     const res: MSearchResponse<ResolverEvent> = await this.client.callAsCurrentUser('msearch', {
       body: searchQuery,
     });
@@ -72,6 +76,8 @@ export class MultiSearcher {
     if (res.responses.length !== queries.length) {
       throw new Error(`Responses length was: ${res.responses.length} expected ${queries.length}`);
     }
-    return res.responses;
+    for (let i = 0; i < queries.length; i++) {
+      queries[i].handler(res.responses[i]);
+    }
   }
 }

x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/stats.ts

@@ -7,13 +7,17 @@ import { SearchResponse } from 'elasticsearch';
 import { ResolverQuery } from './base';
 import { ResolverEvent, EventStats } from '../../../../../common/endpoint/types';
 import { JsonObject } from '../../../../../../../../src/plugins/kibana_utils/common';
-import { AggBucket } from '../utils/pagination';
 
 export interface StatsResult {
   alerts: Record<string, number>;
   events: Record<string, EventStats>;
 }
 
+interface AggBucket {
+  key: string;
+  doc_count: number;
+}
+
 interface CategoriesAgg extends AggBucket {
   /**
    * The reason categories is optional here is because if no data was returned in the query the categories aggregation