[Security Solution] Resolver children pagination

x-pack/plugins/security_solution/common/endpoint/generate_data.test.ts

@@ -26,6 +26,13 @@ describe('data generator', () => {
     generator = new EndpointDocGenerator('seed');
   });
 
+  it('creates events with a numerically increasing sequence value', () => {
+    const event1 = generator.generateEvent();
+    const event2 = generator.generateEvent();
+
+    expect(event2.event.sequence).toBe(event1.event.sequence + 1);
+  });
+
   it('creates the same documents with same random seed', () => {
     const generator1 = new EndpointDocGenerator('seed');
     const generator2 = new EndpointDocGenerator('seed');

x-pack/plugins/security_solution/common/endpoint/generate_data.ts

@@ -333,6 +333,7 @@ export function getTreeOptionsWithDef(options?: TreeOptions): TreeOptionDefaults
 export class EndpointDocGenerator {
   commonInfo: HostInfo;
   random: seedrandom.prng;
+  sequence: number = 0;
   constructor(seed: string | seedrandom.prng = Math.random().toString()) {
     if (typeof seed === 'string') {
       this.random = seedrandom(seed);
@@ -440,6 +441,7 @@ export class EndpointDocGenerator {
         dataset: 'endpoint',
         module: 'endpoint',
         type: 'creation',
+        sequence: this.sequence++,
       },
       file: {
         owner: 'SYSTEM',
@@ -586,6 +588,7 @@ export class EndpointDocGenerator {
         kind: 'event',
         type: options.eventType ? options.eventType : ['start'],
         id: this.seededUUIDv4(),
+        sequence: this.sequence++,
       },
       host: this.commonInfo.host,
       process: {

x-pack/plugins/security_solution/common/endpoint/models/event.ts

@@ -86,6 +86,20 @@ export function eventId(event: ResolverEvent): number | undefined | string {
   return event.event.id;
 }
 
+export function eventSequence(event: ResolverEvent): number | undefined {
+  if (isLegacyEvent(event)) {
+    return firstNonNullValue(event.endgame.serial_event_id);
+  }
+  return firstNonNullValue(event.event?.sequence);
+}
+
+export function eventSequenceSafeVersion(event: SafeResolverEvent): number | undefined {
+  if (isLegacyEventSafeVersion(event)) {
+    return firstNonNullValue(event.endgame.serial_event_id);
+  }
+  return firstNonNullValue(event.event?.sequence);
+}
+
 export function eventIDSafeVersion(event: SafeResolverEvent): number | undefined | string {
   return firstNonNullValue(
     isLegacyEventSafeVersion(event) ? event.endgame?.serial_event_id : event.event?.id

x-pack/plugins/security_solution/common/endpoint/types.ts

@@ -311,6 +311,7 @@ export interface AlertEvent {
     dataset: string;
     module: string;
     type: string;
+    sequence: number;
   };
   Endpoint: {
     policy: {
@@ -515,6 +516,7 @@ export interface EndpointEvent {
     type: string | string[];
     id: string;
     kind: string;
+    sequence: number;
   };
   host: Host;
   network?: {
@@ -591,6 +593,7 @@ export type SafeEndpointEvent = Partial<{
     type: ECSField<string>;
     id: ECSField<string>;
     kind: ECSField<string>;
+    sequence: ECSField<number>;
   }>;
   host: Partial<{
     id: ECSField<string>;

x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/children.ts

@@ -6,15 +6,15 @@
 import { SearchResponse } from 'elasticsearch';
 import { ResolverEvent } from '../../../../../common/endpoint/types';
 import { ResolverQuery } from './base';
-import { PaginationBuilder } from '../utils/pagination';
+import { ChildrenPaginationBuilder } from '../utils/children_pagination';
 import { JsonObject } from '../../../../../../../../src/plugins/kibana_utils/common';
 
 /**
  * Builds a query for retrieving descendants of a node.
  */
 export class ChildrenQuery extends ResolverQuery<ResolverEvent[]> {
   constructor(
-    private readonly pagination: PaginationBuilder,
+    private readonly pagination: ChildrenPaginationBuilder,
     indexPattern: string | string[],
     endpointID?: string
   ) {
@@ -32,6 +32,7 @@ export class ChildrenQuery extends ResolverQuery<ResolverEvent[]> {
       query: {
         bool: {
           filter: [
+            ...paginationFields.filters,
             {
               terms: { 'endgame.unique_ppid': uniquePIDs },
             },
@@ -63,7 +64,7 @@ export class ChildrenQuery extends ResolverQuery<ResolverEvent[]> {
   }
 
   protected query(entityIDs: string[]): JsonObject {
-    const paginationFields = this.pagination.buildQueryFieldsAsInterface('event.id');
+    const paginationFields = this.pagination.buildQueryFields('event.id');
     return {
       /**
        * Using collapse here will only return a single event per occurrence of a process.entity_id. The events are sorted
@@ -80,12 +81,12 @@ export class ChildrenQuery extends ResolverQuery<ResolverEvent[]> {
       collapse: {
         field: 'process.entity_id',
       },
-      // do not set the search_after field because collapse does not work with it
       size: paginationFields.size,
       sort: paginationFields.sort,
       query: {
         bool: {
           filter: [
+            ...paginationFields.filters,
             {
               bool: {
                 should: [

x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/children_helper.ts

@@ -16,7 +16,7 @@ import {
   ResolverChildren,
 } from '../../../../../common/endpoint/types';
 import { createChild } from './node';
-import { PaginationBuilder } from './pagination';
+import { ChildrenPaginationBuilder } from './children_pagination';
 
 /**
  * This class helps construct the children structure when building a resolver tree.
@@ -162,7 +162,7 @@ export class ChildrenNodesHelper {
     for (const nodeEntityID of nodes.values()) {
       const cachedNode = this.entityToNodeCache.get(nodeEntityID);
       if (cachedNode) {
-        cachedNode.nextChild = PaginationBuilder.buildCursor(startEvents);
+        cachedNode.nextChild = ChildrenPaginationBuilder.buildCursor(startEvents);
       }
     }
   }

x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/children_pagination.ts

@@ -0,0 +1,121 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { ResolverEvent } from '../../../../../common/endpoint/types';
+import { eventSequence } from '../../../../../common/endpoint/models/event';
+import { JsonObject } from '../../../../../../../../src/plugins/kibana_utils/common';
+import { urlEncodeCursor, SortFields, urlDecodeCursor } from './pagination';
+
+export interface ChildrenPaginationCursor {
+  timestamp: number;
+  sequence: number;
+}
+
+/**
+ * Interface for defining the returned pagination information.
+ */
+export interface ChildrenPaginationFields {
+  sort: SortFields;
+  size: number;
+  filters: JsonObject[];
+}
+
+/**
+ * This class handles constructing pagination cursors that resolver can use to return additional events in subsequent
+ * queries.
+ */
+export class ChildrenPaginationBuilder {
+  constructor(
+    /**
+     * upper limit of how many results should be returned by the parent query.
+     */
+    private readonly size: number,
+    /**
+     * timestamp that will be used in the search_after section
+     */
+    private readonly timestamp?: number,
+    /**
+     * unique sequence number for the event
+     */
+    private readonly sequence?: number
+  ) {}
+
+  static decode(
+    parsed: ChildrenPaginationCursor | undefined
+  ): ChildrenPaginationCursor | undefined {
+    if (parsed && parsed.timestamp && parsed.sequence) {
+      const { timestamp, sequence } = parsed;
+      return { timestamp, sequence };
+    }
+  }
+
+  /**
+   * Construct a cursor to use in subsequent queries.
+   *
+   * @param results the events that were returned by the ES query
+   */
+  static buildCursor(results: ResolverEvent[]): string | null {
+    const lastResult = results[results.length - 1];
+    const sequence = eventSequence(lastResult);
+    const cursor = {
+      timestamp: lastResult['@timestamp'],
+      sequence: sequence === undefined ? 0 : sequence,
+    };
+    return urlEncodeCursor(cursor);
+  }
+
+  /**
+   * Creates a PaginationBuilder with an upper bound limit of results and a specific cursor to use to retrieve the next
+   * set of results.
+   *
+   * @param limit upper bound for the number of results to return within this query
+   * @param after a cursor to retrieve the next set of results
+   */
+  static createBuilder(limit: number, after?: string): ChildrenPaginationBuilder {
+    if (after) {
+      try {
+        const cursor = urlDecodeCursor(after, ChildrenPaginationBuilder.decode);
+        if (cursor && cursor.timestamp && cursor.sequence) {
+          return new ChildrenPaginationBuilder(limit, cursor.timestamp, cursor.sequence);
+        }
+      } catch (err) {
+        /* tslint:disable:no-empty */
+      } // ignore invalid cursor values
+    }
+    return new ChildrenPaginationBuilder(limit);
+  }
+
+  /**
+   * Helper for creates an object for adding the pagination fields to a query
+   *
+   * @param tiebreaker a unique field to use as the tiebreaker for the search_after
+   * @returns an object containing the pagination information
+   */
+  buildQueryFields(tiebreaker: string): ChildrenPaginationFields {
+    const sort: SortFields = [{ '@timestamp': 'asc' }, { [tiebreaker]: 'asc' }];
+    const filters: JsonObject[] = [];
+    if (this.timestamp && this.sequence) {
+      filters.push(
+        {
+          range: {
+            '@timestamp': {
+              gte: this.timestamp,
+            },
+          },
+        },
+        {
+          range: {
+            'event.sequence': {
+              gt: this.sequence,
+            },
+          },
+        }
+      );
+    }
+
+    return { sort, size: this.size, filters };
+  }
+}

x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/children_start_query_handler.ts

@@ -11,7 +11,7 @@ import { ChildrenQuery } from '../queries/children';
 import { QueryInfo } from '../queries/multi_searcher';
 import { QueryHandler } from './fetch';
 import { ChildrenNodesHelper } from './children_helper';
-import { PaginationBuilder } from './pagination';
+import { ChildrenPaginationBuilder } from './children_pagination';
 
 /**
  * Retrieve the start lifecycle events for the children of a resolver tree.
@@ -32,7 +32,7 @@ export class ChildrenStartQueryHandler implements QueryHandler<ChildrenNodesHelp
     private readonly legacyEndpointID: string | undefined
   ) {
     this.query = new ChildrenQuery(
-      PaginationBuilder.createBuilder(limit, after),
+      ChildrenPaginationBuilder.createBuilder(limit, after),
       indexPattern,
       legacyEndpointID
     );
@@ -56,8 +56,13 @@ export class ChildrenStartQueryHandler implements QueryHandler<ChildrenNodesHelp
     }
 
     this.limitLeft = this.limit - this.childrenHelper.getNumNodes();
+
+    if (this.limitLeft < 0) {
+      this.limitLeft = 0;
+    }
+
     this.query = new ChildrenQuery(
-      PaginationBuilder.createBuilder(this.limitLeft),
+      ChildrenPaginationBuilder.createBuilder(this.limitLeft),
       this.indexPattern,
       this.legacyEndpointID
     );