[Security Solution] [Resolver] Supporting configurable ID (#84365)

* Trying to flesh out new tree route

* Working on the descendants query

* Almost working descendants

* Possible solution for aggs

* Working aggregations extraction

* Working on the ancestry array for descendants

* Making changes to the unique id for  ancestr

* Implementing ancestry funcitonality

* Deleting the multiple edges

* Fleshing out the descendants loop for levels

* Writing tests for ancestors and descendants

* Fixing type errors and writing more tests

* Renaming validation variable and deprecating old tree routes

* Renaming tree integration test file

* Adding some integration tests

* Fixing ancestry to handle multiple nodes in the request and writing more tests

* Adding more tests

* Renaming new tree to handler file

* Renaming new tree directory

* Adding more unit tests

* Using doc value fields and working on types

* Adding comments and more tests

* Fixing timestamp test issue

* Adding more comments

* Adding timerange and filters

* Updating schema

* Fixing timestamp test issue take 2

* Updating tests to use raw filter

* Adding time to generator

* Adding time filter and tests for retrieving lifecycles

* Removing min array size

* Updating the DAL

* Adding time range iso format

* Working on middleware

* Fleshing out middleware and actions

* Adding id, parent, and name fields to the top level response

* Adding logic for identifying when the view is moved

* WIP: updated data layers and selectors

* Switching to use isAnimating

* WIP: tree is displayed

* WIP: need events data for panel and fix tests

* Removing panning logic, adding comments and renaming things

* WIP: added name to graph

* Writing tests for the models

* Fixing generator start and end time generation

* Updating the mocks with the new interface

* Revert "Fixing generator start and end time generation"

This reverts commit c42ffd7.

* WIP: remove unnecessary front end data transformation

* Starting on loading state for nodes and details

* Find the terminated nodes in the middlewaree

* Fixing ingest tests

* Loading states seem to be working

* Removing some todos

* undo graphNodePositions naming

* Node loading state svg and pulse

* Fixing time range

* undo name changes, cleanup

* Creating mock that leverages the generator

* update tree generator

* log nested data in simulator.debugActions()

* change newResolverTree to resolverTree

* fix oneNodeWithPaginatedEvents mock and node events of type test

* Refactoring data reducer test and changing resolverTree DAL

* WIP: updating mocks

* remove deprecation tags

* Fixing the isometric tests

* Fixing process event tests

* updated resolver_tree mocks, update tests

* update additional tests

* fixing eslint

* fixing has more selectors

* update tests

* debugging click test

* Working node loading test

* Adding error cube and another test

* Adding a test for the error case

* use stored indices, update event api call for winlog, cleanup todos

* Adding more comments and restricting the analyze event to only endpoint and winlogbeat

* update to use schema provided by backend

* Fixing some type errors

* Fixing translation issue

* Fixing type errors

* Adding reload functionality

* Fixing translation issue

* Adding more tests for reload

* Cleaning comments up

* adding legend and schema info

* added legend and info popovers

* removed comment

* Adding comments and cleaning up stuff

* add schema and dataSource to mock actions

* Fixing some type errors and starting to address feedback

* Moving mock function

* Handling powershell events

* Adding test for winlogbeat schema

* remove cube loading className in favor of styledComponent

* fix closeAnalyzer jumping from middle of screen when resolver loads

* fix originID casing

* Cleaning up middleware and renaming time range

* Fixing node details test and some of the use selectors

* Fixing tests and types

* fix popover style, cube style, specific timestamp, some translations

* Fixed a test, and continuing to address feedback

* Addressing more feedback

* Refactoring the node data loading tests

* Adding selector for indices

* fix i18n, break apart graph controls, fix process event dot styles

* fix type error, styled description lists, nodeID

* style fix

* Removing unneeded test subjects

* recursion, recursion, recursion

* Calculating ancestors, descendants, generations once in factory and refactoring state

* Removing stringify replacer

* Adding default timerange to be beginning of epoch to max date in future

* refactoring winlog event query to use winlog record_id field

* fix popover toggle

* Fix type issue

* fix popover toggle

* add some tests

* fix types

* Adding link to time range comment

Co-authored-by: Michael Olorunnisola <[email protected]>
Co-authored-by: Kibana Machine <[email protected]>

x-pack/plugins/security_solution/common/endpoint/generate_data.test.ts

@@ -326,6 +326,30 @@ describe('data generator', () => {
       }
     });
 
+    it('groups the children by their parent ID correctly', () => {
+      expect(tree.childrenByParent.size).toBe(13);
+      expect(tree.childrenByParent.get(tree.origin.id)?.size).toBe(3);
+
+      for (const value of tree.childrenByParent.values()) {
+        expect(value.size).toBe(3);
+      }
+
+      // loop over everything but the last level because those nodes won't be parents
+      for (let i = 0; i < tree.childrenLevels.length - 1; i++) {
+        const level = tree.childrenLevels[i];
+        // loop over all the nodes in a level
+        for (const id of level.keys()) {
+          // each node in the level should have 3 children
+          expect(tree.childrenByParent.get(id)?.size).toBe(3);
+
+          // let's make sure the children of this ID are actually in the next level and that they are the same reference
+          for (const [childID, childNode] of tree.childrenByParent.get(id)!.entries()) {
+            expect(tree.childrenLevels[i + 1].get(childID)).toBe(childNode);
+          }
+        }
+      }
+    });
+
     it('has the right related events for each node', () => {
       const checkRelatedEvents = (node: TreeNode) => {
         expect(node.relatedEvents.length).toEqual(4);

x-pack/plugins/security_solution/common/endpoint/generate_data.ts

@@ -31,6 +31,7 @@ import {
 import { EsAssetReference, KibanaAssetReference } from '../../../fleet/common/types/models';
 import { agentPolicyStatuses } from '../../../fleet/common/constants';
 import { firstNonNullValue } from './models/ecs_safety_helpers';
+import { EventOptions } from './types/generator';
 
 export type Event = AlertEvent | SafeEndpointEvent;
 /**
@@ -44,21 +45,6 @@ export type Event = AlertEvent | SafeEndpointEvent;
  */
 export const ANCESTRY_LIMIT: number = 2;
 
-interface EventOptions {
-  timestamp?: number;
-  entityID?: string;
-  parentEntityID?: string;
-  eventType?: string | string[];
-  eventCategory?: string | string[];
-  processName?: string;
-  ancestry?: string[];
-  ancestryArrayLimit?: number;
-  pid?: number;
-  parentPid?: number;
-  extensions?: object;
-  eventsDataStream?: DataStream;
-}
-
 const Windows: OSFields[] = [
   {
     name: 'windows 10.0',
@@ -299,6 +285,10 @@ export interface TreeNode {
  * A resolver tree that makes accessing specific nodes easier for tests.
  */
 export interface Tree {
+  /**
+   * Children grouped by the parent's ID
+   */
+  childrenByParent: Map<string, Map<string, TreeNode>>;
   /**
    * Map of entity_id to node
    */
@@ -648,7 +638,7 @@ export class EndpointDocGenerator {
     const ancestry: string[] =
       options.ancestry?.slice(0, options?.ancestryArrayLimit ?? ANCESTRY_LIMIT) ?? [];
 
-    const processName = options.processName ? options.processName : randomProcessName();
+    const processName = options.processName ? options.processName : this.randomProcessName();
     const detailRecordForEventType =
       options.extensions ||
       ((eventCategory) => {
@@ -761,16 +751,16 @@ export class EndpointDocGenerator {
   public generateTree(options: TreeOptions = {}): Tree {
     const optionsWithDef = getTreeOptionsWithDef(options);
     const addEventToMap = (nodeMap: Map<string, TreeNode>, event: Event) => {
-      const nodeId = entityIDSafeVersion(event);
-      if (!nodeId) {
+      const nodeID = entityIDSafeVersion(event);
+      if (!nodeID) {
         return nodeMap;
       }
 
       // if a node already exists for the entity_id we'll use that one, otherwise let's create a new empty node
       // and add the event to the right array.
-      let node = nodeMap.get(nodeId);
+      let node = nodeMap.get(nodeID);
       if (!node) {
-        node = { id: nodeId, lifecycle: [], relatedEvents: [], relatedAlerts: [] };
+        node = { id: nodeID, lifecycle: [], relatedEvents: [], relatedAlerts: [] };
       }
 
       // place the event in the right array depending on its category
@@ -784,7 +774,7 @@ export class EndpointDocGenerator {
         node.relatedAlerts.push(event);
       }
 
-      return nodeMap.set(nodeId, node);
+      return nodeMap.set(nodeID, node);
     };
 
     const groupNodesByParent = (children: Map<string, TreeNode>) => {
@@ -851,6 +841,7 @@ export class EndpointDocGenerator {
     const { startTime, endTime } = EndpointDocGenerator.getStartEndTimes(allEvents);
 
     return {
+      childrenByParent,
       children: childrenNodes,
       ancestry: ancestryNodes,
       allEvents,
@@ -1640,6 +1631,11 @@ export class EndpointDocGenerator {
       HostPolicyResponseActionStatus.warning,
     ]);
   }
+
+  /** Return a random fake process name */
+  private randomProcessName(): string {
+    return this.randomChoice(fakeProcessNames);
+  }
 }
 
 const fakeProcessNames = [
@@ -1650,7 +1646,3 @@ const fakeProcessNames = [
   'iexlorer.exe',
   'explorer.exe',
 ];
-/** Return a random fake process name */
-function randomProcessName(): string {
-  return fakeProcessNames[Math.floor(Math.random() * fakeProcessNames.length)];
-}

x-pack/plugins/security_solution/common/endpoint/models/event.ts

@@ -3,7 +3,13 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-import { LegacyEndpointEvent, ResolverEvent, SafeResolverEvent, ECSField } from '../types';
+import {
+  LegacyEndpointEvent,
+  ResolverEvent,
+  SafeResolverEvent,
+  ECSField,
+  WinlogEvent,
+} from '../types';
 import { firstNonNullValue, hasValue, values } from './ecs_safety_helpers';
 
 /**
@@ -188,6 +194,15 @@ export function eventID(event: SafeResolverEvent): number | undefined | string {
   );
 }
 
+/**
+ * Retrieve the record_id field from a winlog event.
+ *
+ * @param event a winlog event
+ */
+export function winlogRecordID(event: WinlogEvent): undefined | string {
+  return firstNonNullValue(event.winlog?.record_id);
+}
+
 /**
  * Minimum fields needed from the `SafeResolverEvent` type for the function below to operate correctly.
  */

x-pack/plugins/security_solution/common/endpoint/models/node.ts

@@ -0,0 +1,62 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+import { ResolverNode } from '../types';
+import { firstNonNullValue } from './ecs_safety_helpers';
+
+/**
+ * These functions interact with the generic resolver node structure that does not define a specific format for the data
+ * returned by Elasticsearch. These functions are similar to the events.ts model's function except that they do not
+ * assume that the data will conform to a structure like an Endpoint or LegacyEndgame event.
+ */
+
+/**
+ * @description - Extract the first non null value from the nodeID depending on the datasource. Returns
+ * undefined if the field was never set.
+ */
+export function nodeID(node: ResolverNode): string | undefined {
+  return node?.id ? String(firstNonNullValue(node.id)) : undefined;
+}
+
+/**
+ * @description - Provides the parent for the given node
+ */
+export function parentId(node: ResolverNode): string | undefined {
+  return node?.parent ? String(firstNonNullValue(node?.parent)) : undefined;
+}
+
+/**
+ * The `@timestamp` for the event, as a `Date` object.
+ * If `@timestamp` couldn't be parsed as a `Date`, returns `undefined`.
+ */
+export function timestampAsDate(node: ResolverNode): Date | undefined {
+  const value = nodeDataTimestamp(node);
+  if (value === undefined) {
+    return undefined;
+  }
+
+  const date = new Date(value);
+  // Check if the date is valid
+  if (isFinite(date.getTime())) {
+    return date;
+  } else {
+    return undefined;
+  }
+}
+
+/**
+ * Extracts the first non null value from the `@timestamp` field in the node data attribute.
+ */
+export function nodeDataTimestamp(node: ResolverNode): undefined | number | string {
+  return firstNonNullValue(node?.data['@timestamp']);
+}
+
+/**
+ * @description - Extract the first non null value from the node name depending on the datasource. If it was never set
+ * default to the ID, and if no ID, then undefined
+ */
+export function nodeName(node: ResolverNode): string | undefined {
+  return node?.name ? String(firstNonNullValue(node.name)) : undefined;
+}

x-pack/plugins/security_solution/common/endpoint/schema/resolver.ts

@@ -40,7 +40,7 @@ export const validateTree = {
     descendants: schema.number({ defaultValue: 1000, min: 0, max: 10000 }),
     // if the ancestry array isn't specified allowing 200 might be too high
     ancestors: schema.number({ defaultValue: 200, min: 0, max: 10000 }),
-    timerange: schema.object({
+    timeRange: schema.object({
       from: schema.string(),
       to: schema.string(),
     }),
@@ -70,11 +70,14 @@ export const validateEvents = {
     limit: schema.number({ defaultValue: 1000, min: 1, max: 10000 }),
     afterEvent: schema.maybe(schema.string()),
   }),
-  body: schema.nullable(
-    schema.object({
-      filter: schema.maybe(schema.string()),
-    })
-  ),
+  body: schema.object({
+    timeRange: schema.object({
+      from: schema.string(),
+      to: schema.string(),
+    }),
+    indexPatterns: schema.arrayOf(schema.string()),
+    filter: schema.maybe(schema.string()),
+  }),
 };
 
 /**

x-pack/plugins/security_solution/common/endpoint/types/generator.ts

@@ -0,0 +1,25 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { DataStream } from './index';
+
+/**
+ * The configuration options for generating an event.
+ */
+export interface EventOptions {
+  timestamp?: number;
+  entityID?: string;
+  parentEntityID?: string;
+  eventType?: string | string[];
+  eventCategory?: string | string[];
+  processName?: string;
+  ancestry?: string[];
+  ancestryArrayLimit?: number;
+  pid?: number;
+  parentPid?: number;
+  extensions?: object;
+  eventsDataStream?: DataStream;
+}