[Endpoint] Sample data generator CLI script (#59952)

* start on cli

* make it work

* cleanup

* remove failed attempt code

* update package and tsconfig

* remove empty file

* generate resolver events from multiple endpoints

* re-add child randomization

* align index names with real plugin

* remove duplication

* better naming

* add temporary mapping to sample data generator

* error handling, move tsconfig

* add readme

* Update README.md

* move mapping from common to scripts

* make delete index option

* remove unnecessary map call

* fix import style

Co-authored-by: Elastic Machine <[email protected]>

x-pack/plugins/endpoint/common/generate_data.test.ts

@@ -151,7 +151,7 @@ describe('data generator', () => {
     const timestamp = new Date().getTime();
     const root = generator.generateEvent({ timestamp });
     const generations = 2;
-    const events = generator.generateDescendantsTree(root, generations);
+    const events = [root, ...generator.generateDescendantsTree(root, generations)];
     const rootNode = buildResolverTree(events);
     const visitedEvents = countResolverEvents(rootNode, generations);
     expect(visitedEvents).toEqual(events.length);

x-pack/plugins/endpoint/common/generate_data.ts

@@ -6,7 +6,7 @@
 
 import uuid from 'uuid';
 import seedrandom from 'seedrandom';
-import { AlertEvent, EndpointEvent, EndpointMetadata, OSFields } from './types';
+import { AlertEvent, EndpointEvent, EndpointMetadata, OSFields, HostFields } from './types';
 
 export type Event = AlertEvent | EndpointEvent;
 
@@ -67,31 +67,51 @@ const FILE_OPERATIONS: string[] = ['creation', 'open', 'rename', 'execution', 'd
 // These are from the v1 schemas and aren't all valid ECS event categories, still in flux
 const OTHER_EVENT_CATEGORIES: string[] = ['driver', 'file', 'library', 'network', 'registry'];
 
+interface HostInfo {
+  agent: {
+    version: string;
+    id: string;
+  };
+  host: HostFields;
+  endpoint: {
+    policy: {
+      id: string;
+    };
+  };
+}
+
 export class EndpointDocGenerator {
-  agentId: string;
-  hostId: string;
-  hostname: string;
-  macAddress: string[];
-  ip: string[];
-  agentVersion: string;
-  os: OSFields;
-  policy: { name: string; id: string };
+  commonInfo: HostInfo;
   random: seedrandom.prng;
 
   constructor(seed = Math.random().toString()) {
     this.random = seedrandom(seed);
-    this.hostId = this.seededUUIDv4();
-    this.agentId = this.seededUUIDv4();
-    this.hostname = this.randomHostname();
-    this.ip = this.randomArray(3, () => this.randomIP());
-    this.macAddress = this.randomArray(3, () => this.randomMac());
-    this.agentVersion = this.randomVersion();
-    this.os = this.randomChoice(OS);
-    this.policy = this.randomChoice(POLICIES);
+    this.commonInfo = this.createHostData();
   }
 
-  public randomizeIPs() {
-    this.ip = this.randomArray(3, () => this.randomIP());
+  // This function will create new values for all the host fields, so documents from a different endpoint can be created
+  // This provides a convenient way to make documents from multiple endpoints that are all tied to a single seed value
+  public randomizeHostData() {
+    this.commonInfo = this.createHostData();
+  }
+
+  private createHostData(): HostInfo {
+    return {
+      agent: {
+        version: this.randomVersion(),
+        id: this.seededUUIDv4(),
+      },
+      host: {
+        id: this.seededUUIDv4(),
+        hostname: this.randomHostname(),
+        ip: this.randomArray(3, () => this.randomIP()),
+        mac: this.randomArray(3, () => this.randomMac()),
+        os: this.randomChoice(OS),
+      },
+      endpoint: {
+        policy: this.randomChoice(POLICIES),
+      },
+    };
   }
 
   public generateEndpointMetadata(ts = new Date().getTime()): EndpointMetadata {
@@ -100,22 +120,7 @@ export class EndpointDocGenerator {
       event: {
         created: ts,
       },
-      endpoint: {
-        policy: {
-          id: this.policy.id,
-        },
-      },
-      agent: {
-        version: this.agentVersion,
-        id: this.agentId,
-      },
-      host: {
-        id: this.hostId,
-        hostname: this.hostname,
-        ip: this.ip,
-        mac: this.macAddress,
-        os: this.os,
-      },
+      ...this.commonInfo,
     };
   }
 
@@ -125,11 +130,8 @@ export class EndpointDocGenerator {
     parentEntityID?: string
   ): AlertEvent {
     return {
+      ...this.commonInfo,
       '@timestamp': ts,
-      agent: {
-        id: this.agentId,
-        version: this.agentVersion,
-      },
       event: {
         action: this.randomChoice(FILE_OPERATIONS),
         kind: 'alert',
@@ -139,11 +141,6 @@ export class EndpointDocGenerator {
         module: 'endpoint',
         type: 'creation',
       },
-      endpoint: {
-        policy: {
-          id: this.policy.id,
-        },
-      },
       file: {
         owner: 'SYSTEM',
         name: 'fake_malware.exe',
@@ -169,13 +166,6 @@ export class EndpointDocGenerator {
         },
         temp_file_path: 'C:/temp/fake_malware.exe',
       },
-      host: {
-        id: this.hostId,
-        hostname: this.hostname,
-        ip: this.ip,
-        mac: this.macAddress,
-        os: this.os,
-      },
       process: {
         pid: 2,
         name: 'malware writer',
@@ -243,11 +233,7 @@ export class EndpointDocGenerator {
   public generateEvent(options: EventOptions = {}): EndpointEvent {
     return {
       '@timestamp': options.timestamp ? options.timestamp : new Date().getTime(),
-      agent: {
-        id: this.agentId,
-        version: this.agentVersion,
-        type: 'endpoint',
-      },
+      agent: { ...this.commonInfo.agent, type: 'endgame' },
       ecs: {
         version: '1.4.0',
       },
@@ -257,13 +243,7 @@ export class EndpointDocGenerator {
         type: options.eventType ? options.eventType : 'creation',
         id: this.seededUUIDv4(),
       },
-      host: {
-        id: this.hostId,
-        hostname: this.hostname,
-        ip: this.ip,
-        mac: this.macAddress,
-        os: this.os,
-      },
+      host: this.commonInfo.host,
       process: {
         entity_id: options.entityID ? options.entityID : this.randomString(10),
         parent: options.parentEntityID ? { entity_id: options.parentEntityID } : undefined,
@@ -323,14 +303,13 @@ export class EndpointDocGenerator {
     percentNodesWithRelated = 100,
     percentChildrenTerminated = 100
   ): Event[] {
-    let events: Event[] = [root];
+    let events: Event[] = [];
     let parents = [root];
     let timestamp = root['@timestamp'];
     for (let i = 0; i < generations; i++) {
       const newParents: EndpointEvent[] = [];
       parents.forEach(element => {
-        // const numChildren = randomN(maxChildrenPerNode);
-        const numChildren = maxChildrenPerNode;
+        const numChildren = this.randomN(maxChildrenPerNode);
         for (let j = 0; j < numChildren; j++) {
           timestamp = timestamp + 1000;
           const child = this.generateEvent({

x-pack/plugins/endpoint/package.json

@@ -4,10 +4,11 @@
   "version": "0.0.0",
   "private": true,
   "license": "Elastic-License",
-  "scripts": {},
+  "scripts": {
+    "test:generate": "ts-node --project scripts/cli_tsconfig.json scripts/resolver_generator.ts"
+  },
   "dependencies": {
-    "react-redux": "^7.1.0",
-    "seedrandom": "^3.0.5"
+    "react-redux": "^7.1.0"
   },
   "devDependencies": {
     "@types/seedrandom": ">=2.0.0 <4.0.0",

x-pack/plugins/endpoint/public/applications/endpoint/store/managing/index.test.ts

@@ -20,7 +20,7 @@ describe('endpoint_list store concerns', () => {
     dispatch = store.dispatch;
   };
   const generateEndpoint = (): EndpointMetadata => {
-    return generator.generateEndpointMetadata(new Date().getTime());
+    return generator.generateEndpointMetadata();
   };
   const loadDataToStore = () => {
     dispatch({

x-pack/plugins/endpoint/public/applications/endpoint/store/managing/middleware.test.ts

@@ -27,7 +27,7 @@ describe('endpoint list saga', () => {
   const generator = new EndpointDocGenerator();
   // https://github.com/elastic/endpoint-app-team/issues/131
   const generateEndpoint = (): EndpointMetadata => {
-    return generator.generateEndpointMetadata(new Date().getTime());
+    return generator.generateEndpointMetadata();
   };
 
   let history: History<never>;

x-pack/plugins/endpoint/scripts/README.md

@@ -0,0 +1,46 @@
+This script makes it easy to create the endpoint metadata, alert, and event documents needed to test Resolver in Kibana.
+The default behavior is to create 1 endpoint with 1 alert and a moderate number of events (random, typically on the order of 20).
+A seed value can be provided as a string for the random number generator for repeatable behavior, useful for demos etc.
+Use the `-d` option if you want to delete and remake the indices, otherwise it will add documents to existing indices.
+
+The sample data generator script depends on ts-node, install with npm:
+
+```npm install -g ts-node```
+
+Example command sequence to get ES and kibana running with sample data after installing ts-node:
+
+```yarn es snapshot``` -> starts ES
+
+```npx yarn start --xpack.endpoint.enabled=true --no-base-path``` -> starts kibana
+
+```cd ~/path/to/kibana/x-pack/plugins/endpoint```
+
+```yarn test:generate --auth elastic:changeme``` -> run the resolver_generator.ts script
+
+Resolver generator CLI options:
+```--help                      Show help                                [boolean]
+  --seed, -s                  random seed to use for document generator [string]
+  --node, -n                  elasticsearch node url
+                                     [string] [default: "http://localhost:9200"]
+  --eventIndex, --ei          index to store events in
+                                         [string] [default: "events-endpoint-1"]
+  --metadataIndex, --mi       index to store endpoint metadata in
+                                          [string] [default: "endpoint-agent-1"]
+  --auth                      elasticsearch username and password, separated by
+                              a colon                                   [string]
+  --ancestors, --anc          number of ancestors of origin to create
+                                                           [number] [default: 3]
+  --generations, --gen        number of child generations to create
+                                                           [number] [default: 3]
+  --children, --ch            maximum number of children per node
+                                                           [number] [default: 3]
+  --relatedEvents, --related  number of related events to create for each
+                              process event                [number] [default: 5]
+  --percentWithRelated, --pr  percent of process events to add related events to
+                                                          [number] [default: 30]
+  --percentTerminated, --pt   percent of process events to add termination event
+                              for                         [number] [default: 30]
+  --numEndpoints, --ne        number of different endpoints to generate alerts
+                              for                          [number] [default: 1]
+  --alertsPerEndpoint, --ape  number of resolver trees to make for each endpoint
+                                                           [number] [default: 1]```

x-pack/plugins/endpoint/scripts/cli_tsconfig.json

@@ -0,0 +1,8 @@
+{
+    "extends": "../../../tsconfig.json",
+    "compilerOptions": {
+      "target": "es2019",
+      "resolveJsonModule": true
+    }
+  }
+