Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[7.x] [Endpoint] Sample data generator CLI script (#59952) #60374

Merged
merged 1 commit into from
Mar 17, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion x-pack/plugins/endpoint/common/generate_data.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -151,7 +151,7 @@ describe('data generator', () => {
const timestamp = new Date().getTime();
const root = generator.generateEvent({ timestamp });
const generations = 2;
const events = generator.generateDescendantsTree(root, generations);
const events = [root, ...generator.generateDescendantsTree(root, generations)];
const rootNode = buildResolverTree(events);
const visitedEvents = countResolverEvents(rootNode, generations);
expect(visitedEvents).toEqual(events.length);
Expand Down
111 changes: 45 additions & 66 deletions x-pack/plugins/endpoint/common/generate_data.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@

import uuid from 'uuid';
import seedrandom from 'seedrandom';
import { AlertEvent, EndpointEvent, EndpointMetadata, OSFields } from './types';
import { AlertEvent, EndpointEvent, EndpointMetadata, OSFields, HostFields } from './types';

export type Event = AlertEvent | EndpointEvent;

Expand Down Expand Up @@ -67,31 +67,51 @@ const FILE_OPERATIONS: string[] = ['creation', 'open', 'rename', 'execution', 'd
// These are from the v1 schemas and aren't all valid ECS event categories, still in flux
const OTHER_EVENT_CATEGORIES: string[] = ['driver', 'file', 'library', 'network', 'registry'];

interface HostInfo {
agent: {
version: string;
id: string;
};
host: HostFields;
endpoint: {
policy: {
id: string;
};
};
}

export class EndpointDocGenerator {
agentId: string;
hostId: string;
hostname: string;
macAddress: string[];
ip: string[];
agentVersion: string;
os: OSFields;
policy: { name: string; id: string };
commonInfo: HostInfo;
random: seedrandom.prng;

constructor(seed = Math.random().toString()) {
this.random = seedrandom(seed);
this.hostId = this.seededUUIDv4();
this.agentId = this.seededUUIDv4();
this.hostname = this.randomHostname();
this.ip = this.randomArray(3, () => this.randomIP());
this.macAddress = this.randomArray(3, () => this.randomMac());
this.agentVersion = this.randomVersion();
this.os = this.randomChoice(OS);
this.policy = this.randomChoice(POLICIES);
this.commonInfo = this.createHostData();
}

public randomizeIPs() {
this.ip = this.randomArray(3, () => this.randomIP());
// This function will create new values for all the host fields, so documents from a different endpoint can be created
// This provides a convenient way to make documents from multiple endpoints that are all tied to a single seed value
public randomizeHostData() {
this.commonInfo = this.createHostData();
}

private createHostData(): HostInfo {
return {
agent: {
version: this.randomVersion(),
id: this.seededUUIDv4(),
},
host: {
id: this.seededUUIDv4(),
hostname: this.randomHostname(),
ip: this.randomArray(3, () => this.randomIP()),
mac: this.randomArray(3, () => this.randomMac()),
os: this.randomChoice(OS),
},
endpoint: {
policy: this.randomChoice(POLICIES),
},
};
}

public generateEndpointMetadata(ts = new Date().getTime()): EndpointMetadata {
Expand All @@ -100,22 +120,7 @@ export class EndpointDocGenerator {
event: {
created: ts,
},
endpoint: {
policy: {
id: this.policy.id,
},
},
agent: {
version: this.agentVersion,
id: this.agentId,
},
host: {
id: this.hostId,
hostname: this.hostname,
ip: this.ip,
mac: this.macAddress,
os: this.os,
},
...this.commonInfo,
};
}

Expand All @@ -125,11 +130,8 @@ export class EndpointDocGenerator {
parentEntityID?: string
): AlertEvent {
return {
...this.commonInfo,
'@timestamp': ts,
agent: {
id: this.agentId,
version: this.agentVersion,
},
event: {
action: this.randomChoice(FILE_OPERATIONS),
kind: 'alert',
Expand All @@ -139,11 +141,6 @@ export class EndpointDocGenerator {
module: 'endpoint',
type: 'creation',
},
endpoint: {
policy: {
id: this.policy.id,
},
},
file: {
owner: 'SYSTEM',
name: 'fake_malware.exe',
Expand All @@ -169,13 +166,6 @@ export class EndpointDocGenerator {
},
temp_file_path: 'C:/temp/fake_malware.exe',
},
host: {
id: this.hostId,
hostname: this.hostname,
ip: this.ip,
mac: this.macAddress,
os: this.os,
},
process: {
pid: 2,
name: 'malware writer',
Expand Down Expand Up @@ -243,11 +233,7 @@ export class EndpointDocGenerator {
public generateEvent(options: EventOptions = {}): EndpointEvent {
return {
'@timestamp': options.timestamp ? options.timestamp : new Date().getTime(),
agent: {
id: this.agentId,
version: this.agentVersion,
type: 'endpoint',
},
agent: { ...this.commonInfo.agent, type: 'endgame' },
ecs: {
version: '1.4.0',
},
Expand All @@ -257,13 +243,7 @@ export class EndpointDocGenerator {
type: options.eventType ? options.eventType : 'creation',
id: this.seededUUIDv4(),
},
host: {
id: this.hostId,
hostname: this.hostname,
ip: this.ip,
mac: this.macAddress,
os: this.os,
},
host: this.commonInfo.host,
process: {
entity_id: options.entityID ? options.entityID : this.randomString(10),
parent: options.parentEntityID ? { entity_id: options.parentEntityID } : undefined,
Expand Down Expand Up @@ -323,14 +303,13 @@ export class EndpointDocGenerator {
percentNodesWithRelated = 100,
percentChildrenTerminated = 100
): Event[] {
let events: Event[] = [root];
let events: Event[] = [];
let parents = [root];
let timestamp = root['@timestamp'];
for (let i = 0; i < generations; i++) {
const newParents: EndpointEvent[] = [];
parents.forEach(element => {
// const numChildren = randomN(maxChildrenPerNode);
const numChildren = maxChildrenPerNode;
const numChildren = this.randomN(maxChildrenPerNode);
for (let j = 0; j < numChildren; j++) {
timestamp = timestamp + 1000;
const child = this.generateEvent({
Expand Down
7 changes: 4 additions & 3 deletions x-pack/plugins/endpoint/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,11 @@
"version": "0.0.0",
"private": true,
"license": "Elastic-License",
"scripts": {},
"scripts": {
"test:generate": "ts-node --project scripts/cli_tsconfig.json scripts/resolver_generator.ts"
},
"dependencies": {
"react-redux": "^7.1.0",
"seedrandom": "^3.0.5"
"react-redux": "^7.1.0"
},
"devDependencies": {
"@types/seedrandom": ">=2.0.0 <4.0.0",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ describe('endpoint_list store concerns', () => {
dispatch = store.dispatch;
};
const generateEndpoint = (): EndpointMetadata => {
return generator.generateEndpointMetadata(new Date().getTime());
return generator.generateEndpointMetadata();
};
const loadDataToStore = () => {
dispatch({
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ describe('endpoint list saga', () => {
const generator = new EndpointDocGenerator();
// https://github.com/elastic/endpoint-app-team/issues/131
const generateEndpoint = (): EndpointMetadata => {
return generator.generateEndpointMetadata(new Date().getTime());
return generator.generateEndpointMetadata();
};

let history: History<never>;
Expand Down
46 changes: 46 additions & 0 deletions x-pack/plugins/endpoint/scripts/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
This script makes it easy to create the endpoint metadata, alert, and event documents needed to test Resolver in Kibana.
The default behavior is to create 1 endpoint with 1 alert and a moderate number of events (random, typically on the order of 20).
A seed value can be provided as a string for the random number generator for repeatable behavior, useful for demos etc.
Use the `-d` option if you want to delete and remake the indices, otherwise it will add documents to existing indices.

The sample data generator script depends on ts-node, install with npm:

```npm install -g ts-node```

Example command sequence to get ES and kibana running with sample data after installing ts-node:

```yarn es snapshot``` -> starts ES

```npx yarn start --xpack.endpoint.enabled=true --no-base-path``` -> starts kibana

```cd ~/path/to/kibana/x-pack/plugins/endpoint```

```yarn test:generate --auth elastic:changeme``` -> run the resolver_generator.ts script

Resolver generator CLI options:
```--help Show help [boolean]
--seed, -s random seed to use for document generator [string]
--node, -n elasticsearch node url
[string] [default: "http://localhost:9200"]
--eventIndex, --ei index to store events in
[string] [default: "events-endpoint-1"]
--metadataIndex, --mi index to store endpoint metadata in
[string] [default: "endpoint-agent-1"]
--auth elasticsearch username and password, separated by
a colon [string]
--ancestors, --anc number of ancestors of origin to create
[number] [default: 3]
--generations, --gen number of child generations to create
[number] [default: 3]
--children, --ch maximum number of children per node
[number] [default: 3]
--relatedEvents, --related number of related events to create for each
process event [number] [default: 5]
--percentWithRelated, --pr percent of process events to add related events to
[number] [default: 30]
--percentTerminated, --pt percent of process events to add termination event
for [number] [default: 30]
--numEndpoints, --ne number of different endpoints to generate alerts
for [number] [default: 1]
--alertsPerEndpoint, --ape number of resolver trees to make for each endpoint
[number] [default: 1]```
8 changes: 8 additions & 0 deletions x-pack/plugins/endpoint/scripts/cli_tsconfig.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
{
"extends": "../../../tsconfig.json",
"compilerOptions": {
"target": "es2019",
"resolveJsonModule": true
}
}

Loading