Skip to content

Commit

Permalink
[Security Solution] Refactor resolver children _source (#77343)
Browse files Browse the repository at this point in the history 
* Moving generator to safe type version

* Finished generator and alert

* Gzipping again

* Finishing type conversions for backend

* Trying to cast front end tests back to unsafe type for now

* Working reducer tests

* Adding more comments and fixing alert type

* Restoring resolver test data

* Updating snapshot with timestamp info

* Getting the models figured out

* Event models type fixes

* Adding more comments

* Fixing more comments

* Adding comments
  • Loading branch information
@jonathan-buttner
jonathan-buttner authored Sep 16, 2020
1 parent 3688df2 commit 17fec25
Show file tree
Hide file tree
Showing 5 changed files with 229 additions and 34 deletions.
176 changes: 156 additions & 20 deletions x-pack/plugins/security_solution/common/endpoint/models/event.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,20 +3,24 @@
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import {
LegacyEndpointEvent,
ResolverEvent,
SafeResolverEvent,
SafeLegacyEndpointEvent,
} from '../types';
import { LegacyEndpointEvent, ResolverEvent, SafeResolverEvent, ECSField } from '../types';
import { firstNonNullValue, hasValue, values } from './ecs_safety_helpers';

/**
* Legacy events will define the `endgame` object. This is used to narrow a ResolverEvent.
*/
interface LegacyEvent {
endgame?: object;
}

/*
* Determine if a `ResolverEvent` is the legacy variety. Can be used to narrow `ResolverEvent` to `LegacyEndpointEvent`.
* Determine if a higher level event type is the legacy variety. Can be used to narrow an event type.
* T optionally defines an `endgame` object field used for determining the type of event. If T doesn't contain the
* `endgame` field it will serve as the narrowed type.
*/
export function isLegacyEventSafeVersion(
event: SafeResolverEvent
): event is SafeLegacyEndpointEvent {
export function isLegacyEventSafeVersion<T extends LegacyEvent>(
event: LegacyEvent | {}
): event is T {
return 'endgame' in event && event.endgame !== undefined;
}

Expand All @@ -27,7 +31,30 @@ export function isLegacyEvent(event: ResolverEvent): event is LegacyEndpointEven
return (event as LegacyEndpointEvent).endgame !== undefined;
}

export function isProcessRunning(event: SafeResolverEvent): boolean {
/**
* Minimum fields needed from the `SafeResolverEvent` type for the function below to operate correctly.
*/
type ProcessRunningFields = Partial<
| {
endgame: object;
event: Partial<{
type: ECSField<string>;
action: ECSField<string>;
}>;
}
| {
event: Partial<{
type: ECSField<string>;
}>;
}
>;

/**
* Checks if an event describes a process as running (whether it was started, already running, or changed)
*
* @param event a document to check for running fields
*/
export function isProcessRunning(event: ProcessRunningFields): boolean {
if (isLegacyEventSafeVersion(event)) {
return (
hasValue(event.event?.type, 'process_start') ||
Expand All @@ -43,15 +70,26 @@ export function isProcessRunning(event: SafeResolverEvent): boolean {
);
}

export function timestampSafeVersion(event: SafeResolverEvent): undefined | number {
/**
* Minimum fields needed from the `SafeResolverEvent` type for the function below to operate correctly.
*/
type TimestampFields = Pick<SafeResolverEvent, '@timestamp'>;

/**
* Extracts the first non null value from the `@timestamp` field in the document. Returns undefined if the field doesn't
* exist in the document.
*
* @param event a document from ES
*/
export function timestampSafeVersion(event: TimestampFields): undefined | number {
return firstNonNullValue(event?.['@timestamp']);
}

/**
* The `@timestamp` for the event, as a `Date` object.
* If `@timestamp` couldn't be parsed as a `Date`, returns `undefined`.
*/
export function timestampAsDateSafeVersion(event: SafeResolverEvent): Date | undefined {
export function timestampAsDateSafeVersion(event: TimestampFields): Date | undefined {
const value = timestampSafeVersion(event);
if (value === undefined) {
return undefined;
Expand Down Expand Up @@ -93,9 +131,30 @@ export function eventId(event: ResolverEvent): number | undefined | string {
return event.event.id;
}

export function eventSequence(event: SafeResolverEvent): number | undefined {
/**
* Minimum fields needed from the `SafeResolverEvent` type for the function below to operate correctly.
*/
type EventSequenceFields = Partial<
| {
endgame: Partial<{
serial_event_id: ECSField<number>;
}>;
}
| {
event: Partial<{
sequence: ECSField<number>;
}>;
}
>;

/**
* Extract the first non null event sequence value from a document. Returns undefined if the field doesn't exist in the document.
*
* @param event a document from ES
*/
export function eventSequence(event: EventSequenceFields): number | undefined {
if (isLegacyEventSafeVersion(event)) {
return firstNonNullValue(event.endgame.serial_event_id);
return firstNonNullValue(event.endgame?.serial_event_id);
}
return firstNonNullValue(event.event?.sequence);
}
Expand All @@ -113,7 +172,29 @@ export function entityId(event: ResolverEvent): string {
return event.process.entity_id;
}

export function entityIDSafeVersion(event: SafeResolverEvent): string | undefined {
/**
* Minimum fields needed from the `SafeResolverEvent` type for the function below to operate correctly.
*/
type EntityIDFields = Partial<
| {
endgame: Partial<{
unique_pid: ECSField<number>;
}>;
}
| {
process: Partial<{
entity_id: ECSField<string>;
}>;
}
>;

/**
* Extract the first non null value from either the `entity_id` or `unique_pid` depending on the document type. Returns
* undefined if the field doesn't exist in the document.
*
* @param event a document from ES
*/
export function entityIDSafeVersion(event: EntityIDFields): string | undefined {
if (isLegacyEventSafeVersion(event)) {
return event.endgame?.unique_pid === undefined
? undefined
Expand All @@ -130,14 +211,59 @@ export function parentEntityId(event: ResolverEvent): string | undefined {
return event.process.parent?.entity_id;
}

export function parentEntityIDSafeVersion(event: SafeResolverEvent): string | undefined {
/**
* Minimum fields needed from the `SafeResolverEvent` type for the function below to operate correctly.
*/
type ParentEntityIDFields = Partial<
| {
endgame: Partial<{
unique_ppid: ECSField<number>;
}>;
}
| {
process: Partial<{
parent: Partial<{
entity_id: ECSField<string>;
}>;
}>;
}
>;

/**
* Extract the first non null value from either the `parent.entity_id` or `unique_ppid` depending on the document type. Returns
* undefined if the field doesn't exist in the document.
*
* @param event a document from ES
*/
export function parentEntityIDSafeVersion(event: ParentEntityIDFields): string | undefined {
if (isLegacyEventSafeVersion(event)) {
return String(firstNonNullValue(event.endgame.unique_ppid));
return String(firstNonNullValue(event.endgame?.unique_ppid));
}
return firstNonNullValue(event.process?.parent?.entity_id);
}

export function ancestryArray(event: SafeResolverEvent): string[] | undefined {
/**
* Minimum fields needed from the `SafeResolverEvent` type for the function below to operate correctly.
*/
type AncestryArrayFields = Partial<
| {
endgame: object;
}
| {
process: Partial<{
Ext: Partial<{
ancestry: ECSField<string>;
}>;
}>;
}
>;

/**
* Extracts all ancestry array from a document if it exists.
*
* @param event an ES document
*/
export function ancestryArray(event: AncestryArrayFields): string[] | undefined {
if (isLegacyEventSafeVersion(event)) {
return undefined;
}
Expand All @@ -146,7 +272,17 @@ export function ancestryArray(event: SafeResolverEvent): string[] | undefined {
return values(event.process?.Ext?.ancestry);
}

export function getAncestryAsArray(event: SafeResolverEvent | undefined): string[] {
/**
* Minimum fields needed from the `SafeResolverEvent` type for the function below to operate correctly.
*/
type GetAncestryArrayFields = AncestryArrayFields & ParentEntityIDFields;

/**
* Returns an array of strings representing the ancestry for a process.
*
* @param event an ES document
*/
export function getAncestryAsArray(event: GetAncestryArrayFields | undefined): string[] {
if (!event) {
return [];
}
Expand Down
68 changes: 65 additions & 3 deletions x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/children.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,15 +4,59 @@
* you may not use this file except in compliance with the Elastic License.
*/
import { SearchResponse } from 'elasticsearch';
import { SafeResolverEvent } from '../../../../../common/endpoint/types';
import { ECSField } from '../../../../../common/endpoint/types';
import { ResolverQuery } from './base';
import { ChildrenPaginationBuilder } from '../utils/children_pagination';
import { JsonObject } from '../../../../../../../../src/plugins/kibana_utils/common';

/**
* This type represents the document returned from ES for a legacy event when using the ChildrenQuery to fetch legacy events.
* It contains only the necessary fields that the children api needs to process the results before
* it requests the full lifecycle information for the children in a later query.
*/
export type LegacyChildEvent = Partial<{
'@timestamp': ECSField<number>;
event: Partial<{
type: ECSField<string>;
action: ECSField<string>;
}>;
endgame: Partial<{
serial_event_id: ECSField<number>;
unique_pid: ECSField<number>;
unique_ppid: ECSField<number>;
}>;
}>;

/**
* This type represents the document returned from ES for an event when using the ChildrenQuery to fetch legacy events.
* It contains only the necessary fields that the children api needs to process the results before
* it requests the full lifecycle information for the children in a later query.
*/
export type EndpointChildEvent = Partial<{
'@timestamp': ECSField<number>;
event: Partial<{
type: ECSField<string>;
sequence: ECSField<number>;
}>;
process: Partial<{
entity_id: ECSField<string>;
parent: Partial<{
entity_id: ECSField<string>;
}>;
Ext: Partial<{
ancestry: ECSField<string>;
}>;
}>;
}>;

export type ChildEvent = EndpointChildEvent | LegacyChildEvent;

/**
* Builds a query for retrieving descendants of a node.
* The first type `ChildEvent[]` represents the final formatted result. The second type `ChildEvent` defines the type
* used in the `SearchResponse<T>` field returned from the ES query.
*/
export class ChildrenQuery extends ResolverQuery<SafeResolverEvent[]> {
export class ChildrenQuery extends ResolverQuery<ChildEvent[], ChildEvent> {
constructor(
private readonly pagination: ChildrenPaginationBuilder,
indexPattern: string | string[],
Expand All @@ -24,6 +68,14 @@ export class ChildrenQuery extends ResolverQuery<SafeResolverEvent[]> {
protected legacyQuery(endpointID: string, uniquePIDs: string[]): JsonObject {
const paginationFields = this.pagination.buildQueryFields('endgame.serial_event_id');
return {
_source: [
'@timestamp',
'endgame.serial_event_id',
'endgame.unique_pid',
'endgame.unique_ppid',
'event.type',
'event.action',
],
collapse: {
field: 'endgame.unique_pid',
},
Expand Down Expand Up @@ -64,8 +116,18 @@ export class ChildrenQuery extends ResolverQuery<SafeResolverEvent[]> {
}

protected query(entityIDs: string[]): JsonObject {
// we don't have to include the `event.id` in the source response because it is not needed for processing
// the data returned by ES, it is only used for breaking ties when ES is doing the search
const paginationFields = this.pagination.buildQueryFields('event.id');
return {
_source: [
'@timestamp',
'event.type',
'event.sequence',
'process.entity_id',
'process.parent.entity_id',
'process.Ext.ancestry',
],
/**
* Using collapse here will only return a single event per occurrence of a process.entity_id. The events are sorted
* based on timestamp in ascending order so it will be the first event that ocurred. The actual type of event that
Expand Down Expand Up @@ -126,7 +188,7 @@ export class ChildrenQuery extends ResolverQuery<SafeResolverEvent[]> {
};
}

formatResponse(response: SearchResponse<SafeResolverEvent>): SafeResolverEvent[] {
formatResponse(response: SearchResponse<ChildEvent>): ChildEvent[] {
return this.getResults(response);
}
}
10 changes: 4 additions & 6 deletions x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/children_helper.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import {
} from '../../../../../common/endpoint/types';
import { createChild } from './node';
import { ChildrenPaginationBuilder } from './children_pagination';
import { ChildEvent } from '../queries/children';

/**
* This class helps construct the children structure when building a resolver tree.
Expand Down Expand Up @@ -86,15 +87,12 @@ export class ChildrenNodesHelper {
* @param queriedNodes the entity_ids of the nodes that returned these start events
* @param startEvents an array of start events returned by ES
*/
addStartEvents(
queriedNodes: Set<string>,
startEvents: SafeResolverEvent[]
): Set<string> | undefined {
addStartEvents(queriedNodes: Set<string>, startEvents: ChildEvent[]): Set<string> | undefined {
let largestAncestryArray = 0;
const nodesToQueryNext: Map<number, Set<string>> = new Map();
const nonLeafNodes: Set<SafeResolverChildNode> = new Set();

const isDistantGrandchild = (event: SafeResolverEvent) => {
const isDistantGrandchild = (event: ChildEvent) => {
const ancestry = getAncestryAsArray(event);
return ancestry.length > 0 && queriedNodes.has(ancestry[ancestry.length - 1]);
};
Expand Down Expand Up @@ -161,7 +159,7 @@ export class ChildrenNodesHelper {
return nodesToQueryNext.get(largestAncestryArray);
}

private setPaginationForNodes(nodes: Set<string>, startEvents: SafeResolverEvent[]) {
private setPaginationForNodes(nodes: Set<string>, startEvents: ChildEvent[]) {
for (const nodeEntityID of nodes.values()) {
const cachedNode = this.entityToNodeCache.get(nodeEntityID);
if (cachedNode) {
Expand Down
Loading

0 comments on commit 17fec25

Please sign in to comment.