Skip to content

Commit

Permalink
[SIEM] optimize hosts query (#49409)
Browse files Browse the repository at this point in the history
  • Loading branch information
stephmilovic authored Oct 28, 2019
1 parent 15831b2 commit bd265d7
Show file tree
Hide file tree
Showing 6 changed files with 84 additions and 49 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -49,41 +49,33 @@ describe('hosts elasticsearch_adapter', () => {
describe('#formatHostsData', () => {
const buckets: HostAggEsItem = {
key: 'zeek-london',
host_os_version: {
buckets: [
{
key: '18.04.2 LTS (Bionic Beaver)',
doc_count: 1467783,
timestamp: { value: 1554516350177, value_as_string: '2019-04-06T02:05:50.177Z' },
os: {
hits: {
total: {
value: 242338,
relation: 'eq',
},
],
},
host_os_name: {
buckets: [
{
key: 'Ubuntu',
doc_count: 1467783,
timestamp: { value: 1554516350177, value_as_string: '2019-04-06T02:05:50.177Z' },
},
],
},
host_name: {
buckets: [
{
key: 'zeek-london',
doc_count: 1467783,
timestamp: { value: 1554516350177, value_as_string: '2019-04-06T02:05:50.177Z' },
},
],
},
host_id: {
buckets: [
{
key: '7c21f5ed03b04d0299569d221fe18bbc',
doc_count: 1467783,
timestamp: { value: 1554516350177, value_as_string: '2019-04-06T02:05:50.177Z' },
},
],
max_score: null,
hits: [
{
_index: 'auditbeat-8.0.0-2019.09.06-000022',
_id: 'dl0T_m0BHe9nqdOiF2A8',
_score: null,
_source: {
host: {
os: {
kernel: '5.0.0-1013-gcp',
name: 'Ubuntu',
family: 'debian',
version: '18.04.2 LTS (Bionic Beaver)',
platform: 'ubuntu',
},
},
},
sort: [1571925726017],
},
],
},
},
};

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,6 @@ export const formatHostEdgesData = (fields: readonly string[], bucket: HostAggEs
const hostId = get('key', bucket);
flattenedFields.node._id = hostId || null;
flattenedFields.cursor.value = hostId || '';

const fieldValue = getHostFieldValue(fieldName, bucket);
if (fieldValue != null) {
return set(`node.${fieldName}`, fieldValue, flattenedFields);
Expand Down Expand Up @@ -164,6 +163,15 @@ const getHostFieldValue = (fieldName: string, bucket: HostAggEsItem): string | s
} else if (has(aggField, bucket)) {
const valueObj: HostValue = get(aggField, bucket);
return valueObj.value_as_string;
} else if (['host.name', 'host.os.name', 'host.os.version'].includes(fieldName)) {
switch (fieldName) {
case 'host.name':
return get('key', bucket) || null;
case 'host.os.name':
return get('os.hits.hits[0]._source.host.os.name', bucket) || null;
case 'host.os.version':
return get('os.hits.hits[0]._source.host.os.version', bucket) || null;
}
}
return null;
};
Expand Down
4 changes: 2 additions & 2 deletions x-pack/legacy/plugins/siem/server/lib/hosts/helpers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,12 @@ export const buildFieldsTermAggregation = (esFields: readonly string[]): Aggrega
esFields.reduce<AggregationRequest>(
(res, field) => ({
...res,
...getAggregationTypeFromField(field),
...getTermsAggregationTypeFromField(field),
}),
{}
);

const getAggregationTypeFromField = (field: string): AggregationRequest => {
const getTermsAggregationTypeFromField = (field: string): AggregationRequest => {
return {
[field.replace(/\./g, '_')]: {
terms: {
Expand Down
23 changes: 15 additions & 8 deletions x-pack/legacy/plugins/siem/server/lib/hosts/query.hosts.dsl.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,8 @@

import { Direction, HostsFields, HostsSortField } from '../../graphql/types';
import { assertUnreachable, createQueryFilterClauses } from '../../utils/build_query';
import { reduceFields } from '../../utils/build_query/reduce_fields';
import { hostFieldsMap } from '../ecs_fields';

import { HostsRequestOptions } from '.';
import { buildFieldsTermAggregation } from './helpers';

export const buildHostsQuery = ({
defaultIndex,
Expand All @@ -23,8 +20,6 @@ export const buildHostsQuery = ({
},
timerange: { from, to },
}: HostsRequestOptions) => {
const esFields = reduceFields(fields, hostFieldsMap);

const filter = [
...createQueryFilterClauses(filterQuery),
{
Expand All @@ -50,9 +45,21 @@ export const buildHostsQuery = ({
terms: { size: querySize, field: 'host.name', order: getQueryOrder(sort) },
aggs: {
lastSeen: { max: { field: '@timestamp' } },
...buildFieldsTermAggregation(
esFields.filter(field => !['@timestamp', '_id'].includes(field))
),
os: {
top_hits: {
size: 1,
sort: [
{
'@timestamp': {
order: 'desc',
},
},
],
_source: {
includes: ['host.os.*'],
},
},
},
},
},
},
Expand Down
25 changes: 21 additions & 4 deletions x-pack/legacy/plugins/siem/server/lib/hosts/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,13 @@ import {
HostItem,
HostsData,
HostsSortField,
Maybe,
OsEcsFields,
SourceConfiguration,
TimerangeInput,
} from '../../graphql/types';
import { FrameworkRequest, RequestOptionsPaginated } from '../framework';
import { Hit, Hits, SearchHit } from '../types';
import { Hit, Hits, SearchHit, TotalValue } from '../types';

export interface HostsAdapter {
getHosts(req: FrameworkRequest, options: HostsRequestOptions): Promise<HostsData>;
Expand Down Expand Up @@ -71,23 +73,38 @@ export interface HostBuckets {
buckets: HostBucketItem[];
}

export interface HostOsHitsItem {
hits: {
total: TotalValue | number;
max_score: number | null;
hits: Array<{
_source: { host: { os: Maybe<OsEcsFields> } };
sort?: [number];
_index?: string;
_type?: string;
_id?: string;
_score?: number | null;
}>;
};
}

export interface HostAggEsItem {
cloud_instance_id?: HostBuckets;
cloud_machine_type?: HostBuckets;
cloud_provider?: HostBuckets;
cloud_region?: HostBuckets;
key?: string;
firstSeen?: HostValue;
lastSeen?: HostValue;
host_architecture?: HostBuckets;
host_id?: HostBuckets;
host_ip?: HostBuckets;
host_mac?: HostBuckets;
host_name?: HostBuckets;
host_os?: HostBuckets;
host_os_name?: HostBuckets;
host_os_version?: HostBuckets;
host_type?: HostBuckets;
key?: string;
lastSeen?: HostValue;
os?: HostOsHitsItem;
}

export interface HostEsData extends SearchHit {
Expand Down
11 changes: 11 additions & 0 deletions x-pack/legacy/plugins/siem/server/lib/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -193,5 +193,16 @@ export interface AggregationRequest {
};
};
};
top_hits?: {
size?: number;
sort?: Array<{
[aggSortField: string]: {
order: SortRequestDirection;
};
}>;
_source: {
includes: string[];
};
};
};
}

0 comments on commit bd265d7

Please sign in to comment.