diff --git a/packages/kbn-securitysolution-io-ts-types/index.ts b/packages/kbn-securitysolution-io-ts-types/index.ts index 1eb62ebf494a..1b1e42e03c6f 100644 --- a/packages/kbn-securitysolution-io-ts-types/index.ts +++ b/packages/kbn-securitysolution-io-ts-types/index.ts @@ -25,6 +25,7 @@ export * from './src/non_empty_array'; export * from './src/non_empty_or_nullable_string_array'; export * from './src/non_empty_string_array'; export * from './src/non_empty_string'; +export * from './src/number_between_zero_and_one_inclusive'; export * from './src/only_false_allowed'; export * from './src/operator'; export * from './src/positive_integer_greater_than_zero'; diff --git a/packages/kbn-securitysolution-io-ts-types/src/number_between_zero_and_one_inclusive/index.test.ts b/packages/kbn-securitysolution-io-ts-types/src/number_between_zero_and_one_inclusive/index.test.ts new file mode 100644 index 000000000000..7373cc0e435f --- /dev/null +++ b/packages/kbn-securitysolution-io-ts-types/src/number_between_zero_and_one_inclusive/index.test.ts @@ -0,0 +1,85 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { pipe } from 'fp-ts/lib/pipeable'; +import { left } from 'fp-ts/lib/Either'; +import { NumberBetweenZeroAndOneInclusive } from '.'; +import { foldLeftRight, getPaths } from '@kbn/securitysolution-io-ts-utils'; + +describe('NumberBetweenZeroAndOneInclusive', () => { + test('it should validate 1', () => { + const payload = 1; + const decoded = NumberBetweenZeroAndOneInclusive.decode(payload); + const message = pipe(decoded, foldLeftRight); + + expect(getPaths(left(message.errors))).toEqual([]); + expect(message.schema).toEqual(payload); + }); + + test('it should validate a zero', () => { + const payload = 0; + const decoded = NumberBetweenZeroAndOneInclusive.decode(payload); + const message = pipe(decoded, foldLeftRight); + + expect(getPaths(left(message.errors))).toEqual([]); + expect(message.schema).toEqual(payload); + }); + + test('it should validate a float between 0 and 1', () => { + const payload = 0.58; + const decoded = NumberBetweenZeroAndOneInclusive.decode(payload); + const message = pipe(decoded, foldLeftRight); + + expect(getPaths(left(message.errors))).toEqual([]); + expect(message.schema).toEqual(payload); + }); + + test('it should NOT validate a negative number', () => { + const payload = -1; + const decoded = NumberBetweenZeroAndOneInclusive.decode(payload); + const message = pipe(decoded, foldLeftRight); + + expect(getPaths(left(message.errors))).toEqual([ + 'Invalid value "-1" supplied to "NumberBetweenZeroAndOneInclusive"', + ]); + expect(message.schema).toEqual({}); + }); + + test('it should NOT validate NaN', () => { + const payload = NaN; + const decoded = NumberBetweenZeroAndOneInclusive.decode(payload); + const message = pipe(decoded, foldLeftRight); + + expect(getPaths(left(message.errors))).toEqual([ + 'Invalid value "NaN" supplied to "NumberBetweenZeroAndOneInclusive"', + ]); + expect(message.schema).toEqual({}); + }); + + test('it should NOT validate Infinity', () => { + const payload = Infinity; + const decoded = NumberBetweenZeroAndOneInclusive.decode(payload); + const message = pipe(decoded, foldLeftRight); + + expect(getPaths(left(message.errors))).toEqual([ + 'Invalid value "Infinity" supplied to "NumberBetweenZeroAndOneInclusive"', + ]); + expect(message.schema).toEqual({}); + }); + + test('it should NOT validate a string', () => { + const payload = 'some string'; + const decoded = NumberBetweenZeroAndOneInclusive.decode(payload); + const message = pipe(decoded, foldLeftRight); + + expect(getPaths(left(message.errors))).toEqual([ + 'Invalid value "some string" supplied to "NumberBetweenZeroAndOneInclusive"', + ]); + expect(message.schema).toEqual({}); + }); +}); diff --git a/packages/kbn-securitysolution-io-ts-types/src/number_between_zero_and_one_inclusive/index.ts b/packages/kbn-securitysolution-io-ts-types/src/number_between_zero_and_one_inclusive/index.ts new file mode 100644 index 000000000000..41f96a0b3ac1 --- /dev/null +++ b/packages/kbn-securitysolution-io-ts-types/src/number_between_zero_and_one_inclusive/index.ts @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as t from 'io-ts'; +import { Either } from 'fp-ts/lib/Either'; + +/** + * Types a number between 0 and 1 inclusive. Useful for specifying a probability, weighting, etc. + */ +export const NumberBetweenZeroAndOneInclusive = new t.Type( + 'NumberBetweenZeroAndOneInclusive', + t.number.is, + (input, context): Either => { + return typeof input === 'number' && + !Number.isNaN(input) && + Number.isFinite(input) && + input >= 0 && + input <= 1 + ? t.success(input) + : t.failure(input, context); + }, + t.identity +); diff --git a/x-pack/plugins/security_solution/common/constants.ts b/x-pack/plugins/security_solution/common/constants.ts index ec3c86097117..2d88a0bb0066 100644 --- a/x-pack/plugins/security_solution/common/constants.ts +++ b/x-pack/plugins/security_solution/common/constants.ts @@ -41,6 +41,7 @@ export const DEFAULT_SIGNALS_INDEX = '.siem-signals' as const; export const DEFAULT_PREVIEW_INDEX = '.preview.alerts-security.alerts' as const; export const DEFAULT_LISTS_INDEX = '.lists' as const; export const DEFAULT_ITEMS_INDEX = '.items' as const; +export const DEFAULT_RISK_SCORE_PAGE_SIZE = 1000 as const; // The DEFAULT_MAX_SIGNALS value exists also in `x-pack/plugins/cases/common/constants.ts` // If either changes, engineer should ensure both values are updated export const DEFAULT_MAX_SIGNALS = 100 as const; @@ -314,6 +315,8 @@ export const RISK_SCORE_CREATE_INDEX = `${INTERNAL_RISK_SCORE_URL}/indices/creat export const RISK_SCORE_DELETE_INDICES = `${INTERNAL_RISK_SCORE_URL}/indices/delete`; export const RISK_SCORE_CREATE_STORED_SCRIPT = `${INTERNAL_RISK_SCORE_URL}/stored_scripts/create`; export const RISK_SCORE_DELETE_STORED_SCRIPT = `${INTERNAL_RISK_SCORE_URL}/stored_scripts/delete`; +export const RISK_SCORE_PREVIEW_URL = `${INTERNAL_RISK_SCORE_URL}/preview`; + /** * Internal detection engine routes */ diff --git a/x-pack/plugins/security_solution/common/experimental_features.ts b/x-pack/plugins/security_solution/common/experimental_features.ts index 87651cff4bf9..07d06e4be4e6 100644 --- a/x-pack/plugins/security_solution/common/experimental_features.ts +++ b/x-pack/plugins/security_solution/common/experimental_features.ts @@ -116,6 +116,11 @@ export const allowedExperimentalValues = Object.freeze({ * The flag doesn't have to be documented and has to be removed after the feature is ready to release. */ detectionsCoverageOverview: false, + + /** + * Enables experimental Entity Analytics HTTP endpoints + */ + riskScoringRoutesEnabled: false, }); type ExperimentalConfigKeys = Array; diff --git a/x-pack/plugins/security_solution/common/risk_engine/after_keys.test.ts b/x-pack/plugins/security_solution/common/risk_engine/after_keys.test.ts new file mode 100644 index 000000000000..46664cd992e8 --- /dev/null +++ b/x-pack/plugins/security_solution/common/risk_engine/after_keys.test.ts @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { pipe } from 'fp-ts/lib/pipeable'; +import { left } from 'fp-ts/lib/Either'; +import { foldLeftRight, getPaths } from '@kbn/securitysolution-io-ts-utils'; + +import { afterKeysSchema } from './after_keys'; + +describe('after_keys schema', () => { + it('allows an empty object', () => { + const payload = {}; + const decoded = afterKeysSchema.decode(payload); + const message = pipe(decoded, foldLeftRight); + + expect(getPaths(left(message.errors))).toEqual([]); + expect(message.schema).toEqual(payload); + }); + + it('allows a valid host key', () => { + const payload = { host: { 'host.name': 'hello' } }; + const decoded = afterKeysSchema.decode(payload); + const message = pipe(decoded, foldLeftRight); + + expect(getPaths(left(message.errors))).toEqual([]); + expect(message.schema).toEqual(payload); + }); + + it('allows a valid user key', () => { + const payload = { user: { 'user.name': 'hello' } }; + const decoded = afterKeysSchema.decode(payload); + const message = pipe(decoded, foldLeftRight); + + expect(getPaths(left(message.errors))).toEqual([]); + expect(message.schema).toEqual(payload); + }); + + it('allows both valid host and user keys', () => { + const payload = { user: { 'user.name': 'hello' }, host: { 'host.name': 'hello' } }; + const decoded = afterKeysSchema.decode(payload); + const message = pipe(decoded, foldLeftRight); + + expect(getPaths(left(message.errors))).toEqual([]); + expect(message.schema).toEqual(payload); + }); + + it('removes an unknown identifier key if used', () => { + const payload = { bad: 'key' }; + const decoded = afterKeysSchema.decode(payload); + const message = pipe(decoded, foldLeftRight); + + expect(getPaths(left(message.errors))).toEqual([]); + expect(message.schema).toEqual({}); + }); +}); diff --git a/x-pack/plugins/security_solution/common/risk_engine/after_keys.ts b/x-pack/plugins/security_solution/common/risk_engine/after_keys.ts new file mode 100644 index 000000000000..5c6e7a22025c --- /dev/null +++ b/x-pack/plugins/security_solution/common/risk_engine/after_keys.ts @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import * as t from 'io-ts'; + +const afterKeySchema = t.record(t.string, t.string); +export type AfterKeySchema = t.TypeOf; +export type AfterKey = AfterKeySchema; + +export const afterKeysSchema = t.exact( + t.partial({ + host: afterKeySchema, + user: afterKeySchema, + }) +); +export type AfterKeysSchema = t.TypeOf; +export type AfterKeys = AfterKeysSchema; diff --git a/x-pack/plugins/security_solution/common/risk_engine/identifier_types.ts b/x-pack/plugins/security_solution/common/risk_engine/identifier_types.ts new file mode 100644 index 000000000000..3741e321d4b0 --- /dev/null +++ b/x-pack/plugins/security_solution/common/risk_engine/identifier_types.ts @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import * as t from 'io-ts'; + +export const identifierTypeSchema = t.keyof({ user: null, host: null }); +export type IdentifierTypeSchema = t.TypeOf; +export type IdentifierType = IdentifierTypeSchema; diff --git a/x-pack/plugins/security_solution/common/risk_engine/index.ts b/x-pack/plugins/security_solution/common/risk_engine/index.ts new file mode 100644 index 000000000000..d22652bba881 --- /dev/null +++ b/x-pack/plugins/security_solution/common/risk_engine/index.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './after_keys'; +export * from './risk_weights'; +export * from './identifier_types'; diff --git a/x-pack/plugins/security_solution/common/risk_engine/risk_score_preview/request_schema.ts b/x-pack/plugins/security_solution/common/risk_engine/risk_score_preview/request_schema.ts new file mode 100644 index 000000000000..e76b0673c6d8 --- /dev/null +++ b/x-pack/plugins/security_solution/common/risk_engine/risk_score_preview/request_schema.ts @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import * as t from 'io-ts'; +import { DataViewId } from '../../detection_engine/rule_schema'; +import { afterKeysSchema } from '../after_keys'; +import { identifierTypeSchema } from '../identifier_types'; +import { riskWeightsSchema } from '../risk_weights/schema'; + +export const riskScorePreviewRequestSchema = t.exact( + t.partial({ + after_keys: afterKeysSchema, + data_view_id: DataViewId, + debug: t.boolean, + filter: t.unknown, + page_size: t.number, + identifier_type: identifierTypeSchema, + range: t.type({ + start: t.string, + end: t.string, + }), + weights: riskWeightsSchema, + }) +); +export type RiskScorePreviewRequestSchema = t.TypeOf; diff --git a/x-pack/plugins/security_solution/common/risk_engine/risk_weights/index.ts b/x-pack/plugins/security_solution/common/risk_engine/risk_weights/index.ts new file mode 100644 index 000000000000..8aa51f283cef --- /dev/null +++ b/x-pack/plugins/security_solution/common/risk_engine/risk_weights/index.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './types'; +export type { RiskWeight, RiskWeights, GlobalRiskWeight, RiskCategoryRiskWeight } from './schema'; diff --git a/x-pack/plugins/security_solution/common/risk_engine/risk_weights/schema.test.ts b/x-pack/plugins/security_solution/common/risk_engine/risk_weights/schema.test.ts new file mode 100644 index 000000000000..7052a4d781ce --- /dev/null +++ b/x-pack/plugins/security_solution/common/risk_engine/risk_weights/schema.test.ts @@ -0,0 +1,234 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { pipe } from 'fp-ts/lib/pipeable'; +import { left } from 'fp-ts/lib/Either'; +import { foldLeftRight, getPaths } from '@kbn/securitysolution-io-ts-utils'; + +import { riskWeightSchema } from './schema'; +import { RiskCategories, RiskWeightTypes } from './types'; + +describe('risk weight schema', () => { + let type: string; + + describe('allowed types', () => { + it('allows the global weight type', () => { + const payload = { + type: RiskWeightTypes.global, + host: 0.1, + }; + const decoded = riskWeightSchema.decode(payload); + const message = pipe(decoded, foldLeftRight); + + expect(getPaths(left(message.errors))).toEqual([]); + expect(message.schema).toEqual(payload); + }); + + it('allows the risk category weight type', () => { + const payload = { + type: RiskWeightTypes.global, + host: 0.1, + }; + const decoded = riskWeightSchema.decode(payload); + const message = pipe(decoded, foldLeftRight); + + expect(getPaths(left(message.errors))).toEqual([]); + expect(message.schema).toEqual(payload); + }); + + it('rejects an unknown weight type', () => { + const payload = { + type: 'unknown', + host: 0.1, + }; + const decoded = riskWeightSchema.decode(payload); + const message = pipe(decoded, foldLeftRight); + + expect(getPaths(left(message.errors)).length).toBeGreaterThan(0); + expect(message.schema).toEqual({}); + }); + }); + + describe('conditional fields', () => { + describe('global weights', () => { + beforeEach(() => { + type = RiskWeightTypes.global; + }); + + it('rejects if neither host nor user weight are specified', () => { + const payload = { type }; + const decoded = riskWeightSchema.decode(payload); + const message = pipe(decoded, foldLeftRight); + + expect(getPaths(left(message.errors))).toEqual([ + 'Invalid value "undefined" supplied to "host"', + 'Invalid value "undefined" supplied to "user"', + ]); + expect(message.schema).toEqual({}); + }); + + it('allows a single host weight', () => { + const payload = { type, host: 0.1 }; + const decoded = riskWeightSchema.decode(payload); + const message = pipe(decoded, foldLeftRight); + + expect(getPaths(left(message.errors))).toEqual([]); + expect(message.schema).toEqual(payload); + }); + + it('allows a single user weight', () => { + const payload = { type, user: 0.1 }; + const decoded = riskWeightSchema.decode(payload); + const message = pipe(decoded, foldLeftRight); + + expect(getPaths(left(message.errors))).toEqual([]); + expect(message.schema).toEqual(payload); + }); + + it('allows both a host and user weight', () => { + const payload = { type, host: 0.1, user: 0.5 }; + const decoded = riskWeightSchema.decode(payload); + const message = pipe(decoded, foldLeftRight); + + expect(getPaths(left(message.errors))).toEqual([]); + expect(message.schema).toEqual({ type, host: 0.1, user: 0.5 }); + }); + + it('rejects a weight outside of 0-1', () => { + const payload = { type, user: 55 }; + const decoded = riskWeightSchema.decode(payload); + const message = pipe(decoded, foldLeftRight); + + expect(getPaths(left(message.errors))).toContain('Invalid value "55" supplied to "user"'); + expect(message.schema).toEqual({}); + }); + + it('removes extra keys if specified', () => { + const payload = { + type, + host: 0.1, + value: 'superfluous', + extra: 'even more', + }; + const decoded = riskWeightSchema.decode(payload); + const message = pipe(decoded, foldLeftRight); + + expect(getPaths(left(message.errors))).toEqual([]); + expect(message.schema).toEqual({ type, host: 0.1 }); + }); + }); + + describe('risk category weights', () => { + beforeEach(() => { + type = RiskWeightTypes.riskCategory; + }); + + it('requires a value', () => { + const payload = { type, user: 0.1 }; + const decoded = riskWeightSchema.decode(payload); + const message = pipe(decoded, foldLeftRight); + + expect(getPaths(left(message.errors))).toEqual([ + 'Invalid value "undefined" supplied to "value"', + ]); + expect(message.schema).toEqual({}); + }); + + it('rejects if neither host nor user weight are specified', () => { + const payload = { type, value: RiskCategories.alerts }; + const decoded = riskWeightSchema.decode(payload); + const message = pipe(decoded, foldLeftRight); + + expect(getPaths(left(message.errors))).toEqual([ + 'Invalid value "undefined" supplied to "host"', + 'Invalid value "undefined" supplied to "user"', + ]); + expect(message.schema).toEqual({}); + }); + + it('allows a single host weight', () => { + const payload = { type, value: RiskCategories.alerts, host: 0.1 }; + const decoded = riskWeightSchema.decode(payload); + const message = pipe(decoded, foldLeftRight); + + expect(getPaths(left(message.errors))).toEqual([]); + expect(message.schema).toEqual(payload); + }); + + it('allows a single user weight', () => { + const payload = { type, value: RiskCategories.alerts, user: 0.1 }; + const decoded = riskWeightSchema.decode(payload); + const message = pipe(decoded, foldLeftRight); + + expect(getPaths(left(message.errors))).toEqual([]); + expect(message.schema).toEqual(payload); + }); + + it('allows both a host and user weight', () => { + const payload = { type, value: RiskCategories.alerts, user: 0.1, host: 0.5 }; + const decoded = riskWeightSchema.decode(payload); + const message = pipe(decoded, foldLeftRight); + + expect(getPaths(left(message.errors))).toEqual([]); + expect(message.schema).toEqual(payload); + }); + + it('rejects a weight outside of 0-1', () => { + const payload = { type, value: RiskCategories.alerts, host: -5 }; + const decoded = riskWeightSchema.decode(payload); + const message = pipe(decoded, foldLeftRight); + + expect(getPaths(left(message.errors))).toContain('Invalid value "-5" supplied to "host"'); + expect(message.schema).toEqual({}); + }); + + it('removes extra keys if specified', () => { + const payload = { + type, + value: RiskCategories.alerts, + host: 0.1, + extra: 'even more', + }; + const decoded = riskWeightSchema.decode(payload); + const message = pipe(decoded, foldLeftRight); + + expect(getPaths(left(message.errors))).toEqual([]); + expect(message.schema).toEqual({ type, value: RiskCategories.alerts, host: 0.1 }); + }); + + describe('allowed category values', () => { + it('allows the alerts type for a category', () => { + const payload = { + type, + value: RiskCategories.alerts, + host: 0.1, + }; + const decoded = riskWeightSchema.decode(payload); + const message = pipe(decoded, foldLeftRight); + + expect(getPaths(left(message.errors))).toEqual([]); + expect(message.schema).toEqual(payload); + }); + + it('rejects an unknown category value', () => { + const payload = { + type, + value: 'unknown', + host: 0.1, + }; + const decoded = riskWeightSchema.decode(payload); + const message = pipe(decoded, foldLeftRight); + + expect(getPaths(left(message.errors))).toContain( + 'Invalid value "unknown" supplied to "value"' + ); + expect(message.schema).toEqual({}); + }); + }); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/common/risk_engine/risk_weights/schema.ts b/x-pack/plugins/security_solution/common/risk_engine/risk_weights/schema.ts new file mode 100644 index 000000000000..16a8a6da03ae --- /dev/null +++ b/x-pack/plugins/security_solution/common/risk_engine/risk_weights/schema.ts @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import * as t from 'io-ts'; +import { NumberBetweenZeroAndOneInclusive } from '@kbn/securitysolution-io-ts-types'; + +import { fromEnum } from '../utils'; +import { RiskCategories, RiskWeightTypes } from './types'; + +const hostWeight = t.type({ + host: NumberBetweenZeroAndOneInclusive, +}); + +const userWeight = t.type({ + user: NumberBetweenZeroAndOneInclusive, +}); + +const identifierWeights = t.union([ + t.exact(t.intersection([hostWeight, userWeight])), + t.exact(t.intersection([hostWeight, t.partial({ user: t.undefined })])), + t.exact(t.intersection([userWeight, t.partial({ host: t.undefined })])), +]); + +const riskCategories = fromEnum('riskCategories', RiskCategories); + +const globalRiskWeightSchema = t.intersection([ + t.exact( + t.type({ + type: t.literal(RiskWeightTypes.global), + }) + ), + identifierWeights, +]); +export type GlobalRiskWeight = t.TypeOf; + +const riskCategoryRiskWeightSchema = t.intersection([ + t.exact( + t.type({ + type: t.literal(RiskWeightTypes.riskCategory), + value: riskCategories, + }) + ), + identifierWeights, +]); +export type RiskCategoryRiskWeight = t.TypeOf; + +export const riskWeightSchema = t.union([globalRiskWeightSchema, riskCategoryRiskWeightSchema]); +export type RiskWeightSchema = t.TypeOf; +export type RiskWeight = RiskWeightSchema; + +export const riskWeightsSchema = t.array(riskWeightSchema); +export type RiskWeightsSchema = t.TypeOf; +export type RiskWeights = RiskWeightsSchema; diff --git a/x-pack/plugins/security_solution/common/risk_engine/risk_weights/types.ts b/x-pack/plugins/security_solution/common/risk_engine/risk_weights/types.ts new file mode 100644 index 000000000000..de995d565e69 --- /dev/null +++ b/x-pack/plugins/security_solution/common/risk_engine/risk_weights/types.ts @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export enum RiskWeightTypes { + global = 'global_identifier', + riskCategory = 'risk_category', +} + +export enum RiskCategories { + alerts = 'alerts', +} diff --git a/x-pack/plugins/security_solution/common/risk_engine/utils.ts b/x-pack/plugins/security_solution/common/risk_engine/utils.ts new file mode 100644 index 000000000000..51401eb9ff19 --- /dev/null +++ b/x-pack/plugins/security_solution/common/risk_engine/utils.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import * as t from 'io-ts'; +/* + * This utility function can be used to turn a TypeScript enum into a io-ts codec. + */ +export function fromEnum( + enumName: string, + theEnum: Record +): t.Type { + const isEnumValue = (input: unknown): input is EnumType => + Object.values(theEnum).includes(input); + + return new t.Type( + enumName, + isEnumValue, + (input, context) => (isEnumValue(input) ? t.success(input) : t.failure(input, context)), + t.identity + ); +} diff --git a/x-pack/plugins/security_solution/server/client/client.ts b/x-pack/plugins/security_solution/server/client/client.ts index 03d7afa4b39b..c7e63769ac1e 100644 --- a/x-pack/plugins/security_solution/server/client/client.ts +++ b/x-pack/plugins/security_solution/server/client/client.ts @@ -6,9 +6,14 @@ */ import type { ConfigType } from '../config'; -import { DEFAULT_DATA_VIEW_ID, DEFAULT_PREVIEW_INDEX } from '../../common/constants'; +import { + DEFAULT_ALERTS_INDEX, + DEFAULT_DATA_VIEW_ID, + DEFAULT_PREVIEW_INDEX, +} from '../../common/constants'; export class AppClient { + private readonly alertsIndex: string; private readonly signalsIndex: string; private readonly spaceId: string; private readonly previewIndex: string; @@ -19,6 +24,7 @@ export class AppClient { constructor(spaceId: string, config: ConfigType, kibanaVersion: string, kibanaBranch: string) { const configuredSignalsIndex = config.signalsIndex; + this.alertsIndex = `${DEFAULT_ALERTS_INDEX}-${spaceId}`; this.signalsIndex = `${configuredSignalsIndex}-${spaceId}`; this.previewIndex = `${DEFAULT_PREVIEW_INDEX}-${spaceId}`; this.sourcererDataViewId = `${DEFAULT_DATA_VIEW_ID}-${spaceId}`; @@ -27,6 +33,7 @@ export class AppClient { this.kibanaBranch = kibanaBranch; } + public getAlertsIndex = (): string => this.alertsIndex; public getSignalsIndex = (): string => this.signalsIndex; public getPreviewIndex = (): string => this.previewIndex; public getSourcererDataViewId = (): string => this.sourcererDataViewId; diff --git a/x-pack/plugins/security_solution/server/lib/risk_engine/calculate_risk_scores.mock.ts b/x-pack/plugins/security_solution/server/lib/risk_engine/calculate_risk_scores.mock.ts new file mode 100644 index 000000000000..0c091cdbc494 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/risk_engine/calculate_risk_scores.mock.ts @@ -0,0 +1,61 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { CalculateRiskScoreAggregations, RiskScoreBucket } from './types'; + +const createRiskScoreBucketMock = (overrides: Partial = {}): RiskScoreBucket => ({ + key: { 'user.name': 'username', category: 'alert' }, + doc_count: 2, + risk_details: { + value: { + score: 20, + normalized_score: 30.0, + level: 'Unknown', + notes: [], + alerts_score: 30, + other_score: 0, + }, + }, + riskiest_inputs: { + took: 17, + timed_out: false, + _shards: { + total: 1, + successful: 1, + skipped: 0, + failed: 0, + }, + hits: { + total: { + value: 1, + relation: 'eq', + }, + hits: [{ _id: '_id', _index: '_index', sort: [30] }], + }, + }, + + ...overrides, +}); + +const createAggregationResponseMock = ( + overrides: Partial = {} +): CalculateRiskScoreAggregations => ({ + host: { + after_key: { 'host.name': 'hostname' }, + buckets: [createRiskScoreBucketMock(), createRiskScoreBucketMock()], + }, + user: { + after_key: { 'user.name': 'username' }, + buckets: [createRiskScoreBucketMock(), createRiskScoreBucketMock()], + }, + ...overrides, +}); + +export const calculateRiskScoreMock = { + createAggregationResponse: createAggregationResponseMock, + createRiskScoreBucket: createRiskScoreBucketMock, +}; diff --git a/x-pack/plugins/security_solution/server/lib/risk_engine/calculate_risk_scores.test.ts b/x-pack/plugins/security_solution/server/lib/risk_engine/calculate_risk_scores.test.ts new file mode 100644 index 000000000000..0bca9036dc72 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/risk_engine/calculate_risk_scores.test.ts @@ -0,0 +1,198 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ElasticsearchClient, Logger } from '@kbn/core/server'; +import { elasticsearchServiceMock, loggingSystemMock } from '@kbn/core/server/mocks'; + +import { calculateRiskScores } from './calculate_risk_scores'; +import { calculateRiskScoreMock } from './calculate_risk_scores.mock'; + +describe('calculateRiskScores()', () => { + let params: Parameters[0]; + let esClient: ElasticsearchClient; + let logger: Logger; + + beforeEach(() => { + esClient = elasticsearchServiceMock.createScopedClusterClient().asCurrentUser; + logger = loggingSystemMock.createLogger(); + params = { + afterKeys: {}, + esClient, + logger, + index: 'index', + pageSize: 500, + range: { start: 'now - 15d', end: 'now' }, + }; + }); + + describe('inputs', () => { + it('builds a filter on @timestamp based on the provided range', async () => { + await calculateRiskScores(params); + expect(esClient.search).toHaveBeenCalledWith( + expect.objectContaining({ + query: { + bool: { + filter: expect.arrayContaining([ + { + range: { '@timestamp': { gte: 'now - 15d', lt: 'now' } }, + }, + ]), + }, + }, + }) + ); + }); + + describe('identifierType', () => { + it('creates aggs for both host and user by default', async () => { + await calculateRiskScores(params); + expect(esClient.search).toHaveBeenCalledWith( + expect.objectContaining({ + aggs: expect.objectContaining({ host: expect.anything(), user: expect.anything() }), + }) + ); + }); + + it('creates an aggregation per specified identifierType', async () => { + params = { ...params, identifierType: 'host' }; + await calculateRiskScores(params); + const [[call]] = (esClient.search as jest.Mock).mock.calls; + expect(call).toEqual( + expect.objectContaining({ aggs: expect.objectContaining({ host: expect.anything() }) }) + ); + expect(call.aggs).toHaveProperty('host'); + expect(call.aggs).not.toHaveProperty('user'); + }); + }); + + describe('after_keys', () => { + it('applies a single after_key to the correct aggregation', async () => { + params = { ...params, afterKeys: { host: { 'host.name': 'foo' } } }; + await calculateRiskScores(params); + const [[call]] = (esClient.search as jest.Mock).mock.calls; + expect(call).toEqual( + expect.objectContaining({ + aggs: expect.objectContaining({ + host: expect.objectContaining({ + composite: expect.objectContaining({ after: { 'host.name': 'foo' } }), + }), + }), + }) + ); + }); + + it('applies multiple after_keys to the correct aggregations', async () => { + params = { + ...params, + afterKeys: { + host: { 'host.name': 'foo' }, + user: { 'user.name': 'bar' }, + }, + }; + await calculateRiskScores(params); + const [[call]] = (esClient.search as jest.Mock).mock.calls; + + expect(call).toEqual( + expect.objectContaining({ + aggs: expect.objectContaining({ + host: expect.objectContaining({ + composite: expect.objectContaining({ after: { 'host.name': 'foo' } }), + }), + user: expect.objectContaining({ + composite: expect.objectContaining({ after: { 'user.name': 'bar' } }), + }), + }), + }) + ); + }); + + it('uses an undefined after_key by default', async () => { + await calculateRiskScores(params); + const [[call]] = (esClient.search as jest.Mock).mock.calls; + + expect(call).toEqual( + expect.objectContaining({ + aggs: expect.objectContaining({ + host: expect.objectContaining({ + composite: expect.objectContaining({ after: undefined }), + }), + user: expect.objectContaining({ + composite: expect.objectContaining({ after: undefined }), + }), + }), + }) + ); + }); + }); + }); + + describe('outputs', () => { + beforeEach(() => { + // stub out a reasonable response + (esClient.search as jest.Mock).mockResolvedValueOnce({ + aggregations: calculateRiskScoreMock.createAggregationResponse(), + }); + }); + + it('returns a flattened list of risk scores', async () => { + const response = await calculateRiskScores(params); + expect(response).toHaveProperty('scores'); + expect(response.scores).toHaveLength(4); + }); + + it('returns scores in the expected format', async () => { + const { + scores: [score], + } = await calculateRiskScores(params); + expect(score).toEqual( + expect.objectContaining({ + '@timestamp': expect.any(String), + identifierField: expect.any(String), + identifierValue: expect.any(String), + level: 'Unknown', + totalScore: expect.any(Number), + totalScoreNormalized: expect.any(Number), + alertsScore: expect.any(Number), + otherScore: expect.any(Number), + }) + ); + }); + + it('returns risk inputs in the expected format', async () => { + const { + scores: [score], + } = await calculateRiskScores(params); + expect(score).toEqual( + expect.objectContaining({ + riskiestInputs: expect.arrayContaining([ + expect.objectContaining({ + id: expect.any(String), + index: expect.any(String), + riskScore: expect.any(Number), + }), + ]), + }) + ); + }); + }); + + describe('error conditions', () => { + beforeEach(() => { + // stub out a rejected response + (esClient.search as jest.Mock).mockRejectedValueOnce({ + aggregations: calculateRiskScoreMock.createAggregationResponse(), + }); + }); + + it('raises an error if elasticsearch client rejects', () => { + expect.assertions(1); + expect(() => calculateRiskScores(params)).rejects.toEqual({ + aggregations: calculateRiskScoreMock.createAggregationResponse(), + }); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/risk_engine/calculate_risk_scores.ts b/x-pack/plugins/security_solution/server/lib/risk_engine/calculate_risk_scores.ts new file mode 100644 index 000000000000..c92513649b4f --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/risk_engine/calculate_risk_scores.ts @@ -0,0 +1,277 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { + AggregationsAggregationContainer, + QueryDslQueryContainer, +} from '@elastic/elasticsearch/lib/api/types'; +import type { ElasticsearchClient, Logger } from '@kbn/core/server'; +import { + ALERT_RISK_SCORE, + EVENT_KIND, +} from '@kbn/rule-registry-plugin/common/technical_rule_data_field_names'; +import type { AfterKeys, IdentifierType, RiskWeights } from '../../../common/risk_engine'; +import { withSecuritySpan } from '../../utils/with_security_span'; +import { getAfterKeyForIdentifierType, getFieldForIdentifierAgg } from './helpers'; +import { + buildCategoryScoreAssignment, + buildCategoryScoreDeclarations, + buildWeightingOfScoreByCategory, + getGlobalWeightForIdentifierType, +} from './risk_weights'; +import type { + CalculateRiskScoreAggregations, + GetScoresParams, + GetScoresResponse, + RiskScore, + RiskScoreBucket, +} from './types'; + +const bucketToResponse = ({ + bucket, + now, + identifierField, +}: { + bucket: RiskScoreBucket; + now: string; + identifierField: string; +}): RiskScore => ({ + '@timestamp': now, + identifierField, + identifierValue: bucket.key[identifierField], + level: bucket.risk_details.value.level, + totalScore: bucket.risk_details.value.score, + totalScoreNormalized: bucket.risk_details.value.normalized_score, + alertsScore: bucket.risk_details.value.alerts_score, + otherScore: bucket.risk_details.value.other_score, + notes: bucket.risk_details.value.notes, + riskiestInputs: bucket.riskiest_inputs.hits.hits.map((riskInput) => ({ + id: riskInput._id, + index: riskInput._index, + riskScore: riskInput.sort?.[0] ?? undefined, + })), +}); + +const filterFromRange = (range: GetScoresParams['range']): QueryDslQueryContainer => ({ + range: { '@timestamp': { lt: range.end, gte: range.start } }, +}); + +const buildReduceScript = ({ + globalIdentifierTypeWeight, +}: { + globalIdentifierTypeWeight?: number; +}): string => { + return ` + Map results = new HashMap(); + List inputs = []; + for (state in states) { + inputs.addAll(state.inputs) + } + Collections.sort(inputs, (a, b) -> b.get('weighted_score').compareTo(a.get('weighted_score'))); + + double num_inputs_to_score = Math.min(inputs.length, params.max_risk_inputs_per_identity); + results['notes'] = []; + if (num_inputs_to_score == params.max_risk_inputs_per_identity) { + results['notes'].add('Number of risk inputs (' + inputs.length + ') exceeded the maximum allowed (' + params.max_risk_inputs_per_identity + ').'); + } + + ${buildCategoryScoreDeclarations()} + + double total_score = 0; + double current_score = 0; + for (int i = 0; i < num_inputs_to_score; i++) { + current_score = inputs[i].weighted_score / Math.pow(i + 1, params.p); + + ${buildCategoryScoreAssignment()} + total_score += current_score; + } + + ${globalIdentifierTypeWeight != null ? `total_score *= ${globalIdentifierTypeWeight};` : ''} + double score_norm = 100 * total_score / params.risk_cap; + results['score'] = total_score; + results['normalized_score'] = score_norm; + + if (score_norm < 20) { + results['level'] = 'Unknown' + } + else if (score_norm >= 20 && score_norm < 40) { + results['level'] = 'Low' + } + else if (score_norm >= 40 && score_norm < 70) { + results['level'] = 'Moderate' + } + else if (score_norm >= 70 && score_norm < 90) { + results['level'] = 'High' + } + else if (score_norm >= 90) { + results['level'] = 'Critical' + } + + return results; + `; +}; + +const buildIdentifierTypeAggregation = ({ + afterKeys, + identifierType, + pageSize, + weights, +}: { + afterKeys: AfterKeys; + identifierType: IdentifierType; + pageSize: number; + weights?: RiskWeights; +}): AggregationsAggregationContainer => { + const globalIdentifierTypeWeight = getGlobalWeightForIdentifierType({ identifierType, weights }); + const identifierField = getFieldForIdentifierAgg(identifierType); + + return { + composite: { + size: pageSize, + sources: [ + { + [identifierField]: { + terms: { + field: identifierField, + }, + }, + }, + ], + after: getAfterKeyForIdentifierType({ identifierType, afterKeys }), + }, + aggs: { + riskiest_inputs: { + top_hits: { + size: 10, + sort: { [ALERT_RISK_SCORE]: 'desc' }, + _source: false, + }, + }, + risk_details: { + scripted_metric: { + init_script: 'state.inputs = []', + map_script: ` + Map fields = new HashMap(); + String category = doc['${EVENT_KIND}'].value; + double score = doc['${ALERT_RISK_SCORE}'].value; + double weighted_score = 0.0; + + fields.put('time', doc['@timestamp'].value); + fields.put('category', category); + fields.put('score', score); + ${buildWeightingOfScoreByCategory({ userWeights: weights, identifierType })} + fields.put('weighted_score', weighted_score); + + state.inputs.add(fields); + `, + combine_script: 'return state;', + params: { + max_risk_inputs_per_identity: 999999, + p: 1.5, + risk_cap: 261.2, + }, + reduce_script: buildReduceScript({ globalIdentifierTypeWeight }), + }, + }, + }, + }; +}; + +export const calculateRiskScores = async ({ + afterKeys: userAfterKeys, + debug, + esClient, + filter: userFilter, + identifierType, + index, + logger, + pageSize, + range, + weights, +}: { + esClient: ElasticsearchClient; + logger: Logger; +} & GetScoresParams): Promise => + withSecuritySpan('calculateRiskScores', async () => { + const now = new Date().toISOString(); + + const filter = [{ exists: { field: ALERT_RISK_SCORE } }, filterFromRange(range)]; + if (userFilter) { + filter.push(userFilter as QueryDslQueryContainer); + } + const identifierTypes: IdentifierType[] = identifierType ? [identifierType] : ['host', 'user']; + + const request = { + size: 0, + _source: false, + index, + query: { + bool: { + filter, + }, + }, + aggs: identifierTypes.reduce((aggs, _identifierType) => { + aggs[_identifierType] = buildIdentifierTypeAggregation({ + afterKeys: userAfterKeys, + identifierType: _identifierType, + pageSize, + weights, + }); + return aggs; + }, {} as Record), + }; + + if (debug) { + logger.info(`Executing Risk Score query:\n${JSON.stringify(request)}`); + } + + const response = await esClient.search(request); + + if (debug) { + logger.info(`Received Risk Score response:\n${JSON.stringify(response)}`); + } + + if (response.aggregations == null) { + return { + ...(debug ? { request, response } : {}), + after_keys: {}, + scores: [], + }; + } + + const userBuckets = response.aggregations.user?.buckets ?? []; + const hostBuckets = response.aggregations.host?.buckets ?? []; + + const afterKeys = { + host: response.aggregations.host?.after_key, + user: response.aggregations.user?.after_key, + }; + + const scores = userBuckets + .map((bucket) => + bucketToResponse({ + bucket, + identifierField: 'user.name', + now, + }) + ) + .concat( + hostBuckets.map((bucket) => + bucketToResponse({ + bucket, + identifierField: 'host.name', + now, + }) + ) + ); + + return { + ...(debug ? { request, response } : {}), + after_keys: afterKeys, + scores, + }; + }); diff --git a/x-pack/plugins/security_solution/server/lib/risk_engine/helpers.ts b/x-pack/plugins/security_solution/server/lib/risk_engine/helpers.ts new file mode 100644 index 000000000000..a8f9563ba5e9 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/risk_engine/helpers.ts @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Logger, SavedObjectsClientContract } from '@kbn/core/server'; +import type { DataViewAttributes } from '@kbn/data-views-plugin/common'; +import type { AfterKey, AfterKeys, IdentifierType } from '../../../common/risk_engine'; + +export const getRiskInputsIndex = async ({ + dataViewId, + logger, + soClient, +}: { + dataViewId: string; + logger: Logger; + soClient: SavedObjectsClientContract; +}): Promise => { + try { + const dataView = await soClient.get('index-pattern', dataViewId); + return dataView.attributes.title; + } catch (e) { + logger.debug(`No dataview found for ID '${dataViewId}'`); + } +}; + +export const getFieldForIdentifierAgg = (identifierType: IdentifierType): string => + identifierType === 'host' ? 'host.name' : 'user.name'; + +export const getAfterKeyForIdentifierType = ({ + afterKeys, + identifierType, +}: { + afterKeys: AfterKeys; + identifierType: IdentifierType; +}): AfterKey | undefined => afterKeys[identifierType]; diff --git a/x-pack/plugins/security_solution/server/lib/risk_engine/risk_score_service.mock.ts b/x-pack/plugins/security_solution/server/lib/risk_engine/risk_score_service.mock.ts new file mode 100644 index 000000000000..642e71c155e4 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/risk_engine/risk_score_service.mock.ts @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { RiskScoreService } from './risk_score_service'; +import type { RiskScore } from './types'; + +const createRiskScoreMock = (overrides: Partial = {}): RiskScore => ({ + '@timestamp': '2023-02-15T00:15:19.231Z', + identifierField: 'host.name', + identifierValue: 'hostname', + level: 'High', + totalScore: 149, + totalScoreNormalized: 85.332, + alertsScore: 85, + otherScore: 0, + notes: [], + riskiestInputs: [], + ...overrides, +}); + +const createRiskScoreServiceMock = (): jest.Mocked => ({ + getScores: jest.fn(), +}); + +export const riskScoreServiceMock = { + create: createRiskScoreServiceMock, + createRiskScore: createRiskScoreMock, +}; diff --git a/x-pack/plugins/security_solution/server/lib/risk_engine/risk_score_service.ts b/x-pack/plugins/security_solution/server/lib/risk_engine/risk_score_service.ts new file mode 100644 index 000000000000..c60ea852f560 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/risk_engine/risk_score_service.ts @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ElasticsearchClient, Logger } from '@kbn/core/server'; +import type { GetScoresParams, GetScoresResponse } from './types'; +import { calculateRiskScores } from './calculate_risk_scores'; + +export interface RiskScoreService { + getScores: (params: GetScoresParams) => Promise; +} + +export const riskScoreService = ({ + esClient, + logger, +}: { + esClient: ElasticsearchClient; + logger: Logger; +}): RiskScoreService => ({ + getScores: (params) => calculateRiskScores({ ...params, esClient, logger }), +}); diff --git a/x-pack/plugins/security_solution/server/lib/risk_engine/risk_weights.test.ts b/x-pack/plugins/security_solution/server/lib/risk_engine/risk_weights.test.ts new file mode 100644 index 000000000000..4d93bb83452e --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/risk_engine/risk_weights.test.ts @@ -0,0 +1,108 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { RiskWeightTypes, RiskCategories } from '../../../common/risk_engine'; +import { + buildCategoryScoreAssignment, + buildCategoryWeights, + buildWeightingOfScoreByCategory, +} from './risk_weights'; + +describe('buildCategoryWeights', () => { + it('returns the default weights if nothing else is provided', () => { + const result = buildCategoryWeights(); + + expect(result).toEqual([ + { host: 1, type: RiskWeightTypes.riskCategory, user: 1, value: RiskCategories.alerts }, + ]); + }); + + it('allows user weights to override defaults', () => { + const result = buildCategoryWeights([ + { type: RiskWeightTypes.riskCategory, value: RiskCategories.alerts, host: 0.1, user: 0.2 }, + ]); + + expect(result).toEqual([ + { host: 0.1, type: RiskWeightTypes.riskCategory, user: 0.2, value: RiskCategories.alerts }, + ]); + }); + + it('uses default category weights if unspecified in user-provided weight', () => { + const result = buildCategoryWeights([ + { type: RiskWeightTypes.riskCategory, value: RiskCategories.alerts, host: 0.1 }, + ]); + + expect(result).toEqual([ + { host: 0.1, type: RiskWeightTypes.riskCategory, user: 1, value: RiskCategories.alerts }, + ]); + }); +}); + +describe('buildCategoryScoreAssignment', () => { + it('builds the expected assignment statement', () => { + const result = buildCategoryScoreAssignment(); + + expect(result).toMatchInlineSnapshot( + `"if (inputs[i].category == 'signal') { results['alerts_score'] += current_score; } else { results['other_score'] += current_score; }"` + ); + }); +}); + +describe('buildWeightingOfScoreByCategory', () => { + it('returns default weights if no user values provided', () => { + const result = buildWeightingOfScoreByCategory({ identifierType: 'user' }); + + expect(result).toMatchInlineSnapshot( + `"if (category == 'signal') { weighted_score = score * 1; } else { weighted_score = score; }"` + ); + }); + + it('returns default weights if no weights provided', () => { + const result = buildWeightingOfScoreByCategory({ userWeights: [], identifierType: 'host' }); + + expect(result).toMatchInlineSnapshot( + `"if (category == 'signal') { weighted_score = score * 1; } else { weighted_score = score; }"` + ); + }); + + it('returns default weights if only global weights provided', () => { + const result = buildWeightingOfScoreByCategory({ + userWeights: [{ type: RiskWeightTypes.global, host: 0.1 }], + identifierType: 'host', + }); + + expect(result).toMatchInlineSnapshot( + `"if (category == 'signal') { weighted_score = score * 1; } else { weighted_score = score; }"` + ); + }); + + it('returns specified weight when a category weight is provided', () => { + const result = buildWeightingOfScoreByCategory({ + userWeights: [ + { type: RiskWeightTypes.riskCategory, value: RiskCategories.alerts, host: 0.1, user: 0.2 }, + ], + identifierType: 'host', + }); + + expect(result).toMatchInlineSnapshot( + `"if (category == 'signal') { weighted_score = score * 0.1; } else { weighted_score = score; }"` + ); + }); + + it('returns a default weight when a category weight is provided but not the one being used', () => { + const result = buildWeightingOfScoreByCategory({ + userWeights: [ + { type: RiskWeightTypes.riskCategory, value: RiskCategories.alerts, host: 0.1 }, + ], + identifierType: 'user', + }); + + expect(result).toMatchInlineSnapshot( + `"if (category == 'signal') { weighted_score = score * 1; } else { weighted_score = score; }"` + ); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/risk_engine/risk_weights.ts b/x-pack/plugins/security_solution/server/lib/risk_engine/risk_weights.ts new file mode 100644 index 000000000000..60651140a779 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/risk_engine/risk_weights.ts @@ -0,0 +1,104 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { keyBy, merge } from 'lodash'; +import type { + GlobalRiskWeight, + IdentifierType, + RiskCategoryRiskWeight, + RiskWeight, + RiskWeights, +} from '../../../common/risk_engine'; +import { RiskCategories, RiskWeightTypes } from '../../../common/risk_engine'; + +const RISK_CATEGORIES = Object.values(RiskCategories); + +const DEFAULT_CATEGORY_WEIGHTS: RiskWeights = RISK_CATEGORIES.map((category) => ({ + type: RiskWeightTypes.riskCategory, + value: category, + host: 1, + user: 1, +})); + +/* + * This function and its use can be deleted once we've replaced our use of event.kind with a proper risk category field. + */ +const convertCategoryToEventKindValue = (category?: string): string | undefined => + category === 'alerts' ? 'signal' : category; + +const isGlobalIdentifierTypeWeight = (weight: RiskWeight): weight is GlobalRiskWeight => + weight.type === RiskWeightTypes.global; +const isRiskCategoryWeight = (weight: RiskWeight): weight is RiskCategoryRiskWeight => + weight.type === RiskWeightTypes.riskCategory; + +export const getGlobalWeightForIdentifierType = ({ + identifierType, + weights, +}: { + identifierType: IdentifierType; + weights?: RiskWeights; +}): number | undefined => { + return weights?.find(isGlobalIdentifierTypeWeight)?.[identifierType]; +}; + +const getRiskCategoryWeights = (weights?: RiskWeights): RiskCategoryRiskWeight[] => + weights?.filter(isRiskCategoryWeight) ?? []; + +const getWeightForIdentifierType = (weight: RiskWeight, identifierType: IdentifierType): number => { + const configuredWeight = weight[identifierType]; + return typeof configuredWeight === 'number' ? configuredWeight : 1; +}; + +export const buildCategoryScoreDeclarations = (): string => { + const otherScoreDeclaration = `results['other_score'] = 0;`; + + return RISK_CATEGORIES.map((riskCategory) => `results['${riskCategory}_score'] = 0;`) + .join('') + .concat(otherScoreDeclaration); +}; + +export const buildCategoryWeights = (userWeights?: RiskWeights): RiskCategoryRiskWeight[] => { + const categoryWeights = getRiskCategoryWeights(userWeights); + + return Object.values( + merge({}, keyBy(DEFAULT_CATEGORY_WEIGHTS, 'value'), keyBy(categoryWeights, 'value')) + ); +}; + +export const buildCategoryScoreAssignment = (): string => { + const otherClause = `results['other_score'] += current_score;`; + + return RISK_CATEGORIES.map( + (category) => + `if (inputs[i].category == '${convertCategoryToEventKindValue( + category + )}') { results['${category}_score'] += current_score; }` + ) + .join(' else ') + .concat(` else { ${otherClause} }`); +}; + +export const buildWeightingOfScoreByCategory = ({ + userWeights, + identifierType, +}: { + userWeights?: RiskWeights; + identifierType: IdentifierType; +}): string => { + const otherClause = `weighted_score = score;`; + const categoryWeights = buildCategoryWeights(userWeights); + + return categoryWeights + .map( + (weight) => + `if (category == '${convertCategoryToEventKindValue( + weight.value + )}') { weighted_score = score * ${getWeightForIdentifierType(weight, identifierType)}; }` + ) + .join(' else ') + .concat(` else { ${otherClause} }`); +}; diff --git a/x-pack/plugins/security_solution/server/lib/risk_engine/routes/index.ts b/x-pack/plugins/security_solution/server/lib/risk_engine/routes/index.ts new file mode 100644 index 000000000000..2d76d490948d --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/risk_engine/routes/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { riskScorePreviewRoute } from './risk_score_preview_route'; diff --git a/x-pack/plugins/security_solution/server/lib/risk_engine/routes/risk_score_preview_route.test.ts b/x-pack/plugins/security_solution/server/lib/risk_engine/routes/risk_score_preview_route.test.ts new file mode 100644 index 000000000000..fcf98c1b1ef7 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/risk_engine/routes/risk_score_preview_route.test.ts @@ -0,0 +1,252 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { loggerMock } from '@kbn/logging-mocks'; + +import { RISK_SCORE_PREVIEW_URL } from '../../../../common/constants'; +import { RiskCategories, RiskWeightTypes } from '../../../../common/risk_engine'; +import { + serverMock, + requestContextMock, + requestMock, +} from '../../detection_engine/routes/__mocks__'; +import { riskScoreService } from '../risk_score_service'; +import { riskScoreServiceMock } from '../risk_score_service.mock'; +import { riskScorePreviewRoute } from './risk_score_preview_route'; + +jest.mock('../risk_score_service'); + +describe('POST risk_engine/preview route', () => { + let server: ReturnType; + let { clients, context } = requestContextMock.createTools(); + let logger: ReturnType; + let mockRiskScoreService: ReturnType; + + beforeEach(() => { + jest.resetAllMocks(); + + server = serverMock.create(); + logger = loggerMock.create(); + ({ clients, context } = requestContextMock.createTools()); + mockRiskScoreService = riskScoreServiceMock.create(); + + clients.appClient.getAlertsIndex.mockReturnValue('default-alerts-index'); + (riskScoreService as jest.Mock).mockReturnValue(mockRiskScoreService); + + riskScorePreviewRoute(server.router, logger); + }); + + const buildRequest = (body: object = {}) => + requestMock.create({ + method: 'get', + path: RISK_SCORE_PREVIEW_URL, + body, + }); + + describe('parameters', () => { + describe('index / dataview', () => { + it('defaults to scoring the alerts index if no dataview is provided', async () => { + const request = buildRequest(); + + const response = await server.inject(request, requestContextMock.convertContext(context)); + + expect(response.status).toEqual(200); + expect(mockRiskScoreService.getScores).toHaveBeenCalledWith( + expect.objectContaining({ index: 'default-alerts-index' }) + ); + }); + + it('respects the provided dataview', async () => { + const request = buildRequest({ data_view_id: 'custom-dataview-id' }); + + // mock call to get dataview title + clients.savedObjectsClient.get.mockResolvedValueOnce({ + id: '', + type: '', + references: [], + attributes: { title: 'custom-dataview-index' }, + }); + const response = await server.inject(request, requestContextMock.convertContext(context)); + + expect(response.status).toEqual(200); + expect(mockRiskScoreService.getScores).toHaveBeenCalledWith( + expect.objectContaining({ index: 'custom-dataview-index' }) + ); + }); + + it('returns a 404 if dataview is not found', async () => { + const request = buildRequest({ data_view_id: 'custom-dataview-id' }); + + const response = await server.inject(request, requestContextMock.convertContext(context)); + + expect(response.status).toEqual(404); + expect(response.body.message).toEqual( + 'The specified dataview (custom-dataview-id) was not found. Please use an existing dataview, or omit the parameter to use the default risk inputs.' + ); + expect(mockRiskScoreService.getScores).not.toHaveBeenCalled(); + }); + }); + + describe('date range', () => { + it('defaults to the last 15 days of data', async () => { + const request = buildRequest(); + const response = await server.inject(request, requestContextMock.convertContext(context)); + + expect(response.status).toEqual(200); + expect(mockRiskScoreService.getScores).toHaveBeenCalledWith( + expect.objectContaining({ range: { start: 'now-15d', end: 'now' } }) + ); + }); + + it('respects the provided range if provided', async () => { + const request = buildRequest({ range: { start: 'now-30d', end: 'now-20d' } }); + const response = await server.inject(request, requestContextMock.convertContext(context)); + + expect(response.status).toEqual(200); + expect(mockRiskScoreService.getScores).toHaveBeenCalledWith( + expect.objectContaining({ range: { start: 'now-30d', end: 'now-20d' } }) + ); + }); + + it('rejects an invalid date range', async () => { + const request = buildRequest({ + range: { end: 'now' }, + }); + + const result = await server.validate(request); + expect(result.badRequest).toHaveBeenCalledWith( + expect.stringContaining('Invalid value "undefined" supplied to "range,start"') + ); + }); + }); + + describe('data filter', () => { + it('respects the provided filter if provided', async () => { + const request = buildRequest({ + filter: { + bool: { + filter: [ + { + ids: { + values: '1', + }, + }, + ], + }, + }, + }); + const response = await server.inject(request, requestContextMock.convertContext(context)); + + expect(response.status).toEqual(200); + expect(mockRiskScoreService.getScores).toHaveBeenCalledWith( + expect.objectContaining({ + filter: { + bool: { + filter: [ + { + ids: { + values: '1', + }, + }, + ], + }, + }, + }) + ); + }); + }); + + describe('weights', () => { + it('uses the specified weights when provided', async () => { + const request = buildRequest({ + weights: [ + { + type: RiskWeightTypes.riskCategory, + value: RiskCategories.alerts, + host: 0.1, + user: 0.2, + }, + ], + }); + + const response = await server.inject(request, requestContextMock.convertContext(context)); + + expect(response.status).toEqual(200); + expect(mockRiskScoreService.getScores).toHaveBeenCalledWith( + expect.objectContaining({ + weights: [ + { + type: RiskWeightTypes.riskCategory, + value: RiskCategories.alerts, + host: 0.1, + user: 0.2, + }, + ], + }) + ); + }); + + it('rejects weight values outside the 0-1 range', async () => { + const request = buildRequest({ + weights: [ + { + type: RiskWeightTypes.riskCategory, + value: RiskCategories.alerts, + host: 1.1, + }, + ], + }); + + const result = await server.validate(request); + expect(result.badRequest).toHaveBeenCalledWith( + expect.stringContaining('Invalid value "1.1" supplied to "weights,host"') + ); + }); + + it('rejects unknown weight types', async () => { + const request = buildRequest({ + weights: [ + { + type: 'something new', + host: 1.1, + }, + ], + }); + + const result = await server.validate(request); + expect(result.badRequest).toHaveBeenCalledWith( + 'Invalid value "{"type":"something new","host":1.1}" supplied to "weights"' + ); + }); + }); + + describe('pagination', () => { + it('respects the provided after_key', async () => { + const afterKey = { 'host.name': 'hi mom' }; + const request = buildRequest({ after_keys: { host: afterKey } }); + + const response = await server.inject(request, requestContextMock.convertContext(context)); + + expect(response.status).toEqual(200); + expect(mockRiskScoreService.getScores).toHaveBeenCalledWith( + expect.objectContaining({ afterKeys: { host: afterKey } }) + ); + }); + + it('rejects an invalid after_key', async () => { + const request = buildRequest({ + after_keys: { + bad: 'key', + }, + }); + + const result = await server.validate(request); + expect(result.badRequest).toHaveBeenCalledWith('invalid keys "bad"'); + }); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/risk_engine/routes/risk_score_preview_route.ts b/x-pack/plugins/security_solution/server/lib/risk_engine/routes/risk_score_preview_route.ts new file mode 100644 index 000000000000..c5a699264c32 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/risk_engine/routes/risk_score_preview_route.ts @@ -0,0 +1,94 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Logger } from '@kbn/core/server'; +import { buildSiemResponse } from '@kbn/lists-plugin/server/routes/utils'; +import { transformError } from '@kbn/securitysolution-es-utils'; +import { DEFAULT_RISK_SCORE_PAGE_SIZE, RISK_SCORE_PREVIEW_URL } from '../../../../common/constants'; +import { riskScorePreviewRequestSchema } from '../../../../common/risk_engine/risk_score_preview/request_schema'; +import type { SecuritySolutionPluginRouter } from '../../../types'; +import { buildRouteValidation } from '../../../utils/build_validation/route_validation'; +import { riskScoreService } from '../risk_score_service'; +import { getRiskInputsIndex } from '../helpers'; +import { DATAVIEW_NOT_FOUND } from './translations'; + +export const riskScorePreviewRoute = (router: SecuritySolutionPluginRouter, logger: Logger) => { + router.post( + { + path: RISK_SCORE_PREVIEW_URL, + validate: { body: buildRouteValidation(riskScorePreviewRequestSchema) }, + options: { + tags: ['access:securitySolution'], + }, + }, + async (context, request, response) => { + const siemResponse = buildSiemResponse(response); + const esClient = (await context.core).elasticsearch.client.asCurrentUser; + const soClient = (await context.core).savedObjects.client; + const siemClient = (await context.securitySolution).getAppClient(); + const riskScore = riskScoreService({ + esClient, + logger, + }); + + const { + after_keys: userAfterKeys, + data_view_id: dataViewId, + debug, + page_size: userPageSize, + identifier_type: identifierType, + filter, + range: userRange, + weights, + } = request.body; + + try { + let index: string; + if (dataViewId) { + const dataViewIndex = await getRiskInputsIndex({ + dataViewId, + logger, + soClient, + }); + + if (!dataViewIndex) { + return siemResponse.error({ + statusCode: 404, + body: DATAVIEW_NOT_FOUND(dataViewId), + }); + } + index = dataViewIndex; + } + index ??= siemClient.getAlertsIndex(); + + const afterKeys = userAfterKeys ?? {}; + const range = userRange ?? { start: 'now-15d', end: 'now' }; + const pageSize = userPageSize ?? DEFAULT_RISK_SCORE_PAGE_SIZE; + + const result = await riskScore.getScores({ + afterKeys, + debug, + pageSize, + identifierType, + index, + filter, + range, + weights, + }); + + return response.ok({ body: result }); + } catch (e) { + const error = transformError(e); + + return siemResponse.error({ + statusCode: error.statusCode, + body: { message: error.message, full_error: JSON.stringify(e) }, + }); + } + } + ); +}; diff --git a/x-pack/plugins/security_solution/server/lib/risk_engine/routes/translations.ts b/x-pack/plugins/security_solution/server/lib/risk_engine/routes/translations.ts new file mode 100644 index 000000000000..b825ed497ab1 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/risk_engine/routes/translations.ts @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { i18n } from '@kbn/i18n'; + +export const DATAVIEW_NOT_FOUND = (dataViewId: string): string => + i18n.translate('xpack.securitySolution.riskEngine.calculateScores.dataViewNotFoundError', { + values: { dataViewId }, + defaultMessage: + 'The specified dataview ({dataViewId}) was not found. Please use an existing dataview, or omit the parameter to use the default risk inputs.', + }); diff --git a/x-pack/plugins/security_solution/server/lib/risk_engine/schema/risk_score_apis.yml b/x-pack/plugins/security_solution/server/lib/risk_engine/schema/risk_score_apis.yml new file mode 100644 index 000000000000..a84fae4d7fef --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/risk_engine/schema/risk_score_apis.yml @@ -0,0 +1,223 @@ +openapi: 3.0.0 +info: + version: 1.0.0 + title: Risk Scoring API + description: These APIs allow the consumer to manage Entity Risk Scores within Entity Analytics. +paths: + /preview: + post: + summary: Preview the calculation of Risk Scores + description: Calculates and returns a list of Risk Scores, sorted by identifier_type and risk score. + requestBody: + description: Details about the Risk Scores being requested + content: + application/json: + schema: + $ref: '#/components/schemas/RiskScoresRequest' + required: false + responses: + '200': + description: Successful response + content: + application/json: + schema: + $ref: '#/components/schemas/RiskScoresResponse' + '400': + description: Invalid request + +components: + schemas: + RiskScoresRequest: + type: object + properties: + after_keys: + description: Used to retrieve a specific "page" of risk scores. If unspecified, the first "page" of scores is returned. See also the `after_keys` key in a risk scores response. + allOf: + - $ref: '#/components/schemas/AfterKeys' + data_view_id: + description: The identifier of the Kibana data view to be used when generating risk scores. If unspecified, the Security Alerts data view for the current space will be used. + example: security-solution-default + type: string + debug: + description: If set to `true`, a `debug` key is added to the response, containing both the internal request and response with elasticsearch. + type: boolean + filter: + description: An elasticsearch DSL filter object. Used to filter the data being scored, which implicitly filters the risk scores returned. + $ref: 'https://cloud.elastic.co/api/v1/api-docs/spec.json#/definitions/QueryContainer' + page_size: + description: Specifies how many scores will be returned in a given response. Note that this value is per `identifier_type`, i.e. a value of 10 will return 10 host scores and 10 user scores, if available. To avoid missed data, keep this value consistent while paginating through scores. + default: 1000 + type: number + identifier_type: + description: Used to restrict the type of risk scores being returned. If unspecified, both `host` and `user` scores will be returned. + allOf: + - $ref: '#/components/schemas/IdentifierType' + range: + description: Defines the time period over which scores will be evaluated. If unspecified, a range of `[now, now-30d]` will be used. + type: object + required: + - start + - end + properties: + start: + $ref: '#/components/schemas/KibanaDate' + end: + $ref: '#/components/schemas/KibanaDate' + weights: + description: 'A list of weights to be applied to the scoring calculation.' + type: array + items: + $ref: '#/components/schemas/RiskScoreWeight' + example: + - type: 'risk_category' + value: 'alerts' + host: 0.8 + user: 0.4 + - type: 'global_identifier' + host: 0.5 + user: 0.1 + RiskScoresResponse: + type: object + required: + - scores + properties: + after_keys: + description: Used to obtain the next "page" of risk scores. See also the `after_keys` key in a risk scores request. + allOf: + - $ref: '#/components/schemas/AfterKeys' + debug: + description: Object containing debug information, particularly the internal request and response from elasticsearch + type: object + properties: + request: + type: string + response: + type: string + scores: + type: array + description: A list of risk scores + items: + $ref: '#/components/schemas/RiskScore' + + AfterKeys: + type: object + properties: + host: + type: object + additionalProperties: + type: string + user: + type: object + additionalProperties: + type: string + example: + host: + 'host.name': 'example.host' + user: + 'user.name': 'example_user_name' + KibanaDate: + type: string + oneOf: + - format: date + - format: date-time + - format: datemath + example: '2017-07-21T17:32:28Z' + IdentifierType: + type: string + enum: + - host + - user + RiskScore: + type: object + required: + - '@timestamp' + - identifierField + - identifierValue + - level + - totalScore + - totalScoreNormalized + - alertsScore + - otherScore + - riskiestInputs + properties: + '@timestamp': + type: string + format: 'date-time' + example: '2017-07-21T17:32:28Z' + description: The time at which the risk score was calculated. + identifierField: + type: string + example: 'host.name' + description: The identifier field defining this risk score. Coupled with `identifierValue`, uniquely identifies the entity being scored. + identifierValue: + type: string + example: 'example.host' + description: The identifier value defining this risk score. Coupled with `identifierField`, uniquely identifies the entity being scored. + level: + type: string + example: 'Critical' + description: Lexical description of the entity's risk. + totalScore: + type: number + format: double + description: The raw numeric value of the given entity's risk score. + totalScoreNormalized: + type: number + format: double + minimum: 0 + maximum: 100 + description: The normalized numeric value of the given entity's risk score. Useful for comparing with other entities. + alertsScore: + type: number + format: double + description: The raw numeric risk score attributed to Security Alerts. + otherScore: + type: number + format: double + description: The raw numeric risk score attributed to other data sources + riskiestInputs: + type: array + description: A list of the 10 highest-risk documents contributing to this risk score. Useful for investigative purposes. + items: + $ref: '#/components/schemas/RiskScoreInput' + + RiskScoreInput: + description: A generic representation of a document contributing to a Risk Score. + type: object + properties: + id: + type: string + example: 91a93376a507e86cfbf282166275b89f9dbdb1f0be6c8103c6ff2909ca8e1a1c + index: + type: string + example: .internal.alerts-security.alerts-default-000001 + riskScore: + type: number + format: double + minimum: 0 + maximum: 100 + RiskScoreWeight: + description: "Configuration used to tune risk scoring. Weights can be used to change the score contribution of risk inputs for hosts and users at both a global level and also for Risk Input categories (e.g. 'alerts')." + type: object + required: + - type + properties: + type: + type: string + value: + type: string + host: + type: number + format: double + minimum: 0 + maximum: 1 + user: + type: number + format: double + minimum: 0 + maximum: 1 + example: + type: 'risk_category' + value: 'alerts' + host: 0.8 + user: 0.4 diff --git a/x-pack/plugins/security_solution/server/lib/risk_engine/types.ts b/x-pack/plugins/security_solution/server/lib/risk_engine/types.ts new file mode 100644 index 000000000000..9ab80f7dabfa --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/risk_engine/types.ts @@ -0,0 +1,79 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { EcsSecurityExtension as Ecs } from '@kbn/securitysolution-ecs'; +import type { SearchResponse } from '@elastic/elasticsearch/lib/api/types'; +import type { AfterKey, AfterKeys, IdentifierType, RiskWeights } from '../../../common/risk_engine'; + +export interface GetScoresParams { + afterKeys: AfterKeys; + debug?: boolean; + index: string; + filter?: unknown; + identifierType?: IdentifierType; + pageSize: number; + range: { start: string; end: string }; + weights?: RiskWeights; +} + +export interface GetScoresResponse { + debug?: { + request: unknown; + response: unknown; + }; + after_keys: AfterKeys; + scores: RiskScore[]; +} + +export interface SimpleRiskInput { + id: string; + index: string; + riskScore: string | number | undefined; +} + +export type RiskInput = Ecs; + +export interface RiskScore { + '@timestamp': string; + identifierField: string; + identifierValue: string; + level: string; + totalScore: number; + totalScoreNormalized: number; + alertsScore: number; + otherScore: number; + notes: string[]; + riskiestInputs: SimpleRiskInput[] | RiskInput[]; +} + +export interface CalculateRiskScoreAggregations { + user?: { + after_key: AfterKey; + buckets: RiskScoreBucket[]; + }; + host?: { + after_key: AfterKey; + buckets: RiskScoreBucket[]; + }; +} + +export interface RiskScoreBucket { + key: { [identifierField: string]: string; category: string }; + doc_count: number; + risk_details: { + value: { + score: number; + normalized_score: number; + notes: string[]; + level: string; + alerts_score: number; + other_score: number; + }; + }; + + riskiest_inputs: SearchResponse; +} diff --git a/x-pack/plugins/security_solution/server/mocks.ts b/x-pack/plugins/security_solution/server/mocks.ts index fad5686e2e2e..7575a5d13f4b 100644 --- a/x-pack/plugins/security_solution/server/mocks.ts +++ b/x-pack/plugins/security_solution/server/mocks.ts @@ -10,6 +10,7 @@ import type { AppClient } from './types'; type AppClientMock = jest.Mocked; const createAppClientMock = (): AppClientMock => ({ + getAlertsIndex: jest.fn(), getSignalsIndex: jest.fn(), getSourcererDataViewId: jest.fn().mockReturnValue('security-solution'), } as unknown as AppClientMock); diff --git a/x-pack/plugins/security_solution/server/routes/index.ts b/x-pack/plugins/security_solution/server/routes/index.ts index 38dea982d5a9..3d6027b6b514 100644 --- a/x-pack/plugins/security_solution/server/routes/index.ts +++ b/x-pack/plugins/security_solution/server/routes/index.ts @@ -73,6 +73,7 @@ import { import { registerManageExceptionsRoutes } from '../lib/exceptions/api/register_routes'; import { registerDashboardsRoutes } from '../lib/dashboards/routes'; import { registerTagsRoutes } from '../lib/tags/routes'; +import { riskScorePreviewRoute } from '../lib/risk_engine/routes'; export const initRoutes = ( router: SecuritySolutionPluginRouter, @@ -169,4 +170,8 @@ export const initRoutes = ( // telemetry preview endpoint for e2e integration tests only at the moment. telemetryDetectionRulesPreviewRoute(router, logger, previewTelemetryReceiver, telemetrySender); } + + if (config.experimentalFeatures.riskScoringRoutesEnabled) { + riskScorePreviewRoute(router, logger); + } }; diff --git a/x-pack/test/detection_engine_api_integration/common/config.ts b/x-pack/test/detection_engine_api_integration/common/config.ts index a1a71bf907b8..8454915db9a7 100644 --- a/x-pack/test/detection_engine_api_integration/common/config.ts +++ b/x-pack/test/detection_engine_api_integration/common/config.ts @@ -76,6 +76,7 @@ export function createTestConfig(options: CreateTestConfigOptions, testFiles?: s '--xpack.ruleRegistry.unsafe.legacyMultiTenancy.enabled=true', `--xpack.securitySolution.enableExperimental=${JSON.stringify([ 'previewTelemetryUrlEnabled', + 'riskScoringRoutesEnabled', ])}`, '--xpack.task_manager.poll_interval=1000', `--xpack.actions.preconfigured=${JSON.stringify({ diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/group10/index.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/group10/index.ts index 4449e9ca0780..1758ce0e99c0 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/group10/index.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/group10/index.ts @@ -37,5 +37,6 @@ export default ({ loadTestFile }: FtrProviderContext): void => { loadTestFile(require.resolve('./throttle')); loadTestFile(require.resolve('./ignore_fields')); loadTestFile(require.resolve('./migrations')); + loadTestFile(require.resolve('./risk_engine')); }); }; diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/group10/risk_engine.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/group10/risk_engine.ts new file mode 100644 index 000000000000..1e41f244140f --- /dev/null +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/group10/risk_engine.ts @@ -0,0 +1,554 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { ALERT_RISK_SCORE } from '@kbn/rule-data-utils'; +import { RISK_SCORE_PREVIEW_URL } from '@kbn/security-solution-plugin/common/constants'; +import type { RiskScore } from '@kbn/security-solution-plugin/server/lib/risk_engine/types'; +import { v4 as uuidv4 } from 'uuid'; +import { FtrProviderContext } from '../../common/ftr_provider_context'; +import { + createSignalsIndex, + deleteAllAlerts, + deleteAllRules, + createRule, + waitForSignalsToBePresent, + waitForRuleSuccess, + getRuleForSignalTesting, +} from '../../utils'; +import { dataGeneratorFactory } from '../../utils/data_generator'; + +const removeFields = (scores: any[]) => + scores.map((item) => { + delete item['@timestamp']; + delete item.riskiestInputs; + delete item.notes; + delete item.alertsScore; + delete item.otherScore; + return item; + }); + +const buildDocument = (body: any, id?: string) => { + const firstTimestamp = Date.now(); + const doc = { + id: id || uuidv4(), + '@timestamp': firstTimestamp, + agent: { + name: 'agent-12345', + }, + ...body, + }; + return doc; +}; + +// eslint-disable-next-line import/no-default-export +export default ({ getService }: FtrProviderContext): void => { + const supertest = getService('supertest'); + const esArchiver = getService('esArchiver'); + const es = getService('es'); + const log = getService('log'); + + const createAndSyncRuleAndAlerts = async ({ + alerts = 1, + riskScore = 21, + maxSignals = 100, + query, + riskScoreOverride, + }: { + alerts?: number; + riskScore?: number; + maxSignals?: number; + query: string; + riskScoreOverride?: string; + }): Promise => { + const rule = getRuleForSignalTesting(['ecs_compliant']); + const { id } = await createRule(supertest, log, { + ...rule, + risk_score: riskScore, + query, + max_signals: maxSignals, + ...(riskScoreOverride + ? { + risk_score_mapping: [ + { field: riskScoreOverride, operator: 'equals', value: '', risk_score: undefined }, + ], + } + : {}), + }); + await waitForRuleSuccess({ supertest, log, id }); + await waitForSignalsToBePresent(supertest, log, alerts, [id]); + }; + + const getRiskScores = async ({ body }: { body: object }): Promise<{ scores: RiskScore[] }> => { + const { body: result } = await supertest + .post(RISK_SCORE_PREVIEW_URL) + .set('kbn-xsrf', 'true') + .send(body) + .expect(200); + return result; + }; + + const getRiskScoreAfterRuleCreationAndExecution = async ( + documentId: string, + { + alerts = 1, + riskScore = 21, + maxSignals = 100, + }: { alerts?: number; riskScore?: number; maxSignals?: number } = {} + ) => { + await createAndSyncRuleAndAlerts({ query: `id: ${documentId}`, alerts, riskScore, maxSignals }); + + return await getRiskScores({ body: { debug: true } }); + }; + + describe('Risk engine', () => { + context('with auditbeat data', () => { + const { indexListOfDocuments } = dataGeneratorFactory({ + es, + index: 'ecs_compliant', + log, + }); + + before(async () => { + await esArchiver.load('x-pack/test/functional/es_archives/security_solution/ecs_compliant'); + }); + + after(async () => { + await esArchiver.unload( + 'x-pack/test/functional/es_archives/security_solution/ecs_compliant' + ); + }); + + beforeEach(async () => { + await deleteAllAlerts(supertest, log, es); + + await deleteAllRules(supertest, log); + await createSignalsIndex(supertest, log); + }); + + afterEach(async () => { + await deleteAllAlerts(supertest, log, es); + await deleteAllRules(supertest, log); + }); + + context('with a rule generating alerts with risk_score of 21', () => { + it('calculates risk from a single alert', async () => { + const documentId = uuidv4(); + await indexListOfDocuments([buildDocument({ host: { name: 'host-1' } }, documentId)]); + + const body = await getRiskScoreAfterRuleCreationAndExecution(documentId); + + expect(removeFields(body.scores)).to.eql([ + { + level: 'Unknown', + totalScore: 21, + totalScoreNormalized: 8.039816232771823, + identifierField: 'host.name', + identifierValue: 'host-1', + }, + ]); + }); + + it('calculates risk from two alerts, each representing a unique host', async () => { + const documentId = uuidv4(); + await indexListOfDocuments([ + buildDocument({ host: { name: 'host-1' } }, documentId), + buildDocument({ host: { name: 'host-2' } }, documentId), + ]); + + const body = await getRiskScoreAfterRuleCreationAndExecution(documentId, { + alerts: 2, + }); + + expect(removeFields(body.scores)).to.eql([ + { + level: 'Unknown', + totalScore: 21, + totalScoreNormalized: 8.039816232771823, + identifierField: 'host.name', + identifierValue: 'host-1', + }, + { + level: 'Unknown', + totalScore: 21, + totalScoreNormalized: 8.039816232771823, + identifierField: 'host.name', + identifierValue: 'host-2', + }, + ]); + }); + + it('calculates risk from two alerts, both for the same host', async () => { + const documentId = uuidv4(); + await indexListOfDocuments([ + buildDocument({ host: { name: 'host-1' } }, documentId), + buildDocument({ host: { name: 'host-1' } }, documentId), + ]); + + const body = await getRiskScoreAfterRuleCreationAndExecution(documentId, { + alerts: 2, + }); + + expect(removeFields(body.scores)).to.eql([ + { + level: 'Unknown', + totalScore: 28.42462120245875, + totalScoreNormalized: 10.88232052161514, + identifierField: 'host.name', + identifierValue: 'host-1', + }, + ]); + }); + + it('calculates risk from 30 alerts, all for the same host', async () => { + const documentId = uuidv4(); + const doc = buildDocument({ host: { name: 'host-1' } }, documentId); + await indexListOfDocuments(Array(30).fill(doc)); + + const body = await getRiskScoreAfterRuleCreationAndExecution(documentId, { + alerts: 30, + }); + + expect(removeFields(body.scores)).to.eql([ + { + level: 'Unknown', + totalScore: 47.25513506055279, + totalScoreNormalized: 18.091552473412246, + identifierField: 'host.name', + identifierValue: 'host-1', + }, + ]); + }); + + it('calculates risk from 31 alerts, 30 from the same host', async () => { + const documentId = uuidv4(); + const doc = buildDocument({ host: { name: 'host-1' } }, documentId); + await indexListOfDocuments([ + ...Array(30).fill(doc), + buildDocument({ host: { name: 'host-2' } }, documentId), + ]); + + const body = await getRiskScoreAfterRuleCreationAndExecution(documentId, { + alerts: 31, + }); + + expect(removeFields(body.scores)).to.eql([ + { + level: 'Unknown', + totalScore: 47.25513506055279, + totalScoreNormalized: 18.091552473412246, + identifierField: 'host.name', + identifierValue: 'host-1', + }, + { + level: 'Unknown', + totalScore: 21, + totalScoreNormalized: 8.039816232771823, + identifierField: 'host.name', + identifierValue: 'host-2', + }, + ]); + }); + + it('calculates risk from 100 alerts, all for the same host', async () => { + const documentId = uuidv4(); + const doc = buildDocument({ host: { name: 'host-1' } }, documentId); + await indexListOfDocuments(Array(100).fill(doc)); + + const body = await getRiskScoreAfterRuleCreationAndExecution(documentId, { + alerts: 100, + }); + + expect(removeFields(body.scores)).to.eql([ + { + level: 'Unknown', + totalScore: 50.67035607277805, + totalScoreNormalized: 19.399064346392823, + identifierField: 'host.name', + identifierValue: 'host-1', + }, + ]); + }); + }); + + context('with a rule generating alerts with risk_score of 100', () => { + it('calculates risk from 100 alerts, all for the same host', async () => { + const documentId = uuidv4(); + const doc = buildDocument({ host: { name: 'host-1' } }, documentId); + await indexListOfDocuments(Array(100).fill(doc)); + + const body = await getRiskScoreAfterRuleCreationAndExecution(documentId, { + riskScore: 100, + alerts: 100, + }); + + expect(removeFields(body.scores)).to.eql([ + { + level: 'Critical', + totalScore: 241.2874098703716, + totalScoreNormalized: 92.37649688758484, + identifierField: 'host.name', + identifierValue: 'host-1', + }, + ]); + }); + + it('calculates risk from 1,000 alerts, all for the same host', async () => { + const documentId = uuidv4(); + const doc = buildDocument({ host: { name: 'host-1' } }, documentId); + await indexListOfDocuments( + Array(1000) + .fill(doc) + .map((item, index) => ({ + ...item, + ['@timestamp']: item['@timestamp'] - index, + })) + ); + + const body = await getRiskScoreAfterRuleCreationAndExecution(documentId, { + riskScore: 100, + alerts: 1000, + maxSignals: 1000, + }); + + expect(removeFields(body.scores)).to.eql([ + { + level: 'Critical', + totalScore: 254.91456029175757, + totalScoreNormalized: 97.59362951445543, + identifierField: 'host.name', + identifierValue: 'host-1', + }, + ]); + }); + }); + + describe('risk score pagination', () => { + it('respects the specified after_keys', async () => { + const aaaId = uuidv4(); + const zzzId = uuidv4(); + const aaaDoc = buildDocument({ 'user.name': 'aaa' }, aaaId); + const zzzDoc = buildDocument({ 'user.name': 'zzz' }, zzzId); + await indexListOfDocuments(Array(50).fill(aaaDoc).concat(Array(50).fill(zzzDoc))); + + await createAndSyncRuleAndAlerts({ + query: `id: ${aaaId} OR ${zzzId}`, + alerts: 100, + riskScore: 100, + }); + + const { scores } = await getRiskScores({ + body: { + after_keys: { user: { 'user.name': 'aaa' } }, + }, + }); + // if after_key was not respected, 'aaa' would be included here + expect(scores).to.have.length(1); + expect(scores[0].identifierValue).to.equal('zzz'); + }); + }); + + describe('risk score filtering', () => { + it('restricts the range of risk inputs used for scoring', async () => { + const documentId = uuidv4(); + const doc = buildDocument({ host: { name: 'host-1' } }, documentId); + await indexListOfDocuments( + Array(100) + .fill(doc) + .map((_doc, i) => ({ ...doc, 'event.risk_score': i === 99 ? 1 : 100 })) + ); + + await createAndSyncRuleAndAlerts({ + query: `id: ${documentId}`, + alerts: 100, + riskScore: 100, + riskScoreOverride: 'event.risk_score', + }); + const { scores } = await getRiskScores({ + body: { + filter: { + bool: { + filter: [ + { + range: { + [ALERT_RISK_SCORE]: { + lte: 1, + }, + }, + }, + ], + }, + }, + }, + }); + + expect(scores).to.have.length(1); + expect(scores[0].riskiestInputs).to.have.length(1); + }); + }); + + describe('risk score ordering', () => { + it('aggregates multiple scores such that the highest-risk scores contribute the majority of the score', async () => { + const documentId = uuidv4(); + const doc = buildDocument({ host: { name: 'host-1' } }, documentId); + await indexListOfDocuments( + Array(100) + .fill(doc) + .map((_doc, i) => ({ ...doc, 'event.risk_score': 100 - i })) + ); + + await createAndSyncRuleAndAlerts({ + query: `id: ${documentId}`, + alerts: 100, + riskScore: 100, + riskScoreOverride: 'event.risk_score', + }); + const { scores } = await getRiskScores({ body: {} }); + + expect(removeFields(scores)).to.eql([ + { + level: 'High', + totalScore: 225.1106801442913, + totalScoreNormalized: 86.18326192354185, + identifierField: 'host.name', + identifierValue: 'host-1', + }, + ]); + }); + }); + + context('with global risk weights', () => { + it('weights host scores differently when host risk weight is configured', async () => { + const documentId = uuidv4(); + const doc = buildDocument({ host: { name: 'host-1' } }, documentId); + await indexListOfDocuments(Array(100).fill(doc)); + + await createAndSyncRuleAndAlerts({ + query: `id: ${documentId}`, + alerts: 100, + riskScore: 100, + }); + const { scores } = await getRiskScores({ + body: { weights: [{ type: 'global_identifier', host: 0.5 }] }, + }); + + expect(removeFields(scores)).to.eql([ + { + level: 'Moderate', + totalScore: 120.6437049351858, + totalScoreNormalized: 46.18824844379242, + identifierField: 'host.name', + identifierValue: 'host-1', + }, + ]); + }); + + it('weights user scores differently if user risk weight is configured', async () => { + const documentId = uuidv4(); + const doc = buildDocument({ user: { name: 'user-1' } }, documentId); + await indexListOfDocuments(Array(100).fill(doc)); + + await createAndSyncRuleAndAlerts({ + query: `id: ${documentId}`, + alerts: 100, + riskScore: 100, + }); + const { scores } = await getRiskScores({ + body: { weights: [{ type: 'global_identifier', user: 0.7 }] }, + }); + + expect(removeFields(scores)).to.eql([ + { + level: 'Moderate', + totalScore: 168.9011869092601, + totalScoreNormalized: 64.66354782130938, + identifierField: 'user.name', + identifierValue: 'user-1', + }, + ]); + }); + + it('weights entity scores differently when host and user risk weights are configured', async () => { + const usersId = uuidv4(); + const hostsId = uuidv4(); + const userDocs = buildDocument({ 'user.name': 'user-1' }, usersId); + const hostDocs = buildDocument({ 'host.name': 'host-1' }, usersId); + await indexListOfDocuments(Array(50).fill(userDocs).concat(Array(50).fill(hostDocs))); + + await createAndSyncRuleAndAlerts({ + query: `id: ${hostsId} OR ${usersId}`, + alerts: 100, + riskScore: 100, + }); + const { scores } = await getRiskScores({ + body: { weights: [{ type: 'global_identifier', host: 0.4, user: 0.8 }] }, + }); + + expect(removeFields(scores)).to.eql([ + { + level: 'High', + totalScore: 186.47518232942502, + totalScoreNormalized: 71.39172370958079, + identifierField: 'user.name', + identifierValue: 'user-1', + }, + { + level: 'Low', + totalScore: 93.23759116471251, + totalScoreNormalized: 35.695861854790394, + identifierField: 'host.name', + identifierValue: 'host-1', + }, + ]); + }); + }); + + context('with category weights', () => { + it('weights risk inputs from different categories according to the category weight', async () => { + const documentId = uuidv4(); + const userSignal = buildDocument( + { 'event.kind': 'signal', 'user.name': 'user-1' }, + documentId + ); + const hostSignal = buildDocument( + { 'event.kind': 'signal', 'host.name': 'host-1' }, + documentId + ); + await indexListOfDocuments(Array(50).fill(userSignal).concat(Array(50).fill(hostSignal))); + + await createAndSyncRuleAndAlerts({ + query: `id: ${documentId}`, + alerts: 100, + riskScore: 100, + }); + const { scores } = await getRiskScores({ + body: { + weights: [{ type: 'risk_category', value: 'alerts', host: 0.4, user: 0.8 }], + }, + }); + + expect(removeFields(scores)).to.eql([ + { + level: 'High', + totalScore: 186.475182329425, + totalScoreNormalized: 71.39172370958079, + identifierField: 'user.name', + identifierValue: 'user-1', + }, + { + level: 'Low', + totalScore: 93.2375911647125, + totalScoreNormalized: 35.695861854790394, + identifierField: 'host.name', + identifierValue: 'host-1', + }, + ]); + }); + }); + }); + }); +};