From 805c770f0170aaef2ffc5f7d4d91c38e3608af76 Mon Sep 17 00:00:00 2001 From: Pablo Machado Date: Wed, 26 Jun 2024 12:10:03 +0200 Subject: [PATCH 1/2] Move/Delete risk score calculation APIs (#186881) ## Summary All internal APIs should be located under the `/internal/` subpath. Entity Analytics team still has 2 internal APIs under `/api/` subpath. This PR addressed the problem by: * Create `/internal/risk_score/calculation/entity` API route * Before we can delete `/api/risk_scores/calculation/entity` we need to create `/internal/risk_score/calculation/entity` and keep both available. We can delete the deprecated API in a later release. * Delete `/api/risk_scores/calculation` API. It was unused and internal. I believe we can safely delete it. --- .../risk_engine/calculation_route.gen.ts | 43 +- .../risk_engine/calculation_route.schema.yaml | 67 +--- .../entity_calculation_route.schema.yaml | 23 ++ .../entity_analytics/risk_score/constants.ts | 10 +- .../risk_score/routes/calculation.test.ts | 143 ------- .../risk_score/routes/calculation.ts | 116 ------ .../risk_score/routes/entity_calculation.ts | 286 ++++++++------ .../routes/register_risk_score_routes.ts | 8 +- .../trial_license_complete_tier/index.ts | 1 - .../risk_score_calculation.ts | 374 ------------------ 10 files changed, 196 insertions(+), 875 deletions(-) delete mode 100644 x-pack/plugins/security_solution/server/lib/entity_analytics/risk_score/routes/calculation.test.ts delete mode 100644 x-pack/plugins/security_solution/server/lib/entity_analytics/risk_score/routes/calculation.ts delete mode 100644 x-pack/test/security_solution_api_integration/test_suites/entity_analytics/risk_engine/trial_license_complete_tier/risk_score_calculation.ts diff --git a/x-pack/plugins/security_solution/common/api/entity_analytics/risk_engine/calculation_route.gen.ts b/x-pack/plugins/security_solution/common/api/entity_analytics/risk_engine/calculation_route.gen.ts index dfec5ee92c36..2abe745e97f4 100644 --- a/x-pack/plugins/security_solution/common/api/entity_analytics/risk_engine/calculation_route.gen.ts +++ b/x-pack/plugins/security_solution/common/api/entity_analytics/risk_engine/calculation_route.gen.ts @@ -10,52 +10,13 @@ * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator. * * info: - * title: Risk Scoring API + * title: RiskScoresCalculation types * version: 1 */ import { z } from 'zod'; -import { - AfterKeys, - DataViewId, - Filter, - PageSize, - IdentifierType, - DateRange, - RiskScoreWeights, - EntityRiskScoreRecord, -} from '../common/common.gen'; - -export type RiskScoresCalculationRequest = z.infer; -export const RiskScoresCalculationRequest = z.object({ - /** - * Used to calculate a specific "page" of risk scores. If unspecified, the first "page" of scores is returned. See also the `after_keys` key in a risk scores response. - */ - after_keys: AfterKeys.optional(), - /** - * The identifier of the Kibana data view to be used when generating risk scores. If a data view is not found, the provided ID will be used as the query's index pattern instead. - */ - data_view_id: DataViewId, - /** - * If set to `true`, the internal ES requests/responses will be logged in Kibana. - */ - debug: z.boolean().optional(), - /** - * An elasticsearch DSL filter object. Used to filter the data being scored, which implicitly filters the risk scores calculated. - */ - filter: Filter.optional(), - page_size: PageSize.optional(), - /** - * Used to restrict the type of risk scores calculated. - */ - identifier_type: IdentifierType, - /** - * Defines the time period over which scores will be evaluated. If unspecified, a range of `[now, now-30d]` will be used. - */ - range: DateRange, - weights: RiskScoreWeights.optional(), -}); +import { AfterKeys, EntityRiskScoreRecord } from '../common/common.gen'; export type RiskScoresCalculationResponse = z.infer; export const RiskScoresCalculationResponse = z.object({ diff --git a/x-pack/plugins/security_solution/common/api/entity_analytics/risk_engine/calculation_route.schema.yaml b/x-pack/plugins/security_solution/common/api/entity_analytics/risk_engine/calculation_route.schema.yaml index 5a290ce7930a..857971ddbf55 100644 --- a/x-pack/plugins/security_solution/common/api/entity_analytics/risk_engine/calculation_route.schema.yaml +++ b/x-pack/plugins/security_solution/common/api/entity_analytics/risk_engine/calculation_route.schema.yaml @@ -1,75 +1,12 @@ openapi: 3.0.0 info: + title: RiskScoresCalculation types version: '1' - title: Risk Scoring API - description: These APIs allow the consumer to manage Entity Risk Scores within Entity Analytics. - -servers: - - url: 'http://{kibana_host}:{port}' - variables: - kibana_host: - default: localhost - port: - default: '5601' - -paths: - /api/risk_scores/calculation: - post: - x-labels: [ess, serverless] - x-internal: true - summary: Trigger calculation of Risk Scores - description: Calculates and persists a segment of Risk Scores, returning details about the calculation. - requestBody: - description: Details about the Risk Scores being calculated - content: - application/json: - schema: - $ref: '#/components/schemas/RiskScoresCalculationRequest' - required: true - responses: - '200': - description: Successful response - content: - application/json: - schema: - $ref: '#/components/schemas/RiskScoresCalculationResponse' - '400': - description: Invalid request +paths: {} components: schemas: - RiskScoresCalculationRequest: - type: object - required: - - data_view_id - - identifier_type - - range - properties: - after_keys: - description: Used to calculate a specific "page" of risk scores. If unspecified, the first "page" of scores is returned. See also the `after_keys` key in a risk scores response. - $ref: '../common/common.schema.yaml#/components/schemas/AfterKeys' - data_view_id: - $ref: '../common/common.schema.yaml#/components/schemas/DataViewId' - description: The identifier of the Kibana data view to be used when generating risk scores. If a data view is not found, the provided ID will be used as the query's index pattern instead. - debug: - description: If set to `true`, the internal ES requests/responses will be logged in Kibana. - type: boolean - filter: - $ref: '../common/common.schema.yaml#/components/schemas/Filter' - description: An elasticsearch DSL filter object. Used to filter the data being scored, which implicitly filters the risk scores calculated. - page_size: - $ref: '../common/common.schema.yaml#/components/schemas/PageSize' - identifier_type: - description: Used to restrict the type of risk scores calculated. - allOf: - - $ref: '../common/common.schema.yaml#/components/schemas/IdentifierType' - range: - $ref: '../common/common.schema.yaml#/components/schemas/DateRange' - description: Defines the time period over which scores will be evaluated. If unspecified, a range of `[now, now-30d]` will be used. - weights: - $ref: '../common/common.schema.yaml#/components/schemas/RiskScoreWeights' - RiskScoresCalculationResponse: type: object required: diff --git a/x-pack/plugins/security_solution/common/api/entity_analytics/risk_engine/entity_calculation_route.schema.yaml b/x-pack/plugins/security_solution/common/api/entity_analytics/risk_engine/entity_calculation_route.schema.yaml index 328c67184e0f..bb9430525488 100644 --- a/x-pack/plugins/security_solution/common/api/entity_analytics/risk_engine/entity_calculation_route.schema.yaml +++ b/x-pack/plugins/security_solution/common/api/entity_analytics/risk_engine/entity_calculation_route.schema.yaml @@ -14,10 +14,33 @@ servers: default: '5601' paths: + # TODO delete on a future serverless release /api/risk_scores/calculation/entity: post: x-labels: [ess, serverless] x-internal: true + summary: Deprecated Trigger calculation of Risk Scores for an entity. Moved to /internal/risk_score/calculation/entity + description: Calculates and persists Risk Scores for an entity, returning the calculated risk score. + requestBody: + description: The entity type and identifier + content: + application/json: + schema: + $ref: '#/components/schemas/RiskScoresEntityCalculationRequest' + required: true + responses: + '200': + description: Successful response + content: + application/json: + schema: + $ref: '#/components/schemas/RiskScoresEntityCalculationResponse' + '400': + description: Invalid request + + /internal/risk_score/calculation/entity: + post: + x-labels: [ess, serverless] summary: Trigger calculation of Risk Scores for an entity description: Calculates and persists Risk Scores for an entity, returning the calculated risk score. requestBody: diff --git a/x-pack/plugins/security_solution/common/entity_analytics/risk_score/constants.ts b/x-pack/plugins/security_solution/common/entity_analytics/risk_score/constants.ts index 808a68871e96..ff31aa502fd2 100644 --- a/x-pack/plugins/security_solution/common/entity_analytics/risk_score/constants.ts +++ b/x-pack/plugins/security_solution/common/entity_analytics/risk_score/constants.ts @@ -5,14 +5,6 @@ * 2.0. */ -/** - * Public Risk Score routes - */ -export const RISK_ENGINE_PUBLIC_PREFIX = '/api/risk_scores' as const; -export const RISK_SCORE_CALCULATION_URL = `${RISK_ENGINE_PUBLIC_PREFIX}/calculation` as const; -export const RISK_SCORE_ENTITY_CALCULATION_URL = - `${RISK_ENGINE_PUBLIC_PREFIX}/calculation/entity` as const; - /** * Internal Risk Score routes */ @@ -36,3 +28,5 @@ export const RISK_SCORE_CREATE_STORED_SCRIPT = export const RISK_SCORE_DELETE_STORED_SCRIPT = `${INTERNAL_RISK_SCORE_URL}/stored_scripts/delete` as const; export const RISK_SCORE_PREVIEW_URL = `${INTERNAL_RISK_SCORE_URL}/preview` as const; +export const RISK_SCORE_ENTITY_CALCULATION_URL = + `${INTERNAL_RISK_SCORE_URL}/calculation/entity` as const; diff --git a/x-pack/plugins/security_solution/server/lib/entity_analytics/risk_score/routes/calculation.test.ts b/x-pack/plugins/security_solution/server/lib/entity_analytics/risk_score/routes/calculation.test.ts deleted file mode 100644 index 9ef1cc8bc210..000000000000 --- a/x-pack/plugins/security_solution/server/lib/entity_analytics/risk_score/routes/calculation.test.ts +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { riskScoreCalculationRoute } from './calculation'; - -import { loggerMock } from '@kbn/logging-mocks'; -import { RISK_SCORE_CALCULATION_URL } from '../../../../../common/constants'; -import { - serverMock, - requestContextMock, - requestMock, -} from '../../../detection_engine/routes/__mocks__'; -import { riskScoreServiceFactory } from '../risk_score_service'; -import { riskScoreServiceMock } from '../risk_score_service.mock'; -import { getRiskInputsIndex } from '../get_risk_inputs_index'; -import { calculateAndPersistRiskScoresMock } from '../calculate_and_persist_risk_scores.mock'; - -jest.mock('../get_risk_inputs_index'); -jest.mock('../risk_score_service'); - -describe('risk score calculation route', () => { - let server: ReturnType; - let { clients, context } = requestContextMock.createTools(); - let logger: ReturnType; - let mockRiskScoreService: ReturnType; - - beforeEach(() => { - jest.resetAllMocks(); - - server = serverMock.create(); - logger = loggerMock.create(); - ({ clients, context } = requestContextMock.createTools()); - mockRiskScoreService = riskScoreServiceMock.create(); - - (getRiskInputsIndex as jest.Mock).mockResolvedValue({ - index: 'default-dataview-index', - runtimeMappings: {}, - }); - clients.appClient.getAlertsIndex.mockReturnValue('default-alerts-index'); - (riskScoreServiceFactory as jest.Mock).mockReturnValue(mockRiskScoreService); - - riskScoreCalculationRoute(server.router, logger); - }); - - const buildRequest = (overrides: object = {}) => { - const defaults = { - data_view_id: 'default-dataview-id', - range: { start: 'now-30d', end: 'now' }, - identifier_type: 'host', - }; - - return requestMock.create({ - method: 'post', - path: RISK_SCORE_CALCULATION_URL, - body: { ...defaults, ...overrides }, - }); - }; - - it('should return 200 when risk score calculation is successful', async () => { - mockRiskScoreService.calculateAndPersistScores.mockResolvedValue( - calculateAndPersistRiskScoresMock.buildResponse() - ); - const request = buildRequest(); - - const response = await server.inject(request, requestContextMock.convertContext(context)); - - expect(response.status).toEqual(200); - }); - - describe('parameters', () => { - it('accepts a parameter for the dataview', async () => { - const request = buildRequest({ data_view_id: 'custom-dataview-id' }); - - const response = await server.inject(request, requestContextMock.convertContext(context)); - - expect(response.status).toEqual(200); - expect(getRiskInputsIndex).toHaveBeenCalledWith( - expect.objectContaining({ dataViewId: 'custom-dataview-id' }) - ); - }); - - it('accepts a parameter for the range', async () => { - const request = buildRequest({ range: { start: 'now-30d', end: 'now-20d' } }); - const response = await server.inject(request, requestContextMock.convertContext(context)); - - expect(response.status).toEqual(200); - expect(mockRiskScoreService.calculateAndPersistScores).toHaveBeenCalledWith( - expect.objectContaining({ range: { start: 'now-30d', end: 'now-20d' } }) - ); - }); - }); - - describe('validation', () => { - describe('required parameters', () => { - it('requires a parameter for the dataview', async () => { - const request = buildRequest({ data_view_id: undefined }); - const result = await server.validate(request); - - expect(result.badRequest).toHaveBeenCalledWith('data_view_id: Required'); - }); - - it('requires a parameter for the date range', async () => { - const request = buildRequest({ range: undefined }); - const result = await server.validate(request); - - expect(result.badRequest).toHaveBeenCalledWith('range: Required'); - }); - - it('requires a parameter for the identifier type', async () => { - const request = buildRequest({ identifier_type: undefined }); - const result = await server.validate(request); - - expect(result.badRequest).toHaveBeenCalledWith('identifier_type: Required'); - }); - }); - - it('uses an unknown dataview as index pattern', async () => { - const request = buildRequest({ data_view_id: 'unknown-dataview' }); - (getRiskInputsIndex as jest.Mock).mockResolvedValue({ - index: 'unknown-dataview', - runtimeMappings: {}, - }); - - const response = await server.inject(request, requestContextMock.convertContext(context)); - - expect(response.status).toEqual(200); - expect(mockRiskScoreService.calculateAndPersistScores).toHaveBeenCalledWith( - expect.objectContaining({ index: 'unknown-dataview', runtimeMappings: {} }) - ); - }); - - it('rejects an invalid date range', async () => { - const request = buildRequest({ range: 'bad range' }); - const result = await server.validate(request); - - expect(result.badRequest).toHaveBeenCalledWith('range: Expected object, received string'); - }); - }); -}); diff --git a/x-pack/plugins/security_solution/server/lib/entity_analytics/risk_score/routes/calculation.ts b/x-pack/plugins/security_solution/server/lib/entity_analytics/risk_score/routes/calculation.ts deleted file mode 100644 index 1602e724db22..000000000000 --- a/x-pack/plugins/security_solution/server/lib/entity_analytics/risk_score/routes/calculation.ts +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { Logger } from '@kbn/core/server'; -import { buildSiemResponse } from '@kbn/lists-plugin/server/routes/utils'; -import { transformError } from '@kbn/securitysolution-es-utils'; -import { buildRouteValidationWithZod } from '@kbn/zod-helpers'; -import { RiskScoresCalculationRequest } from '../../../../../common/api/entity_analytics/risk_engine/calculation_route.gen'; -import { - APP_ID, - DEFAULT_RISK_SCORE_PAGE_SIZE, - RISK_SCORE_CALCULATION_URL, -} from '../../../../../common/constants'; -import { getRiskInputsIndex } from '../get_risk_inputs_index'; -import type { EntityAnalyticsRoutesDeps } from '../../types'; -import { RiskScoreAuditActions } from '../audit'; -import { AUDIT_CATEGORY, AUDIT_OUTCOME, AUDIT_TYPE } from '../../audit'; -import { buildRiskScoreServiceForRequest } from './helpers'; - -export const riskScoreCalculationRoute = ( - router: EntityAnalyticsRoutesDeps['router'], - logger: Logger -) => { - router.versioned - .post({ - path: RISK_SCORE_CALCULATION_URL, - access: 'internal', - options: { - tags: ['access:securitySolution', `access:${APP_ID}-entity-analytics`], - }, - }) - .addVersion( - { - version: '1', - validate: { request: { body: buildRouteValidationWithZod(RiskScoresCalculationRequest) } }, - }, - async (context, request, response) => { - const securityContext = await context.securitySolution; - - securityContext.getAuditLogger()?.log({ - message: 'User triggered custom manual scoring', - event: { - action: RiskScoreAuditActions.RISK_ENGINE_MANUAL_SCORING, - category: AUDIT_CATEGORY.DATABASE, - type: AUDIT_TYPE.CHANGE, - outcome: AUDIT_OUTCOME.UNKNOWN, - }, - }); - - const siemResponse = buildSiemResponse(response); - const coreContext = await context.core; - const soClient = coreContext.savedObjects.client; - const securityConfig = await securityContext.getConfig(); - - const riskScoreService = buildRiskScoreServiceForRequest( - securityContext, - coreContext, - logger - ); - - const { - after_keys: userAfterKeys, - data_view_id: dataViewId, - debug, - page_size: userPageSize, - identifier_type: identifierType, - filter, - range, - weights, - } = request.body; - - try { - const { index, runtimeMappings } = await getRiskInputsIndex({ - dataViewId, - logger, - soClient, - }); - - const afterKeys = userAfterKeys ?? {}; - const pageSize = userPageSize ?? DEFAULT_RISK_SCORE_PAGE_SIZE; - const entityAnalyticsConfig = await riskScoreService.getConfigurationWithDefaults( - securityConfig.entityAnalytics - ); - - const alertSampleSizePerShard = entityAnalyticsConfig?.alertSampleSizePerShard; - - const result = await riskScoreService.calculateAndPersistScores({ - afterKeys, - debug, - pageSize, - identifierType, - index, - filter, - range, - runtimeMappings, - weights, - alertSampleSizePerShard, - }); - - return response.ok({ body: result }); - } catch (e) { - const error = transformError(e); - - return siemResponse.error({ - statusCode: error.statusCode, - body: { message: error.message, full_error: JSON.stringify(e) }, - bypassErrorFormat: true, - }); - } - } - ); -}; diff --git a/x-pack/plugins/security_solution/server/lib/entity_analytics/risk_score/routes/entity_calculation.ts b/x-pack/plugins/security_solution/server/lib/entity_analytics/risk_score/routes/entity_calculation.ts index eeb773b41a18..c521d11d1970 100644 --- a/x-pack/plugins/security_solution/server/lib/entity_analytics/risk_score/routes/entity_calculation.ts +++ b/x-pack/plugins/security_solution/server/lib/entity_analytics/risk_score/routes/entity_calculation.ts @@ -6,10 +6,16 @@ */ import { isEmpty } from 'lodash/fp'; -import type { Logger } from '@kbn/core/server'; +import type { + IKibanaResponse, + KibanaRequest, + KibanaResponseFactory, + Logger, +} from '@kbn/core/server'; import { buildSiemResponse } from '@kbn/lists-plugin/server/routes/utils'; import { transformError } from '@kbn/securitysolution-es-utils'; import { buildRouteValidationWithZod } from '@kbn/zod-helpers'; +import type { SecuritySolutionRequestHandlerContext } from '../../../../types'; import type { RiskScoresCalculationResponse } from '../../../../../common/api/entity_analytics/risk_engine/calculation_route.gen'; import type { AfterKeys } from '../../../../../common/api/entity_analytics/common'; import { RiskScoresEntityCalculationRequest } from '../../../../../common/api/entity_analytics/risk_engine/entity_calculation_route.gen'; @@ -23,6 +29,160 @@ import { buildRiskScoreServiceForRequest } from './helpers'; import { getFieldForIdentifier } from '../helpers'; import { withRiskEnginePrivilegeCheck } from '../../risk_engine/risk_engine_privileges'; +type Handler = ( + context: SecuritySolutionRequestHandlerContext, + request: KibanaRequest, + response: KibanaResponseFactory +) => Promise; + +const handler: (logger: Logger) => Handler = (logger) => async (context, request, response) => { + const securityContext = await context.securitySolution; + + securityContext.getAuditLogger()?.log({ + message: 'User triggered custom manual scoring', + event: { + action: RiskScoreAuditActions.RISK_ENGINE_ENTITY_MANUAL_SCORING, + category: AUDIT_CATEGORY.DATABASE, + type: AUDIT_TYPE.CHANGE, + outcome: AUDIT_OUTCOME.UNKNOWN, + }, + }); + + const coreContext = await context.core; + const securityConfig = await securityContext.getConfig(); + const siemResponse = buildSiemResponse(response); + const soClient = coreContext.savedObjects.client; + + const riskScoreService = buildRiskScoreServiceForRequest(securityContext, coreContext, logger); + + const { identifier_type: identifierType, identifier, refresh } = request.body; + + try { + const entityAnalyticsConfig = await riskScoreService.getConfigurationWithDefaults( + securityConfig.entityAnalytics + ); + + if (entityAnalyticsConfig == null) { + return siemResponse.error({ + statusCode: 400, + body: 'No Risk engine configuration found', + }); + } + + const { + dataViewId, + enabled, + range: configuredRange, + pageSize, + alertSampleSizePerShard, + filter: userFilter, + } = entityAnalyticsConfig; + + if (!enabled) { + return siemResponse.error({ + statusCode: 400, + body: 'Risk engine is disabled', + }); + } + + const { index, runtimeMappings } = await getRiskInputsIndex({ + dataViewId, + logger, + soClient, + }); + + const range = convertRangeToISO(configuredRange); + + const afterKeys: AfterKeys = {}; + + const identifierFilter = { + term: { [getFieldForIdentifier(identifierType)]: identifier }, + }; + + const filter = isEmpty(userFilter) ? [identifierFilter] : [userFilter, identifierFilter]; + + const result: RiskScoresCalculationResponse = await riskScoreService.calculateAndPersistScores({ + pageSize, + identifierType, + index, + filter: { + bool: { + filter, + }, + }, + range, + runtimeMappings, + weights: [], + alertSampleSizePerShard, + afterKeys, + returnScores: true, + refresh, + }); + + if (result.errors.length) { + return siemResponse.error({ + statusCode: 500, + body: { + message: 'Error calculating the risk score for an entity.', + full_error: JSON.stringify(result.errors), + }, + bypassErrorFormat: true, + }); + } + + if (result.scores_written > 0) { + await riskScoreService.scheduleLatestTransformNow(); + } + + const score = result.scores_written === 1 ? result.scores?.[identifierType]?.[0] : undefined; + + return response.ok({ + body: { + success: true, + score, + }, + }); + } catch (e) { + const error = transformError(e); + + return siemResponse.error({ + statusCode: error.statusCode, + body: { message: error.message, full_error: JSON.stringify(e) }, + bypassErrorFormat: true, + }); + } +}; + +/** + * @deprecated + * It will be deleted on a future Serverless release. + */ +export const deprecatedRiskScoreEntityCalculationRoute = ( + router: EntityAnalyticsRoutesDeps['router'], + getStartServices: EntityAnalyticsRoutesDeps['getStartServices'], + logger: Logger +) => { + router.versioned + .post({ + path: '/api/risk_scores/calculation/entity', + access: 'internal', + options: { + tags: ['access:securitySolution', `access:${APP_ID}-entity-analytics`], + }, + }) + .addVersion( + { + version: '1', + validate: { + request: { + body: buildRouteValidationWithZod(RiskScoresEntityCalculationRequest), + }, + }, + }, + withRiskEnginePrivilegeCheck(getStartServices, handler(logger)) + ); +}; + export const riskScoreEntityCalculationRoute = ( router: EntityAnalyticsRoutesDeps['router'], getStartServices: EntityAnalyticsRoutesDeps['getStartServices'], @@ -45,128 +205,6 @@ export const riskScoreEntityCalculationRoute = ( }, }, }, - withRiskEnginePrivilegeCheck(getStartServices, async (context, request, response) => { - const securityContext = await context.securitySolution; - - securityContext.getAuditLogger()?.log({ - message: 'User triggered custom manual scoring', - event: { - action: RiskScoreAuditActions.RISK_ENGINE_ENTITY_MANUAL_SCORING, - category: AUDIT_CATEGORY.DATABASE, - type: AUDIT_TYPE.CHANGE, - outcome: AUDIT_OUTCOME.UNKNOWN, - }, - }); - - const coreContext = await context.core; - const securityConfig = await securityContext.getConfig(); - const siemResponse = buildSiemResponse(response); - const soClient = coreContext.savedObjects.client; - - const riskScoreService = buildRiskScoreServiceForRequest( - securityContext, - coreContext, - logger - ); - - const { identifier_type: identifierType, identifier, refresh } = request.body; - - try { - const entityAnalyticsConfig = await riskScoreService.getConfigurationWithDefaults( - securityConfig.entityAnalytics - ); - - if (entityAnalyticsConfig == null) { - return siemResponse.error({ - statusCode: 400, - body: 'No Risk engine configuration found', - }); - } - - const { - dataViewId, - enabled, - range: configuredRange, - pageSize, - alertSampleSizePerShard, - filter: userFilter, - } = entityAnalyticsConfig; - - if (!enabled) { - return siemResponse.error({ - statusCode: 400, - body: 'Risk engine is disabled', - }); - } - - const { index, runtimeMappings } = await getRiskInputsIndex({ - dataViewId, - logger, - soClient, - }); - - const range = convertRangeToISO(configuredRange); - - const afterKeys: AfterKeys = {}; - - const identifierFilter = { - term: { [getFieldForIdentifier(identifierType)]: identifier }, - }; - - const filter = isEmpty(userFilter) ? [identifierFilter] : [userFilter, identifierFilter]; - - const result: RiskScoresCalculationResponse = - await riskScoreService.calculateAndPersistScores({ - pageSize, - identifierType, - index, - filter: { - bool: { - filter, - }, - }, - range, - runtimeMappings, - weights: [], - alertSampleSizePerShard, - afterKeys, - returnScores: true, - refresh, - }); - - if (result.errors.length) { - return siemResponse.error({ - statusCode: 500, - body: { - message: 'Error calculating the risk score for an entity.', - full_error: JSON.stringify(result.errors), - }, - bypassErrorFormat: true, - }); - } - - if (result.scores_written > 0) { - await riskScoreService.scheduleLatestTransformNow(); - } - - const score = - result.scores_written === 1 ? result.scores?.[identifierType]?.[0] : undefined; - - return response.ok({ - body: { - success: true, - score, - }, - }); - } catch (e) { - const error = transformError(e); - - return siemResponse.error({ - statusCode: error.statusCode, - body: { message: error.message, full_error: JSON.stringify(e) }, - bypassErrorFormat: true, - }); - } - }) + withRiskEnginePrivilegeCheck(getStartServices, handler(logger)) ); }; diff --git a/x-pack/plugins/security_solution/server/lib/entity_analytics/risk_score/routes/register_risk_score_routes.ts b/x-pack/plugins/security_solution/server/lib/entity_analytics/risk_score/routes/register_risk_score_routes.ts index 015b12c5d8ee..1b32ce0bf52b 100644 --- a/x-pack/plugins/security_solution/server/lib/entity_analytics/risk_score/routes/register_risk_score_routes.ts +++ b/x-pack/plugins/security_solution/server/lib/entity_analytics/risk_score/routes/register_risk_score_routes.ts @@ -5,9 +5,11 @@ * 2.0. */ import { riskScorePreviewRoute } from './preview'; -import { riskScoreCalculationRoute } from './calculation'; import type { EntityAnalyticsRoutesDeps } from '../../types'; -import { riskScoreEntityCalculationRoute } from './entity_calculation'; +import { + deprecatedRiskScoreEntityCalculationRoute, + riskScoreEntityCalculationRoute, +} from './entity_calculation'; export const registerRiskScoreRoutes = ({ router, @@ -15,6 +17,6 @@ export const registerRiskScoreRoutes = ({ logger, }: EntityAnalyticsRoutesDeps) => { riskScorePreviewRoute(router, logger); - riskScoreCalculationRoute(router, logger); riskScoreEntityCalculationRoute(router, getStartServices, logger); + deprecatedRiskScoreEntityCalculationRoute(router, getStartServices, logger); }; diff --git a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/risk_engine/trial_license_complete_tier/index.ts b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/risk_engine/trial_license_complete_tier/index.ts index e2af055597f9..4ccce93c790f 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/risk_engine/trial_license_complete_tier/index.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/risk_engine/trial_license_complete_tier/index.ts @@ -10,7 +10,6 @@ import { FtrProviderContext } from '../../../../ftr_provider_context'; export default function ({ loadTestFile }: FtrProviderContext) { describe('Entity Analytics - Risk Engine', function () { loadTestFile(require.resolve('./init_and_status_apis')); - loadTestFile(require.resolve('./risk_score_calculation')); loadTestFile(require.resolve('./risk_score_preview')); loadTestFile(require.resolve('./risk_scoring_task/task_execution')); loadTestFile(require.resolve('./risk_scoring_task/task_execution_nondefault_spaces')); diff --git a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/risk_engine/trial_license_complete_tier/risk_score_calculation.ts b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/risk_engine/trial_license_complete_tier/risk_score_calculation.ts deleted file mode 100644 index 29451ef9dacb..000000000000 --- a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/risk_engine/trial_license_complete_tier/risk_score_calculation.ts +++ /dev/null @@ -1,374 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import expect from '@kbn/expect'; -import { X_ELASTIC_INTERNAL_ORIGIN_REQUEST } from '@kbn/core-http-common'; - -import { RISK_SCORE_CALCULATION_URL } from '@kbn/security-solution-plugin/common/constants'; -import { v4 as uuidv4 } from 'uuid'; -import { EntityRiskScoreRecord } from '@kbn/security-solution-plugin/common/api/entity_analytics/common'; -import { dataGeneratorFactory } from '../../../detections_response/utils'; -import { deleteAllAlerts, deleteAllRules } from '../../../../../common/utils/security_solution'; -import { - buildDocument, - createAndSyncRuleAndAlertsFactory, - deleteAllRiskScores, - readRiskScores, - normalizeScores, - waitForRiskScoresToBePresent, - assetCriticalityRouteHelpersFactory, - cleanAssetCriticality, - waitForAssetCriticalityToBePresent, - getLatestRiskScoreIndexMapping, - riskEngineRouteHelpersFactory, - cleanRiskEngine, - enableAssetCriticalityAdvancedSetting, -} from '../../utils'; -import { FtrProviderContext } from '../../../../ftr_provider_context'; - -export default ({ getService }: FtrProviderContext): void => { - const supertest = getService('supertest'); - - const esArchiver = getService('esArchiver'); - const es = getService('es'); - const log = getService('log'); - const kibanaServer = getService('kibanaServer'); - - const riskEngineRoutes = riskEngineRouteHelpersFactory(supertest); - - const createAndSyncRuleAndAlerts = createAndSyncRuleAndAlertsFactory({ supertest, log }); - - const calculateRiskScores = async ({ - body, - }: { - body: object; - }): Promise<{ scores: EntityRiskScoreRecord[] }> => { - const { body: result } = await supertest - .post(RISK_SCORE_CALCULATION_URL) - .set('kbn-xsrf', 'true') - .set('elastic-api-version', '1') - .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana') - .send(body) - .expect(200); - return result; - }; - - const calculateRiskScoreAfterRuleCreationAndExecution = async ( - documentId: string, - { - alerts = 1, - riskScore = 21, - maxSignals = 100, - }: { alerts?: number; riskScore?: number; maxSignals?: number } = {} - ) => { - await createAndSyncRuleAndAlerts({ query: `id: ${documentId}`, alerts, riskScore, maxSignals }); - - return await calculateRiskScores({ - body: { - data_view_id: '.alerts-security.alerts-default', - range: { start: 'now-30d', end: 'now' }, - identifier_type: 'host', - }, - }); - }; - - describe('@ess @serverless Risk Scoring Calculation API', () => { - before(async () => { - enableAssetCriticalityAdvancedSetting(kibanaServer, log); - }); - - context('with auditbeat data', () => { - const { indexListOfDocuments } = dataGeneratorFactory({ - es, - index: 'ecs_compliant', - log, - }); - - before(async () => { - await esArchiver.load('x-pack/test/functional/es_archives/security_solution/ecs_compliant'); - }); - - after(async () => { - await esArchiver.unload( - 'x-pack/test/functional/es_archives/security_solution/ecs_compliant' - ); - }); - - beforeEach(async () => { - await deleteAllAlerts(supertest, log, es); - await deleteAllRules(supertest, log); - - await cleanRiskEngine({ kibanaServer, es, log }); - await riskEngineRoutes.init(); - }); - - afterEach(async () => { - await deleteAllRiskScores(log, es); - await deleteAllAlerts(supertest, log, es); - await deleteAllRules(supertest, log); - - await cleanRiskEngine({ kibanaServer, es, log }); - }); - - it('calculates and persists risk score', async () => { - const documentId = uuidv4(); - await indexListOfDocuments([buildDocument({ host: { name: 'host-1' } }, documentId)]); - - const results = await calculateRiskScoreAfterRuleCreationAndExecution(documentId); - expect(results).to.eql({ - after_keys: { - host: { - 'host.name': 'host-1', - }, - }, - errors: [], - scores_written: 1, - }); - - await waitForRiskScoresToBePresent({ es, log }); - const scores = await readRiskScores(es); - - expect(scores.length).to.eql(1); - const [score] = normalizeScores(scores); - - expect(score).to.eql({ - calculated_level: 'Unknown', - calculated_score: 21, - calculated_score_norm: 8.10060175898781, - category_1_score: 8.10060175898781, - category_1_count: 1, - id_field: 'host.name', - id_value: 'host-1', - }); - }); - - it('upgrades latest risk score index dynamic setting before persisting risk scores', async () => { - const documentId = uuidv4(); - await indexListOfDocuments([buildDocument({ host: { name: 'host-1' } }, documentId)]); - - await calculateRiskScoreAfterRuleCreationAndExecution(documentId); - - const unmodifiedIndexMapping = await getLatestRiskScoreIndexMapping(es); - // by default, the dynamic mapping is set to false. - expect(unmodifiedIndexMapping?.dynamic).to.eql('false'); - - // set the 'dynamic' configuration to an undesirable value - await es.indices.putMapping({ - index: 'risk-score.risk-score-latest-default', - dynamic: 'strict', - }); - - expect((await getLatestRiskScoreIndexMapping(es))?.dynamic).to.eql('strict'); - - // before re-running risk score persistence, the dynamic configuration should be reset to the desired value - await calculateRiskScoreAfterRuleCreationAndExecution(documentId); - - const finalIndexMapping = await getLatestRiskScoreIndexMapping(es); - - expect(finalIndexMapping?.dynamic).to.eql('false'); - - // after all processing is complete, the mapping should be exactly the same as before - expect(unmodifiedIndexMapping).to.eql(finalIndexMapping); - }); - - describe('paging through calculations', () => { - let documentId: string; - beforeEach(async () => { - documentId = uuidv4(); - const baseEvent = buildDocument({ host: { name: 'host-1' } }, documentId); - await indexListOfDocuments( - Array(10) - .fill(baseEvent) - .map((_baseEvent, index) => ({ - ..._baseEvent, - 'host.name': `host-${index}`, - })) - ); - - await createAndSyncRuleAndAlerts({ - query: `id: ${documentId}`, - alerts: 10, - riskScore: 40, - }); - }); - - it('calculates and persists a single page of risk scores', async () => { - const results = await calculateRiskScores({ - body: { - data_view_id: '.alerts-security.alerts-default', - identifier_type: 'host', - range: { start: 'now-30d', end: 'now' }, - }, - }); - expect(results).to.eql({ - after_keys: { - host: { - 'host.name': 'host-9', - }, - }, - errors: [], - scores_written: 10, - }); - - await waitForRiskScoresToBePresent({ es, log, scoreCount: 10 }); - const scores = await readRiskScores(es); - - expect(scores.length).to.eql(10); - }); - - it('calculates and persists multiple pages of risk scores', async () => { - const results = await calculateRiskScores({ - body: { - data_view_id: '.alerts-security.alerts-default', - identifier_type: 'host', - range: { start: 'now-30d', end: 'now' }, - page_size: 5, - }, - }); - expect(results).to.eql({ - after_keys: { - host: { - 'host.name': 'host-4', - }, - }, - errors: [], - scores_written: 5, - }); - - const secondResults = await calculateRiskScores({ - body: { - after_keys: { - host: { - 'host.name': 'host-4', - }, - }, - data_view_id: '.alerts-security.alerts-default', - identifier_type: 'host', - range: { start: 'now-30d', end: 'now' }, - page_size: 5, - }, - }); - - expect(secondResults).to.eql({ - after_keys: { - host: { - 'host.name': 'host-9', - }, - }, - errors: [], - scores_written: 5, - }); - - await waitForRiskScoresToBePresent({ es, log, scoreCount: 10 }); - const scores = await readRiskScores(es); - - expect(scores.length).to.eql(10); - }); - - it('returns an appropriate response if there are no inputs left to score/persist', async () => { - const results = await calculateRiskScores({ - body: { - data_view_id: '.alerts-security.alerts-default', - identifier_type: 'host', - range: { start: 'now-30d', end: 'now' }, - page_size: 10, - }, - }); - expect(results).to.eql({ - after_keys: { - host: { - 'host.name': 'host-9', - }, - }, - errors: [], - scores_written: 10, - }); - - const noopCalculationResults = await calculateRiskScores({ - body: { - after_keys: { - host: { - 'host.name': 'host-9', - }, - }, - debug: true, - data_view_id: '.alerts-security.alerts-default', - identifier_type: 'host', - range: { start: 'now-30d', end: 'now' }, - page_size: 5, - }, - }); - - expect(noopCalculationResults).to.eql({ - after_keys: {}, - errors: [], - scores_written: 0, - }); - - await waitForRiskScoresToBePresent({ es, log, scoreCount: 10 }); - const scores = await readRiskScores(es); - - expect(scores.length).to.eql(10); - }); - }); - - describe('@skipInServerless with asset criticality data', () => { - const assetCriticalityRoutes = assetCriticalityRouteHelpersFactory(supertest); - - beforeEach(async () => { - await assetCriticalityRoutes.upsert({ - id_field: 'host.name', - id_value: 'host-1', - criticality_level: 'high_impact', - }); - }); - - afterEach(async () => { - await cleanAssetCriticality({ log, es }); - }); - - it('calculates and persists risk scores with additional criticality metadata and modifiers', async () => { - const documentId = uuidv4(); - await indexListOfDocuments([buildDocument({ host: { name: 'host-1' } }, documentId)]); - await waitForAssetCriticalityToBePresent({ es, log }); - - const results = await calculateRiskScoreAfterRuleCreationAndExecution(documentId); - expect(results).to.eql({ - after_keys: { host: { 'host.name': 'host-1' } }, - errors: [], - scores_written: 1, - }); - - await waitForRiskScoresToBePresent({ es, log }); - const scores = await readRiskScores(es); - expect(scores.length).to.eql(1); - - const [score] = normalizeScores(scores); - expect(score).to.eql({ - criticality_level: 'high_impact', - criticality_modifier: 1.5, - calculated_level: 'Unknown', - calculated_score: 21, - calculated_score_norm: 11.677912063468526, - category_1_score: 8.10060175898781, - category_1_count: 1, - id_field: 'host.name', - id_value: 'host-1', - }); - const [rawScore] = scores; - - expect( - rawScore.host?.risk.category_1_score! + rawScore.host?.risk.category_2_score! - ).to.be.within( - score.calculated_score_norm! - 0.000000000000001, - score.calculated_score_norm! + 0.000000000000001 - ); - }); - }); - }); - }); -}; From 34f76adc75be6a88fe6f2b9598e82a37e02363ec Mon Sep 17 00:00:00 2001 From: Georgii Gorbachev Date: Wed, 26 Jun 2024 13:44:49 +0200 Subject: [PATCH 2/2] [Security Solution] Fix `prebuiltRulesCustomizationEnabled` feature flag (#186964) **Resolves: https://github.com/elastic/kibana/issues/180130** **Follow-up to:** https://github.com/elastic/kibana/pull/186823 ## Summary - Adds more information to the feature flag's JSDoc comment according to the template we use for feature flags. - Changes the ticket's link to a public one. --- .../common/experimental_features.ts | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/x-pack/plugins/security_solution/common/experimental_features.ts b/x-pack/plugins/security_solution/common/experimental_features.ts index 0612a7515ea7..4f7fc6c442eb 100644 --- a/x-pack/plugins/security_solution/common/experimental_features.ts +++ b/x-pack/plugins/security_solution/common/experimental_features.ts @@ -233,6 +233,17 @@ export const allowedExperimentalValues = Object.freeze({ */ perFieldPrebuiltRulesDiffingEnabled: true, + /** + * Enables an ability to customize Elastic prebuilt rules. + * + * Ticket: https://github.com/elastic/kibana/issues/174168 + * Owners: https://github.com/orgs/elastic/teams/security-detection-rule-management + * Added: on Jun 24, 2024 in https://github.com/elastic/kibana/pull/186823 + * Turned: TBD + * Expires: TBD + */ + prebuiltRulesCustomizationEnabled: false, + /** * Makes Elastic Defend integration's Malware On-Write Scan option available to edit. */ @@ -267,14 +278,6 @@ export const allowedExperimentalValues = Object.freeze({ * Adds a new option to filter descendants of a process for Management / Event Filters */ filterProcessDescendantsForEventFiltersEnabled: false, - - /** - * Enables an ability to customize Elastic prebuilt rules. - * - * Ticket: https://github.com/elastic/security-team/issues/1974 - * Owners: https://github.com/orgs/elastic/teams/security-detection-rule-management - */ - prebuiltRulesCustomizationEnabled: false, }); type ExperimentalConfigKeys = Array;