From 6cebf21924b6cfa7f0e05d65c51eed9533b4e5b5 Mon Sep 17 00:00:00 2001 From: Khristinin Nikita Date: Mon, 19 Sep 2022 20:15:04 +0200 Subject: [PATCH] Alert Enrichments at ingest time (#139478) * Add threat indicator enrichemnt * [CI] Auto-commit changed files from 'node scripts/eslint --no-cache --fix' * [CI] Auto-commit changed files from 'node scripts/eslint --no-cache --fix' * some temp logs * Add 5 enrichments * some temp logs * Add listClient * Add value list functionalityu * [CI] Auto-commit changed files from 'node scripts/precommit_hook.js --ref HEAD~1..HEAD --fix' * 10 enrichment * [CI] Auto-commit changed files from 'node scripts/eslint --no-cache --fix' * 1 enrichment - 10 idnex * Usage of enrichments * Add host and user risk score enrichments * remove unused loger * check that risk exist on enrichment * typos * sucesfully proceed if some enrichment fails * [CI] Auto-commit changed files from 'node scripts/precommit_hook.js --ref HEAD~1..HEAD --fix' * remove throwing error * Add try catch for enrichAlerts * Add enrichmenst for other rule types * [CI] Auto-commit changed files from 'node scripts/precommit_hook.js --ref HEAD~1..HEAD --fix' * [CI] Auto-commit changed files from 'node scripts/eslint --no-cache --fix' * Add some logging * remove user risk score enablament * [CI] Auto-commit changed files from 'node scripts/precommit_hook.js --ref HEAD~1..HEAD --fix' * [CI] Auto-commit changed files from 'node scripts/eslint --no-cache --fix' * Fix for threshold * chaneg log message * Fix wrong build * [CI] Auto-commit changed files from 'node scripts/eslint --no-cache --fix' * Remove wrong merges * add ecs mapping to alerts * Add default columns * Add score_norm to enrichment * Add host risks UI * fix some types * [CI] Auto-commit changed files from 'node scripts/eslint --no-cache --fix' * Unit tests and refactoring * Add integrations tests * Remove unused tpes * Add some unit tests * Do chunk if there more than 1000 values * Add cypress tests * Change search enrichments to field * Fix translations * Fix types * [CI] Auto-commit changed files from 'node scripts/eslint --no-cache --fix' * remove types * Remove types * Change index name back * Fix types * fix user risk score cypress data * Fix entity tests * Add license check for show the columns * fix typo * Fix tests issue * Add try catch for enrichment * [CI] Auto-commit changed files from 'node scripts/eslint --no-cache --fix' * Remove it after rebase * Add user rusk score support for flyout * Fix typos * Try to fix test * skip enrichment test for now Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- .../create_persistence_rule_type_wrapper.ts | 20 +- .../server/utils/persistence_types.ts | 14 +- .../e2e/dashboards/entity_analytics.cy.ts | 6 +- .../e2e/detection_alerts/enrichments.cy.ts | 79 +++++++ .../cypress/screens/alerts.ts | 14 ++ .../cypress/screens/alerts_details.ts | 2 + .../security_solution/cypress/tasks/alerts.ts | 3 + .../cti_details/host_risk_summary.tsx | 96 ++++---- .../cti_details/threat_summary_view.tsx | 13 +- .../event_details/cti_details/translations.ts | 24 +- .../cti_details/user_risk_summary.tsx | 94 ++++---- .../event_details/event_details.tsx | 27 +-- .../register_alerts_table_configuration.tsx | 4 +- .../alerts_table/default_config.tsx | 21 +- .../components/alerts_table/index.tsx | 16 +- .../rules/rule_preview/preview_histogram.tsx | 7 +- .../security_solution_detections/columns.ts | 127 ++++++----- .../security_solution_detections/index.ts | 4 +- .../factories/bulk_create_factory.ts | 9 +- .../new_terms/create_new_terms_alert_type.ts | 7 +- .../signals/bulk_create_ml_signals.ts | 10 +- .../signals/enrichments/__mocks__/alerts.ts | 193 ++++++++++++++++ ...eate_single_field_match_enrichment.test.ts | 206 +++++++++++++++++ .../create_single_field_match_enrichment.ts | 95 ++++++++ .../enrichment_by_type/host_risk.ts | 65 ++++++ .../enrichment_by_type/user_risk.ts | 64 ++++++ .../signals/enrichments/index.test.ts | 208 ++++++++++++++++++ .../signals/enrichments/index.ts | 82 +++++++ .../signals/enrichments/search_enrichments.ts | 27 +++ .../signals/enrichments/types.ts | 125 +++++++++++ .../signals/enrichments/utils/events.test.ts | 62 ++++++ .../signals/enrichments/utils/events.ts | 21 ++ .../enrichments/utils/requests.test.ts | 66 ++++++ .../signals/enrichments/utils/requests.ts | 33 +++ .../enrichments/utils/transforms.test.ts | 109 +++++++++ .../signals/enrichments/utils/transforms.ts | 37 ++++ .../detection_engine/signals/executors/eql.ts | 11 +- .../signals/executors/threshold.ts | 2 + .../signals/search_after_bulk_create.ts | 10 +- .../bulk_create_threshold_signals.ts | 12 +- .../lib/detection_engine/signals/types.ts | 11 +- .../security_and_spaces/group1/create_ml.ts | 20 ++ .../group1/create_new_terms.ts | 32 +++ .../group1/create_threat_matching.ts | 57 +++++ .../group1/generating_signals.ts | 146 ++++++++++++ .../utils/get_signals_by_ids.ts | 5 +- .../es_archives/entity/host_risk/data.json | 100 +++++++++ .../entity/host_risk/mappings.json | 35 +++ .../es_archives/entity/user_risk/data.json | 39 ++++ .../entity/user_risk/mappings.json | 35 +++ .../es_archives/risky_hosts_updated/data.json | 24 ++ .../risky_hosts_updated/mappings.json | 102 +++++++++ .../es_archives/risky_users/data.json | 25 +++ 53 files changed, 2457 insertions(+), 199 deletions(-) create mode 100644 x-pack/plugins/security_solution/cypress/e2e/detection_alerts/enrichments.cy.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/enrichments/__mocks__/alerts.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/enrichments/create_single_field_match_enrichment.test.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/enrichments/create_single_field_match_enrichment.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/enrichments/enrichment_by_type/host_risk.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/enrichments/enrichment_by_type/user_risk.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/enrichments/index.test.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/enrichments/index.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/enrichments/search_enrichments.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/enrichments/types.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/enrichments/utils/events.test.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/enrichments/utils/events.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/enrichments/utils/requests.test.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/enrichments/utils/requests.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/enrichments/utils/transforms.test.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/enrichments/utils/transforms.ts create mode 100644 x-pack/test/functional/es_archives/entity/host_risk/data.json create mode 100644 x-pack/test/functional/es_archives/entity/host_risk/mappings.json create mode 100644 x-pack/test/functional/es_archives/entity/user_risk/data.json create mode 100644 x-pack/test/functional/es_archives/entity/user_risk/mappings.json create mode 100644 x-pack/test/security_solution_cypress/es_archives/risky_hosts_updated/data.json create mode 100644 x-pack/test/security_solution_cypress/es_archives/risky_hosts_updated/mappings.json diff --git a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts index a1ce484016211..530758772daea 100644 --- a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts +++ b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts @@ -22,7 +22,7 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper ...options, services: { ...options.services, - alertWithPersistence: async (alerts, refresh, maxAlerts = undefined) => { + alertWithPersistence: async (alerts, refresh, maxAlerts = undefined, enrichAlerts) => { const numAlerts = alerts.length; logger.debug(`Found ${numAlerts} alerts.`); @@ -85,13 +85,25 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper return { createdAlerts: [], errors: {}, alertsWereTruncated: false }; } + let enrichedAlerts = filteredAlerts; + + if (enrichAlerts) { + try { + enrichedAlerts = await enrichAlerts(filteredAlerts, { + spaceId: options.spaceId, + }); + } catch (e) { + logger.debug('Enrichemnts failed'); + } + } + let alertsWereTruncated = false; - if (maxAlerts && filteredAlerts.length > maxAlerts) { - filteredAlerts.length = maxAlerts; + if (maxAlerts && enrichedAlerts.length > maxAlerts) { + enrichedAlerts.length = maxAlerts; alertsWereTruncated = true; } - const augmentedAlerts = filteredAlerts.map((alert) => { + const augmentedAlerts = enrichedAlerts.map((alert) => { return { ...alert, _source: { diff --git a/x-pack/plugins/rule_registry/server/utils/persistence_types.ts b/x-pack/plugins/rule_registry/server/utils/persistence_types.ts index 9e5570ffa1afd..63d9cfdd55aea 100644 --- a/x-pack/plugins/rule_registry/server/utils/persistence_types.ts +++ b/x-pack/plugins/rule_registry/server/utils/persistence_types.ts @@ -25,7 +25,19 @@ export type PersistenceAlertService = ( _source: T; }>, refresh: boolean | 'wait_for', - maxAlerts?: number + maxAlerts?: number, + enrichAlerts?: ( + alerts: Array<{ + _id: string; + _source: T; + }>, + params: { spaceId: string } + ) => Promise< + Array<{ + _id: string; + _source: T; + }> + > ) => Promise>; export interface PersistenceAlertServiceResult { diff --git a/x-pack/plugins/security_solution/cypress/e2e/dashboards/entity_analytics.cy.ts b/x-pack/plugins/security_solution/cypress/e2e/dashboards/entity_analytics.cy.ts index 484de2edfd427..39790f602e3af 100644 --- a/x-pack/plugins/security_solution/cypress/e2e/dashboards/entity_analytics.cy.ts +++ b/x-pack/plugins/security_solution/cypress/e2e/dashboards/entity_analytics.cy.ts @@ -83,7 +83,7 @@ describe('Entity Analytics Dashboard', () => { }); it('renders donut chart', () => { - cy.get(USERS_DONUT_CHART).should('include.text', '6Total'); + cy.get(USERS_DONUT_CHART).should('include.text', '7Total'); }); it('renders table', () => { @@ -94,8 +94,8 @@ describe('Entity Analytics Dashboard', () => { it('filters by risk classification', () => { openRiskTableFilterAndSelectTheLowOption(); - cy.get(USERS_DONUT_CHART).should('include.text', '1Total'); - cy.get(USERS_TABLE_ROWS).should('have.length', 1); + cy.get(USERS_DONUT_CHART).should('include.text', '2Total'); + cy.get(USERS_TABLE_ROWS).should('have.length', 2); }); }); diff --git a/x-pack/plugins/security_solution/cypress/e2e/detection_alerts/enrichments.cy.ts b/x-pack/plugins/security_solution/cypress/e2e/detection_alerts/enrichments.cy.ts new file mode 100644 index 0000000000000..685b06009721f --- /dev/null +++ b/x-pack/plugins/security_solution/cypress/e2e/detection_alerts/enrichments.cy.ts @@ -0,0 +1,79 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { getNewRule } from '../../objects/rule'; +import { + NUMBER_OF_ALERTS, + HOST_RISK_HEADER_COLIMN, + USER_RISK_HEADER_COLIMN, + HOST_RISK_COLUMN, + USER_RISK_COLUMN, + ACTION_COLUMN, +} from '../../screens/alerts'; +import { ENRICHED_DATA_ROW } from '../../screens/alerts_details'; +import { esArchiverLoad, esArchiverUnload } from '../../tasks/es_archiver'; + +import { createCustomRuleEnabled } from '../../tasks/api_calls/rules'; +import { cleanKibana, deleteAlertsAndRules } from '../../tasks/common'; +import { waitForAlertsToPopulate } from '../../tasks/create_new_rule'; +import { + expandFirstAlert, + scrollAlertTableColumnIntoView, + closeAlertFlyout, +} from '../../tasks/alerts'; + +import { login, visit } from '../../tasks/login'; + +import { ALERTS_URL } from '../../urls/navigation'; + +describe.skip('Enrichment', () => { + before(() => { + cleanKibana(); + esArchiverLoad('risky_hosts'); + esArchiverLoad('risky_users'); + login(); + }); + + after(() => { + esArchiverUnload('risky_hosts'); + esArchiverUnload('risky_users'); + esArchiverUnload('risky_hosts_updated'); + }); + describe('Custom query rule', () => { + beforeEach(() => { + deleteAlertsAndRules(); + createCustomRuleEnabled(getNewRule(), 'rule1'); + visit(ALERTS_URL); + waitForAlertsToPopulate(); + }); + + it('Should has enrichment fields', function () { + cy.get(NUMBER_OF_ALERTS) + .invoke('text') + .should('match', /^[1-9].+$/); // Any number of alerts + cy.get(HOST_RISK_HEADER_COLIMN).contains('host.risk.calculated_level'); + cy.get(USER_RISK_HEADER_COLIMN).contains('user.risk.calculated_level'); + scrollAlertTableColumnIntoView(HOST_RISK_COLUMN); + cy.get(HOST_RISK_COLUMN).contains('Low'); + scrollAlertTableColumnIntoView(USER_RISK_COLUMN); + cy.get(USER_RISK_COLUMN).contains('Low'); + scrollAlertTableColumnIntoView(ACTION_COLUMN); + expandFirstAlert(); + cy.get(ENRICHED_DATA_ROW).contains('Low'); + cy.get(ENRICHED_DATA_ROW).contains('Current host risk classification'); + cy.get(ENRICHED_DATA_ROW).contains('Critical').should('not.exist'); + cy.get(ENRICHED_DATA_ROW).contains('Original host risk classification').should('not.exist'); + + closeAlertFlyout(); + esArchiverUnload('risky_hosts'); + esArchiverLoad('risky_hosts_updated'); + expandFirstAlert(); + cy.get(ENRICHED_DATA_ROW).contains('Critical'); + cy.get(ENRICHED_DATA_ROW).contains('Original host risk classification'); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/cypress/screens/alerts.ts b/x-pack/plugins/security_solution/cypress/screens/alerts.ts index 8d484bf0111d1..28ccdef683971 100644 --- a/x-pack/plugins/security_solution/cypress/screens/alerts.ts +++ b/x-pack/plugins/security_solution/cypress/screens/alerts.ts @@ -44,6 +44,8 @@ export const EMPTY_ALERT_TABLE = '[data-test-subj="tGridEmptyState"]'; export const EXPAND_ALERT_BTN = '[data-test-subj="expand-event"]'; +export const CLOSE_FLYOUT = '[data-test-subj="euiFlyoutCloseButton"]'; + export const GROUP_BY_TOP_INPUT = '[data-test-subj="groupByTop"] [data-test-subj="comboBoxInput"]'; export const HOST_NAME = '[data-test-subj^=formatted-field][data-test-subj$=host\\.name]'; @@ -92,3 +94,15 @@ export const USER_NAME = '[data-test-subj^=formatted-field][data-test-subj$=user export const ATTACH_ALERT_TO_CASE_BUTTON = '[data-test-subj="add-to-existing-case-action"]'; export const USER_COLUMN = '[data-gridcell-column-id="user.name"]'; + +export const HOST_RISK_HEADER_COLIMN = + '[data-test-subj="dataGridHeaderCell-host.risk.calculated_level"]'; + +export const HOST_RISK_COLUMN = '[data-gridcell-column-id="host.risk.calculated_level"]'; + +export const USER_RISK_HEADER_COLIMN = + '[data-test-subj="dataGridHeaderCell-user.risk.calculated_level"]'; + +export const USER_RISK_COLUMN = '[data-gridcell-column-id="user.risk.calculated_level"]'; + +export const ACTION_COLUMN = '[data-gridcell-column-id="default-timeline-control-column"]'; diff --git a/x-pack/plugins/security_solution/cypress/screens/alerts_details.ts b/x-pack/plugins/security_solution/cypress/screens/alerts_details.ts index 3384deafbb7c5..ee8bbd66b771d 100644 --- a/x-pack/plugins/security_solution/cypress/screens/alerts_details.ts +++ b/x-pack/plugins/security_solution/cypress/screens/alerts_details.ts @@ -81,3 +81,5 @@ export const INSIGHTS_INVESTIGATE_IN_TIMELINE_BUTTON = `${INSIGHTS_RELATED_ALERT export const INSIGHTS_RELATED_ALERTS_BY_ANCESTRY = `[data-test-subj='related-alerts-by-ancestry']`; export const INSIGHTS_INVESTIGATE_ANCESTRY_ALERTS_IN_TIMELINE_BUTTON = `[data-test-subj='investigate-ancestry-in-timeline']`; + +export const ENRICHED_DATA_ROW = `[data-test-subj='EnrichedDataRow']`; diff --git a/x-pack/plugins/security_solution/cypress/tasks/alerts.ts b/x-pack/plugins/security_solution/cypress/tasks/alerts.ts index 16f8568bf7a82..69a0159f7f44d 100644 --- a/x-pack/plugins/security_solution/cypress/tasks/alerts.ts +++ b/x-pack/plugins/security_solution/cypress/tasks/alerts.ts @@ -24,6 +24,7 @@ import { SELECT_TABLE, TAKE_ACTION_POPOVER_BTN, TIMELINE_CONTEXT_MENU_BTN, + CLOSE_FLYOUT, } from '../screens/alerts'; import { REFRESH_BUTTON } from '../screens/security_header'; import { @@ -82,6 +83,8 @@ export const expandFirstAlert = () => { .pipe(($el) => $el.trigger('click')); }; +export const closeAlertFlyout = () => cy.get(CLOSE_FLYOUT).click(); + export const viewThreatIntelTab = () => cy.get(THREAT_INTEL_TAB).click(); export const setEnrichmentDates = (from?: string, to?: string) => { diff --git a/x-pack/plugins/security_solution/public/common/components/event_details/cti_details/host_risk_summary.tsx b/x-pack/plugins/security_solution/public/common/components/event_details/cti_details/host_risk_summary.tsx index 536d77c8c8319..258064917b9c0 100644 --- a/x-pack/plugins/security_solution/public/common/components/event_details/cti_details/host_risk_summary.tsx +++ b/x-pack/plugins/security_solution/public/common/components/event_details/cti_details/host_risk_summary.tsx @@ -11,56 +11,66 @@ import { FormattedMessage } from '@kbn/i18n-react'; import * as i18n from './translations'; import { EnrichedDataRow, ThreatSummaryPanelHeader } from './threat_summary_view'; import { RiskScore } from '../../severity/common'; +import type { RiskSeverity } from '../../../../../common/search_strategy'; import type { HostRisk } from '../../../../risk_score/containers'; import { getEmptyValue } from '../../empty_value'; import { RISKY_HOSTS_DOC_LINK } from '../../../../../common/constants'; const HostRiskSummaryComponent: React.FC<{ hostRisk: HostRisk; -}> = ({ hostRisk }) => ( - <> - - - - - ), - }} - /> - } - /> + originalHostRisk?: RiskSeverity | undefined; +}> = ({ hostRisk, originalHostRisk }) => { + const currentHostRiskScore = hostRisk?.result?.[0]?.host?.risk?.calculated_level; + return ( + <> + + + + + ), + }} + /> + } + /> - {hostRisk.loading && } + {hostRisk.loading && } - {!hostRisk.loading && ( - <> - 0 ? ( - - ) : ( - getEmptyValue() - ) - } - /> - - )} - - -); + {!hostRisk.loading && ( + <> + + ) : ( + getEmptyValue() + ) + } + /> + {originalHostRisk && currentHostRiskScore !== originalHostRisk && ( + <> + } + /> + + )} + + )} + + + ); +}; export const HostRiskSummary = React.memo(HostRiskSummaryComponent); diff --git a/x-pack/plugins/security_solution/public/common/components/event_details/cti_details/threat_summary_view.tsx b/x-pack/plugins/security_solution/public/common/components/event_details/cti_details/threat_summary_view.tsx index 866d7ae0a6c39..1b28162ad3c55 100644 --- a/x-pack/plugins/security_solution/public/common/components/event_details/cti_details/threat_summary_view.tsx +++ b/x-pack/plugins/security_solution/public/common/components/event_details/cti_details/threat_summary_view.tsx @@ -24,6 +24,7 @@ import type { CtiEnrichment } from '../../../../../common/search_strategy/securi import type { BrowserFields, TimelineEventsDetailsItem, + RiskSeverity, } from '../../../../../common/search_strategy'; import { HostRiskSummary } from './host_risk_summary'; import { UserRiskSummary } from './user_risk_summary'; @@ -141,6 +142,14 @@ const ThreatSummaryViewComponent: React.FC<{ isDraggable, isReadOnly, }) => { + const originalHostRisk = data?.find( + (eventDetail) => eventDetail?.field === 'host.risk.calculated_level' + )?.values?.[0] as RiskSeverity | undefined; + + const originalUserRisk = data?.find( + (eventDetail) => eventDetail?.field === 'user.risk.calculated_level' + )?.values?.[0] as RiskSeverity | undefined; + return ( <> @@ -152,11 +161,11 @@ const ThreatSummaryViewComponent: React.FC<{ - + - + = ({ userRisk }) => ( - <> - - - - - ), - }} - /> - } - /> + originalUserRisk?: RiskSeverity | undefined; +}> = ({ userRisk, originalUserRisk }) => { + const currentUserRiskScore = userRisk?.result?.[0]?.user?.risk?.calculated_level; + return ( + <> + + + + + ), + }} + /> + } + /> - {userRisk.loading && } + {userRisk.loading && } - {!userRisk.loading && ( - <> - 0 ? ( - + + ) : ( + getEmptyValue() + ) + } + /> + {originalUserRisk && currentUserRiskScore !== originalUserRisk && ( + <> + } /> - ) : ( - getEmptyValue() - ) - } - /> - - )} - - -); - + + )} + + )} + + + ); +}; export const UserRiskSummary = React.memo(UserRiskSummaryComponent); diff --git a/x-pack/plugins/security_solution/public/common/components/event_details/event_details.tsx b/x-pack/plugins/security_solution/public/common/components/event_details/event_details.tsx index 92f4c57964e10..c000803f04a5b 100644 --- a/x-pack/plugins/security_solution/public/common/components/event_details/event_details.tsx +++ b/x-pack/plugins/security_solution/public/common/components/event_details/event_details.tsx @@ -193,19 +193,20 @@ const EventDetailsComponent: React.FC = ({ isReadOnly={isReadOnly} /> - {enrichmentCount > 0 && isLicenseValid && ( - - )} + {enrichmentCount > 0 || + (isLicenseValid && (hostRisk || userRisk) && ( + + ))} {isEnrichmentsLoading && ( <> diff --git a/x-pack/plugins/security_solution/public/common/lib/triggers_actions_ui/register_alerts_table_configuration.tsx b/x-pack/plugins/security_solution/public/common/lib/triggers_actions_ui/register_alerts_table_configuration.tsx index d8f1b984b8b09..6fcb511a755e7 100644 --- a/x-pack/plugins/security_solution/public/common/lib/triggers_actions_ui/register_alerts_table_configuration.tsx +++ b/x-pack/plugins/security_solution/public/common/lib/triggers_actions_ui/register_alerts_table_configuration.tsx @@ -14,7 +14,7 @@ import type { import { APP_ID, CASES_FEATURE_ID } from '../../../../common/constants'; import { getTimelinesInStorageByIds } from '../../../timelines/containers/local_storage'; import { TimelineId } from '../../../../common/types'; -import { columns } from '../../../detections/configurations/security_solution_detections'; +import { getColumns } from '../../../detections/configurations/security_solution_detections'; import { useRenderCellValue } from '../../../detections/configurations/security_solution_detections/render_cell_value'; import { useToGetInternalFlyout } from '../../../timelines/components/side_panel/event_details/flyout'; @@ -27,7 +27,7 @@ const registerAlertsTableConfiguration = ( } const timelineStorage = getTimelinesInStorageByIds(storage, [TimelineId.detectionsPage]); const columnsFormStorage = timelineStorage?.[TimelineId.detectionsPage]?.columns ?? []; - const alertColumns = columnsFormStorage.length ? columnsFormStorage : columns; + const alertColumns = columnsFormStorage.length ? columnsFormStorage : getColumns(); registry.register({ id: APP_ID, diff --git a/x-pack/plugins/security_solution/public/detections/components/alerts_table/default_config.tsx b/x-pack/plugins/security_solution/public/detections/components/alerts_table/default_config.tsx index f1a52ac7b6997..0c577287b3919 100644 --- a/x-pack/plugins/security_solution/public/detections/components/alerts_table/default_config.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/alerts_table/default_config.tsx @@ -17,9 +17,10 @@ import type { Status } from '../../../../common/detection_engine/schemas/common/ import type { SubsetTimelineModel } from '../../../timelines/store/timeline/model'; import { timelineDefaults } from '../../../timelines/store/timeline/defaults'; import { - columns, - rulePreviewColumns, + getColumns, + getRulePreviewColumns, } from '../../configurations/security_solution_detections/columns'; +import type { LicenseService } from '../../../../common/license'; export const buildAlertStatusFilter = (status: Status): Filter[] => { const combinedQuery = @@ -152,17 +153,17 @@ export const buildThreatMatchFilter = (showOnlyThreatIndicatorAlerts: boolean): ] : []; -export const alertsDefaultModel: SubsetTimelineModel = { +export const getAlertsDefaultModel = (license?: LicenseService): SubsetTimelineModel => ({ ...timelineDefaults, - columns, + columns: getColumns(license), showCheckboxes: true, excludedRowRendererIds: Object.values(RowRendererId), -}; +}); -export const alertsPreviewDefaultModel: SubsetTimelineModel = { - ...alertsDefaultModel, - columns: rulePreviewColumns, - defaultColumns: rulePreviewColumns, +export const getAlertsPreviewDefaultModel = (license?: LicenseService): SubsetTimelineModel => ({ + ...getAlertsDefaultModel(license), + columns: getColumns(license), + defaultColumns: getRulePreviewColumns(license), sort: [ { columnId: 'kibana.alert.original_time', @@ -171,7 +172,7 @@ export const alertsPreviewDefaultModel: SubsetTimelineModel = { sortDirection: 'desc', }, ], -}; +}); export const requiredFieldsForActions = [ '@timestamp', diff --git a/x-pack/plugins/security_solution/public/detections/components/alerts_table/index.tsx b/x-pack/plugins/security_solution/public/detections/components/alerts_table/index.tsx index d4dead20989a6..0a647b2469d2c 100644 --- a/x-pack/plugins/security_solution/public/detections/components/alerts_table/index.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/alerts_table/index.tsx @@ -29,15 +29,17 @@ import { combineQueries } from '../../../timelines/components/timeline/helpers'; import { timelineActions, timelineSelectors } from '../../../timelines/store/timeline'; import { timelineDefaults } from '../../../timelines/store/timeline/defaults'; import type { TimelineModel } from '../../../timelines/store/timeline/model'; -import { columns, RenderCellValue } from '../../configurations/security_solution_detections'; +import { getColumns, RenderCellValue } from '../../configurations/security_solution_detections'; import { AdditionalFiltersAction } from './additional_filters_action'; import { - alertsDefaultModel, + getAlertsDefaultModel, buildAlertStatusFilter, requiredFieldsForActions, } from './default_config'; import { buildTimeRangeFilter } from './helpers'; import * as i18n from './translations'; +import { useLicense } from '../../../common/hooks/use_license'; + interface OwnProps { defaultFilters?: Filter[]; from: string; @@ -83,6 +85,7 @@ export const AlertsTableComponent: React.FC = ({ } = useSourcererDataView(SourcererScopeName.detections); const kibana = useKibana(); const ACTION_BUTTON_COUNT = 5; + const license = useLicense(); const getGlobalQuery = useCallback( (customFilters: Filter[]) => { @@ -163,7 +166,7 @@ export const AlertsTableComponent: React.FC = ({ useEffect(() => { dispatch( timelineActions.initializeTGridSettings({ - defaultColumns: columns.map((c) => + defaultColumns: getColumns(license).map((c) => !tGridEnabled && c.initialWidth == null ? { ...c, @@ -172,7 +175,8 @@ export const AlertsTableComponent: React.FC = ({ : c ), documentType: i18n.ALERTS_DOCUMENT_TYPE, - excludedRowRendererIds: alertsDefaultModel.excludedRowRendererIds as RowRendererId[], + excludedRowRendererIds: getAlertsDefaultModel(license) + .excludedRowRendererIds as RowRendererId[], filterManager, footerText: i18n.TOTAL_COUNT_OF_ALERTS, id: timelineId, @@ -183,7 +187,7 @@ export const AlertsTableComponent: React.FC = ({ showCheckboxes: true, }) ); - }, [dispatch, filterManager, tGridEnabled, timelineId]); + }, [dispatch, filterManager, tGridEnabled, timelineId, license]); const leadingControlColumns = useMemo(() => getDefaultControlColumn(ACTION_BUTTON_COUNT), []); @@ -196,7 +200,7 @@ export const AlertsTableComponent: React.FC = ({ additionalFilters={additionalFiltersComponent} currentFilter={filterGroup} defaultCellActions={defaultCellActions} - defaultModel={alertsDefaultModel} + defaultModel={getAlertsDefaultModel(license)} end={to} entityType="events" hasAlertsCrud={hasIndexWrite && hasIndexMaintenance} diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/rule_preview/preview_histogram.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/rule_preview/preview_histogram.tsx index 278919555303c..df841c954375d 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/rule_preview/preview_histogram.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/rule_preview/preview_histogram.tsx @@ -26,7 +26,7 @@ import { Panel } from '../../../../common/components/panel'; import { HeaderSection } from '../../../../common/components/header_section'; import { BarChart } from '../../../../common/components/charts/barchart'; import { usePreviewHistogram } from './use_preview_histogram'; -import { alertsPreviewDefaultModel } from '../../alerts_table/default_config'; +import { getAlertsPreviewDefaultModel } from '../../alerts_table/default_config'; import { SourcererScopeName } from '../../../../common/store/sourcerer/model'; import { defaultRowRenderers } from '../../../../timelines/components/timeline/body/renderers'; import { TimelineId } from '../../../../../common/types'; @@ -41,6 +41,7 @@ import { InspectButtonContainer } from '../../../../common/components/inspect'; import { timelineActions } from '../../../../timelines/store/timeline'; import type { State } from '../../../../common/store'; import type { TimeframePreviewOptions } from '../../../pages/detection_engine/rules/types'; +import { useLicense } from '../../../../common/hooks/use_license'; const LoadingChart = styled(EuiLoadingChart)` display: block; @@ -94,7 +95,7 @@ export const PreviewHistogram = ({ indexPattern, ruleType, }); - + const license = useLicense(); const { timeline: { columns, @@ -105,7 +106,7 @@ export const PreviewHistogram = ({ itemsPerPageOptions, kqlMode, sort, - } = alertsPreviewDefaultModel, + } = getAlertsPreviewDefaultModel(license), } = useSelector((state: State) => eventsViewerSelector(state, TimelineId.rulePreview)); const { diff --git a/x-pack/plugins/security_solution/public/detections/configurations/security_solution_detections/columns.ts b/x-pack/plugins/security_solution/public/detections/configurations/security_solution_detections/columns.ts index 229694a4d3b43..616578b38bcf6 100644 --- a/x-pack/plugins/security_solution/public/detections/configurations/security_solution_detections/columns.ts +++ b/x-pack/plugins/security_solution/public/detections/configurations/security_solution_detections/columns.ts @@ -6,6 +6,7 @@ */ import type { EuiDataGridColumn } from '@elastic/eui'; +import type { LicenseService } from '../../../../common/license'; import type { ColumnHeaderOptions } from '../../../../common/types'; import { defaultColumnHeaderType } from '../../../timelines/components/timeline/body/column_headers/default_headers'; @@ -16,60 +17,82 @@ import { import * as i18n from '../../components/alerts_table/translations'; -const baseColumns: Array< +const getBaseColumns = ( + license?: LicenseService +): Array< Pick & ColumnHeaderOptions -> = [ - { - columnHeaderType: defaultColumnHeaderType, - displayAsText: i18n.ALERTS_HEADERS_SEVERITY, - id: 'kibana.alert.severity', - initialWidth: 105, - }, - { - columnHeaderType: defaultColumnHeaderType, - displayAsText: i18n.ALERTS_HEADERS_RISK_SCORE, - id: 'kibana.alert.risk_score', - initialWidth: 100, - }, - { - columnHeaderType: defaultColumnHeaderType, - displayAsText: i18n.ALERTS_HEADERS_REASON, - id: 'kibana.alert.reason', - initialWidth: 450, - }, - { - columnHeaderType: defaultColumnHeaderType, - id: 'host.name', - }, - { - columnHeaderType: defaultColumnHeaderType, - id: 'user.name', - }, - { - columnHeaderType: defaultColumnHeaderType, - id: 'process.name', - }, - { - columnHeaderType: defaultColumnHeaderType, - id: 'file.name', - }, - { - columnHeaderType: defaultColumnHeaderType, - id: 'source.ip', - }, - { - columnHeaderType: defaultColumnHeaderType, - id: 'destination.ip', - }, -]; +> => { + const isPlatinumPlus = license?.isPlatinumPlus?.() ?? false; + return [ + { + columnHeaderType: defaultColumnHeaderType, + displayAsText: i18n.ALERTS_HEADERS_SEVERITY, + id: 'kibana.alert.severity', + initialWidth: 105, + }, + { + columnHeaderType: defaultColumnHeaderType, + displayAsText: i18n.ALERTS_HEADERS_RISK_SCORE, + id: 'kibana.alert.risk_score', + initialWidth: 100, + }, + { + columnHeaderType: defaultColumnHeaderType, + displayAsText: i18n.ALERTS_HEADERS_REASON, + id: 'kibana.alert.reason', + initialWidth: 450, + }, + { + columnHeaderType: defaultColumnHeaderType, + id: 'host.name', + }, + isPlatinumPlus + ? { + columnHeaderType: defaultColumnHeaderType, + id: 'host.risk.calculated_level', + } + : null, + { + columnHeaderType: defaultColumnHeaderType, + id: 'user.name', + }, + isPlatinumPlus + ? { + columnHeaderType: defaultColumnHeaderType, + id: 'user.risk.calculated_level', + } + : null, + { + columnHeaderType: defaultColumnHeaderType, + id: 'process.name', + }, + { + columnHeaderType: defaultColumnHeaderType, + id: 'file.name', + }, + { + columnHeaderType: defaultColumnHeaderType, + id: 'source.ip', + }, + { + columnHeaderType: defaultColumnHeaderType, + id: 'destination.ip', + }, + ].filter((column) => column != null) as Array< + Pick & + ColumnHeaderOptions + >; +}; /** * columns implements a subset of `EuiDataGrid`'s `EuiDataGridColumn` interface, * plus additional TGrid column properties */ -export const columns: Array< +export const getColumns = ( + license?: LicenseService +): Array< Pick & ColumnHeaderOptions -> = [ +> => [ { columnHeaderType: defaultColumnHeaderType, id: '@timestamp', @@ -82,16 +105,18 @@ export const columns: Array< initialWidth: DEFAULT_COLUMN_MIN_WIDTH, linkField: 'kibana.alert.rule.uuid', }, - ...baseColumns, + ...getBaseColumns(license), ]; -export const rulePreviewColumns: Array< +export const getRulePreviewColumns = ( + license?: LicenseService +): Array< Pick & ColumnHeaderOptions -> = [ +> => [ { columnHeaderType: defaultColumnHeaderType, id: 'kibana.alert.original_time', initialWidth: DEFAULT_DATE_COLUMN_MIN_WIDTH + 10, }, - ...baseColumns, + ...getBaseColumns(license), ]; diff --git a/x-pack/plugins/security_solution/public/detections/configurations/security_solution_detections/index.ts b/x-pack/plugins/security_solution/public/detections/configurations/security_solution_detections/index.ts index dfd4d9499f6e5..3a96684d59dc6 100644 --- a/x-pack/plugins/security_solution/public/detections/configurations/security_solution_detections/index.ts +++ b/x-pack/plugins/security_solution/public/detections/configurations/security_solution_detections/index.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { columns } from './columns'; +import { getColumns } from './columns'; import { RenderCellValue } from './render_cell_value'; -export { columns, RenderCellValue }; +export { getColumns, RenderCellValue }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/bulk_create_factory.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/bulk_create_factory.ts index fadba855baa32..81f15364ff128 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/bulk_create_factory.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/bulk_create_factory.ts @@ -35,7 +35,11 @@ export const bulkCreateFactory = ) => async ( wrappedDocs: Array>, - maxAlerts?: number + maxAlerts?: number, + enrichAlerts?: ( + alerts: Array, '_id' | '_source'>>, + params: { spaceId: string } + ) => Promise, '_id' | '_source'>>> ): Promise> => { if (wrappedDocs.length === 0) { return { @@ -57,7 +61,8 @@ export const bulkCreateFactory = _source: doc._source, })), refreshForBulkCreate, - maxAlerts + maxAlerts, + enrichAlerts ); const end = performance.now(); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts index 8a1f941a92f31..3a4f913712e3e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts @@ -30,6 +30,7 @@ import type { SignalSource } from '../../signals/types'; import { validateIndexPatterns } from '../utils'; import { parseDateString, validateHistoryWindowStart } from './utils'; import { addToSearchAfterReturn, createSearchAfterReturnType } from '../../signals/utils'; +import { createEnrichEventsFunction } from '../../signals/enrichments'; export const createNewTermsAlertType = ( createOptions: CreateRuleOptions @@ -275,7 +276,11 @@ export const createNewTermsAlertType = ( const bulkCreateResult = await bulkCreate( wrappedAlerts, - params.maxSignals - result.createdSignalsCount + params.maxSignals - result.createdSignalsCount, + createEnrichEventsFunction({ + services, + logger: ruleExecutionLogger, + }) ); addToSearchAfterReturn({ current: result, next: bulkCreateResult }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_ml_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_ml_signals.ts index 11b7a035d8a2f..f9a4c21898b09 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_ml_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_ml_signals.ts @@ -20,6 +20,7 @@ import type { CompleteRule, MachineLearningRuleParams } from '../schemas/rule_sc import { buildReasonMessageForMlAlert } from './reason_formatters'; import type { BaseFieldsLatest } from '../../../../common/detection_engine/schemas/alerts'; import type { IRuleExecutionLogForExecutors } from '../rule_monitoring'; +import { createEnrichEventsFunction } from './enrichments'; interface BulkCreateMlSignalsParams { anomalyHits: Array>; @@ -81,5 +82,12 @@ export const bulkCreateMlSignals = async ( const ecsResults = transformAnomalyResultsToEcs(anomalyResults); const wrappedDocs = params.wrapHits(ecsResults, buildReasonMessageForMlAlert); - return params.bulkCreate(wrappedDocs); + return params.bulkCreate( + wrappedDocs, + undefined, + createEnrichEventsFunction({ + services: params.services, + logger: params.ruleExecutionLogger, + }) + ); }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/enrichments/__mocks__/alerts.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/enrichments/__mocks__/alerts.ts new file mode 100644 index 0000000000000..f837553b8b062 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/enrichments/__mocks__/alerts.ts @@ -0,0 +1,193 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { + ALERT_BUILDING_BLOCK_TYPE, + ALERT_REASON, + ALERT_RISK_SCORE, + ALERT_RULE_AUTHOR, + ALERT_RULE_CATEGORY, + ALERT_RULE_CONSUMER, + ALERT_RULE_CREATED_AT, + ALERT_RULE_CREATED_BY, + ALERT_RULE_DESCRIPTION, + ALERT_RULE_ENABLED, + ALERT_RULE_EXECUTION_UUID, + ALERT_RULE_FROM, + ALERT_RULE_INTERVAL, + ALERT_RULE_LICENSE, + ALERT_RULE_NAME, + ALERT_RULE_NAMESPACE_FIELD, + ALERT_RULE_NOTE, + ALERT_RULE_PARAMETERS, + ALERT_RULE_PRODUCER, + ALERT_RULE_REFERENCES, + ALERT_RULE_RULE_ID, + ALERT_RULE_RULE_NAME_OVERRIDE, + ALERT_RULE_TAGS, + ALERT_RULE_TO, + ALERT_RULE_TYPE, + ALERT_RULE_TYPE_ID, + ALERT_RULE_UPDATED_AT, + ALERT_RULE_UPDATED_BY, + ALERT_RULE_UUID, + ALERT_RULE_VERSION, + ALERT_SEVERITY, + ALERT_STATUS, + ALERT_STATUS_ACTIVE, + ALERT_UUID, + ALERT_WORKFLOW_STATUS, + EVENT_KIND, + SPACE_IDS, + TIMESTAMP, +} from '@kbn/rule-data-utils'; + +import type { EventsForEnrichment } from '../types'; +import type { BaseFieldsLatest } from '../../../../../../common/detection_engine/schemas/alerts'; + +import { + ALERT_ANCESTORS, + ALERT_DEPTH, + ALERT_ORIGINAL_TIME, + ALERT_RULE_ACTIONS, + ALERT_RULE_EXCEPTIONS_LIST, + ALERT_RULE_FALSE_POSITIVES, + ALERT_RULE_IMMUTABLE, + ALERT_RULE_MAX_SIGNALS, + ALERT_RULE_RISK_SCORE_MAPPING, + ALERT_RULE_SEVERITY_MAPPING, + ALERT_RULE_THREAT, + ALERT_RULE_THROTTLE, + ALERT_RULE_TIMELINE_ID, + ALERT_RULE_TIMELINE_TITLE, + ALERT_RULE_INDICES, + ALERT_RULE_TIMESTAMP_OVERRIDE, +} from '../../../../../../common/field_maps/field_names'; + +export const createAlert = ( + someUuid: string = '1', + data?: object +): EventsForEnrichment => ({ + _id: someUuid, + _source: { + someKey: 'someValue', + + [TIMESTAMP]: '2020-04-20T21:27:45+0000', + [SPACE_IDS]: ['default'], + [EVENT_KIND]: 'signal', + [ALERT_RULE_CONSUMER]: '', + [ALERT_RULE_TIMESTAMP_OVERRIDE]: undefined, + [ALERT_ANCESTORS]: [ + { + id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + rule: undefined, + }, + ], + [ALERT_BUILDING_BLOCK_TYPE]: undefined, + [ALERT_ORIGINAL_TIME]: undefined, + [ALERT_STATUS]: ALERT_STATUS_ACTIVE, + [ALERT_WORKFLOW_STATUS]: 'open', + [ALERT_DEPTH]: 1, + [ALERT_REASON]: 'reasonable reason', + [ALERT_SEVERITY]: 'high', + [ALERT_RISK_SCORE]: 50, + [ALERT_RULE_ACTIONS]: [], + [ALERT_RULE_AUTHOR]: ['Elastic'], + [ALERT_RULE_CATEGORY]: 'Custom Query Rule', + [ALERT_RULE_CREATED_AT]: '2020-03-27T22:55:59.577Z', + [ALERT_RULE_CREATED_BY]: 'sample user', + [ALERT_RULE_DESCRIPTION]: 'Descriptive description', + [ALERT_RULE_ENABLED]: true, + [ALERT_RULE_EXCEPTIONS_LIST]: [], + [ALERT_RULE_EXECUTION_UUID]: '97e8f53a-4971-4935-bb54-9b8f86930cc7', + [ALERT_RULE_FALSE_POSITIVES]: [], + [ALERT_RULE_FROM]: 'now-6m', + [ALERT_RULE_IMMUTABLE]: false, + [ALERT_RULE_INTERVAL]: '5m', + [ALERT_RULE_INDICES]: ['auditbeat-*'], + [ALERT_RULE_LICENSE]: 'Elastic License', + [ALERT_RULE_MAX_SIGNALS]: 10000, + [ALERT_RULE_NAME]: 'rule-name', + [ALERT_RULE_NAMESPACE_FIELD]: undefined, + [ALERT_RULE_NOTE]: undefined, + [ALERT_RULE_PRODUCER]: 'siem', + [ALERT_RULE_REFERENCES]: ['http://example.com', 'https://example.com'], + [ALERT_RULE_RISK_SCORE_MAPPING]: [], + [ALERT_RULE_RULE_ID]: 'rule-1', + [ALERT_RULE_RULE_NAME_OVERRIDE]: undefined, + [ALERT_RULE_TYPE_ID]: 'siem.queryRule', + [ALERT_RULE_SEVERITY_MAPPING]: [], + [ALERT_RULE_TAGS]: ['some fake tag 1', 'some fake tag 2'], + [ALERT_RULE_THREAT]: [ + { + framework: 'MITRE ATT&CK', + tactic: { + id: 'TA0000', + name: 'test tactic', + reference: 'https://attack.mitre.org/tactics/TA0000/', + }, + technique: [ + { + id: 'T0000', + name: 'test technique', + reference: 'https://attack.mitre.org/techniques/T0000/', + subtechnique: [ + { + id: 'T0000.000', + name: 'test subtechnique', + reference: 'https://attack.mitre.org/techniques/T0000/000/', + }, + ], + }, + ], + }, + ], + [ALERT_RULE_THROTTLE]: 'no_actions', + [ALERT_RULE_TIMELINE_ID]: 'some-timeline-id', + [ALERT_RULE_TIMELINE_TITLE]: 'some-timeline-title', + [ALERT_RULE_TO]: 'now', + [ALERT_RULE_TYPE]: 'query', + [ALERT_RULE_UPDATED_AT]: '2020-03-27T22:55:59.577Z', + [ALERT_RULE_UPDATED_BY]: 'sample user', + [ALERT_RULE_UUID]: '2e051244-b3c6-4779-a241-e1b4f0beceb9', + [ALERT_RULE_VERSION]: 1, + [ALERT_UUID]: someUuid, + 'kibana.alert.rule.risk_score': 50, + 'kibana.alert.rule.severity': 'high', + 'kibana.alert.rule.building_block_type': undefined, + [ALERT_RULE_PARAMETERS]: { + description: 'Descriptive description', + meta: { someMeta: 'someField' }, + timeline_id: 'some-timeline-id', + timeline_title: 'some-timeline-title', + risk_score: 50, + severity: 'high', + note: 'Noteworthy notes', + license: 'Elastic License', + author: ['Elastic'], + false_positives: [], + from: 'now-6m', + rule_id: 'rule-1', + max_signals: 10000, + risk_score_mapping: [], + severity_mapping: [], + to: 'now', + references: ['http://example.com', 'https://example.com'], + version: 1, + immutable: false, + namespace: 'default', + output_index: '', + building_block_type: undefined, + exceptions_list: [], + rule_name_override: undefined, + timestamp_override: undefined, + }, + ...data, + }, +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/enrichments/create_single_field_match_enrichment.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/enrichments/create_single_field_match_enrichment.test.ts new file mode 100644 index 0000000000000..d7e29d51a0025 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/enrichments/create_single_field_match_enrichment.test.ts @@ -0,0 +1,206 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { RuleExecutorServicesMock } from '@kbn/alerting-plugin/server/mocks'; +import { alertsMock } from '@kbn/alerting-plugin/server/mocks'; +import { createSingleFieldMatchEnrichment } from './create_single_field_match_enrichment'; +import { searchEnrichments } from './search_enrichments'; +import { ruleExecutionLogMock } from '../../rule_monitoring/mocks'; +import { createAlert } from './__mocks__/alerts'; +import type { EnrichmentFunction } from './types'; + +jest.mock('./search_enrichments', () => ({ + searchEnrichments: jest.fn(), +})); +const mockSearchEnrichments = searchEnrichments as jest.Mock; + +describe('createSingleFieldMatchEnrichment', () => { + let ruleExecutionLogger: ReturnType; + let alertServices: RuleExecutorServicesMock; + + beforeEach(() => { + ruleExecutionLogger = ruleExecutionLogMock.forExecutors.create(); + alertServices = alertsMock.createRuleExecutorServices(); + }); + + afterEach(() => { + mockSearchEnrichments.mockClear(); + }); + + it('return empty object if there no enrichments for events', async () => { + mockSearchEnrichments.mockImplementation(() => []); + + const enrichmentResult = await createSingleFieldMatchEnrichment({ + name: 'host', + index: ['host-enrichment'], + events: [createAlert('1')], + logger: ruleExecutionLogger, + services: alertServices, + mappingField: { + eventField: 'host.name', + enrichmentField: 'host.name', + }, + enrichmentResponseFields: ['host.name'], + createEnrichmentFunction: () => (a) => a, + }); + + expect(enrichmentResult).toEqual({}); + }); + + it('return map with events to enrich', async () => { + mockSearchEnrichments.mockImplementation(() => [ + { + fields: { + 'host.name': ['host name 1'], + }, + }, + { + fields: { + 'host.name': ['host name 3'], + }, + }, + ]); + + const enrichFunction: EnrichmentFunction = (a) => a; + + const enrichmentResult = await createSingleFieldMatchEnrichment({ + name: 'host', + index: ['host-enrichment'], + events: [ + createAlert('1', { host: { name: 'host name 1' } }), + createAlert('2', { host: { name: 'host name 2' } }), + createAlert('3'), + ], + logger: ruleExecutionLogger, + services: alertServices, + enrichmentResponseFields: ['host.name'], + mappingField: { + eventField: 'host.name', + enrichmentField: 'host.name', + }, + createEnrichmentFunction: () => enrichFunction, + }); + + expect(enrichmentResult).toEqual({ 1: [enrichFunction] }); + }); + + it('make request only with unique values', async () => { + mockSearchEnrichments.mockImplementation(() => [ + { + fields: { + 'host.name': ['host name 1'], + }, + }, + { + fields: { + 'host.name': ['host name 3'], + }, + }, + ]); + + const enrichFunction: EnrichmentFunction = (a) => a; + + await createSingleFieldMatchEnrichment({ + name: 'host', + index: ['host-enrichment'], + events: [ + createAlert('1', { host: { name: 'host name 1' } }), + createAlert('2', { host: { name: 'host name 1' } }), + createAlert('3', { host: { name: 'host name 1' } }), + ], + enrichmentResponseFields: ['host.name'], + logger: ruleExecutionLogger, + services: alertServices, + mappingField: { + eventField: 'host.name', + enrichmentField: 'host.name', + }, + createEnrichmentFunction: () => enrichFunction, + }); + + expect( + mockSearchEnrichments.mock.calls[mockSearchEnrichments.mock.calls.length - 1][0].query.query + .bool.should + ).toEqual([{ match: { 'host.name': { minimum_should_match: 1, query: 'host name 1' } } }]); + }); + + it('return empty object if there some exception happen', async () => { + mockSearchEnrichments.mockImplementation(() => { + throw new Error('1'); + }); + + const enrichFunction: EnrichmentFunction = (a) => a; + + const enrichmentResult = await createSingleFieldMatchEnrichment({ + name: 'host', + index: ['host-enrichment'], + events: [createAlert('1', { host: { name: 'host name 1' } })], + logger: ruleExecutionLogger, + services: alertServices, + enrichmentResponseFields: ['host.name'], + mappingField: { + eventField: 'host.name', + enrichmentField: 'host.name', + }, + createEnrichmentFunction: () => enrichFunction, + }); + + expect(enrichmentResult).toEqual({}); + }); + + it('skip request to search ernichments if there no fields', async () => { + mockSearchEnrichments.mockImplementation(() => {}); + + const enrichFunction: EnrichmentFunction = (a) => a; + + await createSingleFieldMatchEnrichment({ + name: 'host', + index: ['host-enrichment'], + events: [createAlert('1')], + logger: ruleExecutionLogger, + services: alertServices, + enrichmentResponseFields: ['host.name'], + mappingField: { + eventField: 'host.name', + enrichmentField: 'host.name', + }, + createEnrichmentFunction: () => enrichFunction, + }); + + expect(mockSearchEnrichments).not.toHaveBeenCalled(); + }); + + it('make several request to enrichment index, if there more than 1000 values to search', async () => { + mockSearchEnrichments.mockImplementation(() => + [...Array(3000).keys()].map((item) => ({ + fields: { 'host.name': [`host name ${item}`] }, + })) + ); + + const events = [...Array(3000).keys()].map((item) => + createAlert(item.toString(), { host: { name: `host name ${item}` } }) + ); + const enrichFunction: EnrichmentFunction = (a) => a; + + const enrichmentResult = await createSingleFieldMatchEnrichment({ + name: 'host', + index: ['host-enrichment'], + events, + logger: ruleExecutionLogger, + enrichmentResponseFields: ['host.name'], + services: alertServices, + mappingField: { + eventField: 'host.name', + enrichmentField: 'host.name', + }, + createEnrichmentFunction: () => enrichFunction, + }); + + expect(mockSearchEnrichments.mock.calls.length).toEqual(3); + expect(Object.keys(enrichmentResult).length).toEqual(3000); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/enrichments/create_single_field_match_enrichment.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/enrichments/create_single_field_match_enrichment.ts new file mode 100644 index 0000000000000..5971c7685a443 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/enrichments/create_single_field_match_enrichment.ts @@ -0,0 +1,95 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { flatten, chunk } from 'lodash'; +import { searchEnrichments } from './search_enrichments'; +import { makeSingleFieldMatchQuery } from './utils/requests'; +import { getEventValue, getFieldValue } from './utils/events'; +import type { CreateFieldsMatchEnrichment, EventsMapByEnrichments, EnrichmentType } from './types'; + +const MAX_CLAUSES = 1000; + +export const createSingleFieldMatchEnrichment: CreateFieldsMatchEnrichment = async ({ + index, + services, + logger, + events, + mappingField, + createEnrichmentFunction, + name, + enrichmentResponseFields, +}) => { + try { + logger.debug(`Enrichment ${name}: started`); + + const eventsWithField = events.filter((event) => getEventValue(event, mappingField.eventField)); + const eventsMapByFieldValue = eventsWithField.reduce((acc, event) => { + const eventFieldValue = getEventValue(event, mappingField.eventField); + + if (!eventFieldValue) return {}; + + acc[eventFieldValue] ??= []; + acc[eventFieldValue].push(event); + + return acc; + }, {} as { [key: string]: typeof events }); + + const uniqueEventsValuesToSearchBy = Object.keys(eventsMapByFieldValue); + const chunksUniqueEventsValuesToSearchBy = chunk(uniqueEventsValuesToSearchBy, MAX_CLAUSES); + + const getAllEnrichment = chunksUniqueEventsValuesToSearchBy + .map((enrichmentValuesChunk) => + makeSingleFieldMatchQuery({ + values: enrichmentValuesChunk, + searchByField: mappingField.enrichmentField, + }) + ) + .filter((query) => query.query?.bool?.should?.length > 0) + .map((query) => + searchEnrichments({ + index, + services, + logger, + query, + fields: enrichmentResponseFields, + }) + ); + + const enrichmentsResults = (await Promise.allSettled(getAllEnrichment)) + .filter((result) => result.status === 'fulfilled') + .map((result) => (result as PromiseFulfilledResult)?.value); + + const enrichments = flatten(enrichmentsResults); + + if (enrichments.length === 0) { + logger.debug(`Enrichment ${name}: no enrichment found`); + return {}; + } + + const eventsMapById = enrichments.reduce((acc, enrichment) => { + const enrichmentValue = getFieldValue(enrichment, mappingField.enrichmentField); + + if (!enrichmentValue) return acc; + + const eventsWithoutEnrchment = eventsMapByFieldValue[enrichmentValue]; + + eventsWithoutEnrchment?.forEach((event) => { + acc[event._id] = [createEnrichmentFunction(enrichment)]; + }); + + return acc; + }, {}); + + logger.debug( + `Enrichment ${name}: return ${Object.keys(eventsMapById).length} events ready to be enriched` + ); + return eventsMapById; + } catch (error) { + logger.error(`Enrichment ${name}: throw error ${error}`); + return {}; + } +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/enrichments/enrichment_by_type/host_risk.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/enrichments/enrichment_by_type/host_risk.ts new file mode 100644 index 0000000000000..c621c83f49ecd --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/enrichments/enrichment_by_type/host_risk.ts @@ -0,0 +1,65 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { set, cloneDeep } from 'lodash'; + +import { getHostRiskIndex } from '../../../../../../common/search_strategy/security_solution/risk_score/common'; +import { RiskScoreFields } from '../../../../../../common/search_strategy/security_solution/risk_score/all'; +import { createSingleFieldMatchEnrichment } from '../create_single_field_match_enrichment'; +import type { CreateRiskEnrichment, GetIsRiskScoreAvailable } from '../types'; +import { getFieldValue } from '../utils/events'; + +export const getIsHostRiskScoreAvailable: GetIsRiskScoreAvailable = async ({ + spaceId, + services, +}) => { + const isHostRiskScoreIndexExist = await services.scopedClusterClient.asCurrentUser.indices.exists( + { + index: getHostRiskIndex(spaceId), + } + ); + + return isHostRiskScoreIndexExist; +}; + +export const createHostRiskEnrichments: CreateRiskEnrichment = async ({ + services, + logger, + events, + spaceId, +}) => { + return createSingleFieldMatchEnrichment({ + name: 'Host Risk', + index: [getHostRiskIndex(spaceId)], + services, + logger, + events, + mappingField: { + eventField: 'host.name', + enrichmentField: RiskScoreFields.hostName, + }, + enrichmentResponseFields: [ + RiskScoreFields.hostName, + RiskScoreFields.hostRisk, + RiskScoreFields.hostRiskScore, + ], + createEnrichmentFunction: (enrichment) => (event) => { + const riskLevel = getFieldValue(enrichment, RiskScoreFields.hostRisk); + const riskScore = getFieldValue(enrichment, RiskScoreFields.hostRiskScore); + if (!riskLevel && !riskScore) { + return event; + } + const newEvent = cloneDeep(event); + if (riskLevel) { + set(newEvent, '_source.host.risk.calculated_level', riskLevel); + } + if (riskScore) { + set(newEvent, '_source.host.risk.calculated_score_norm', riskScore); + } + return newEvent; + }, + }); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/enrichments/enrichment_by_type/user_risk.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/enrichments/enrichment_by_type/user_risk.ts new file mode 100644 index 0000000000000..d6499e9977d54 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/enrichments/enrichment_by_type/user_risk.ts @@ -0,0 +1,64 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { set, cloneDeep } from 'lodash'; +import { getUserRiskIndex } from '../../../../../../common/search_strategy/security_solution/risk_score/common'; +import { RiskScoreFields } from '../../../../../../common/search_strategy/security_solution/risk_score/all'; +import { createSingleFieldMatchEnrichment } from '../create_single_field_match_enrichment'; +import type { CreateRiskEnrichment, GetIsRiskScoreAvailable } from '../types'; +import { getFieldValue } from '../utils/events'; + +export const getIsUserRiskScoreAvailable: GetIsRiskScoreAvailable = async ({ + services, + spaceId, +}) => { + const isUserRiskScoreIndexExist = await services.scopedClusterClient.asCurrentUser.indices.exists( + { + index: getUserRiskIndex(spaceId), + } + ); + + return isUserRiskScoreIndexExist; +}; + +export const createUserRiskEnrichments: CreateRiskEnrichment = async ({ + services, + logger, + events, + spaceId, +}) => { + return createSingleFieldMatchEnrichment({ + name: 'User Risk', + index: [getUserRiskIndex(spaceId)], + services, + logger, + events, + mappingField: { + eventField: 'user.name', + enrichmentField: RiskScoreFields.userName, + }, + enrichmentResponseFields: [ + RiskScoreFields.userName, + RiskScoreFields.userRisk, + RiskScoreFields.userRiskScore, + ], + createEnrichmentFunction: (enrichment) => (event) => { + const riskLevel = getFieldValue(enrichment, RiskScoreFields.userRisk); + const riskScore = getFieldValue(enrichment, RiskScoreFields.userRiskScore); + if (!riskLevel && !riskScore) { + return event; + } + const newEvent = cloneDeep(event); + if (riskLevel) { + set(newEvent, '_source.user.risk.calculated_level', riskLevel); + } + if (riskScore) { + set(newEvent, '_source.user.risk.calculated_score_norm', riskScore); + } + return newEvent; + }, + }); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/enrichments/index.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/enrichments/index.test.ts new file mode 100644 index 0000000000000..4ea17dfbc0092 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/enrichments/index.test.ts @@ -0,0 +1,208 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { RuleExecutorServicesMock } from '@kbn/alerting-plugin/server/mocks'; +import { alertsMock } from '@kbn/alerting-plugin/server/mocks'; +import { enrichEvents } from '.'; +import { searchEnrichments } from './search_enrichments'; +import { ruleExecutionLogMock } from '../../rule_monitoring/mocks'; +import { createAlert } from './__mocks__/alerts'; +import { getIsHostRiskScoreAvailable } from './enrichment_by_type/host_risk'; + +import { getIsUserRiskScoreAvailable } from './enrichment_by_type/user_risk'; + +jest.mock('./search_enrichments', () => ({ + searchEnrichments: jest.fn(), +})); +const mockSearchEnrichments = searchEnrichments as jest.Mock; + +jest.mock('./enrichment_by_type/host_risk', () => ({ + ...jest.requireActual('./enrichment_by_type/host_risk'), + getIsHostRiskScoreAvailable: jest.fn(), +})); +const mockGetIsHostRiskScoreAvailable = getIsHostRiskScoreAvailable as jest.Mock; + +jest.mock('./enrichment_by_type/user_risk', () => ({ + ...jest.requireActual('./enrichment_by_type/user_risk'), + getIsUserRiskScoreAvailable: jest.fn(), +})); +const mockGetIsUserRiskScoreAvailable = getIsUserRiskScoreAvailable as jest.Mock; + +const hostEnrichmentResponse = [ + { + fields: { + 'host.name': ['host name 1'], + 'host.risk.calculated_level': ['Low'], + 'host.risk.calculated_score_norm': [20], + }, + }, + { + fields: { + 'host.name': ['host name 3'], + 'host.risk.calculated_level': ['Critical'], + 'host.risk.calculated_score_norm': [90], + }, + }, +]; + +const userEnrichmentResponse = [ + { + fields: { + 'user.name': ['user name 1'], + 'user.risk.calculated_level': ['Moderate'], + 'user.risk.calculated_score_norm': [50], + }, + }, + { + fields: { + 'user.name': ['user name 2'], + 'user.risk.calculated_level': ['Critical'], + 'user.risk.calculated_score_norm': [90], + }, + }, +]; + +describe('enrichEvents', () => { + let ruleExecutionLogger: ReturnType; + let alertServices: RuleExecutorServicesMock; + const createEntity = (entity: string, name: string) => ({ + [entity]: { name }, + }); + beforeEach(() => { + ruleExecutionLogger = ruleExecutionLogMock.forExecutors.create(); + alertServices = alertsMock.createRuleExecutorServices(); + }); + + it('return the same events, if risk indexes are not available', async () => { + mockSearchEnrichments.mockImplementation(() => []); + mockGetIsUserRiskScoreAvailable.mockImplementation(() => false); + mockGetIsHostRiskScoreAvailable.mockImplementation(() => false); + const events = [ + createAlert('1', createEntity('host', 'host name')), + createAlert('2', createEntity('user', 'user name')), + ]; + const enrichedEvents = await enrichEvents({ + logger: ruleExecutionLogger, + services: alertServices, + events, + spaceId: 'default', + }); + + expect(enrichedEvents).toEqual(events); + }); + + it('return the same events, if there no fields', async () => { + mockSearchEnrichments.mockImplementation(() => []); + mockGetIsUserRiskScoreAvailable.mockImplementation(() => true); + mockGetIsHostRiskScoreAvailable.mockImplementation(() => true); + const events = [createAlert('1'), createAlert('2')]; + const enrichedEvents = await enrichEvents({ + logger: ruleExecutionLogger, + services: alertServices, + events, + spaceId: 'default', + }); + + expect(enrichedEvents).toEqual(events); + }); + + it('return enriched events', async () => { + mockSearchEnrichments + .mockReturnValueOnce(hostEnrichmentResponse) + .mockReturnValueOnce(userEnrichmentResponse); + mockGetIsUserRiskScoreAvailable.mockImplementation(() => true); + mockGetIsHostRiskScoreAvailable.mockImplementation(() => true); + + const enrichedEvents = await enrichEvents({ + logger: ruleExecutionLogger, + services: alertServices, + events: [ + createAlert('1', { + ...createEntity('host', 'host name 1'), + ...createEntity('user', 'user name 1'), + }), + createAlert('2', createEntity('user', 'user name 2')), + ], + spaceId: 'default', + }); + + expect(enrichedEvents).toEqual([ + createAlert('1', { + host: { + name: 'host name 1', + risk: { + calculated_level: 'Low', + calculated_score_norm: 20, + }, + }, + user: { + name: 'user name 1', + risk: { + calculated_level: 'Moderate', + calculated_score_norm: 50, + }, + }, + }), + createAlert('2', { + user: { + name: 'user name 2', + risk: { + calculated_level: 'Critical', + calculated_score_norm: 90, + }, + }, + }), + ]); + }); + + it('if some enrichments failed, another work as expected', async () => { + mockSearchEnrichments + .mockImplementationOnce(() => { + throw new Error('1'); + }) + .mockImplementationOnce(() => userEnrichmentResponse); + mockGetIsUserRiskScoreAvailable.mockImplementation(() => true); + mockGetIsHostRiskScoreAvailable.mockImplementation(() => true); + + const enrichedEvents = await enrichEvents({ + logger: ruleExecutionLogger, + services: alertServices, + events: [ + createAlert('1', { + ...createEntity('host', 'host name 1'), + ...createEntity('user', 'user name 1'), + }), + createAlert('2', createEntity('user', 'user name 2')), + ], + spaceId: 'default', + }); + + expect(enrichedEvents).toEqual([ + createAlert('1', { + host: { + name: 'host name 1', + }, + user: { + name: 'user name 1', + risk: { + calculated_level: 'Moderate', + calculated_score_norm: 50, + }, + }, + }), + createAlert('2', { + user: { + name: 'user name 2', + risk: { + calculated_level: 'Critical', + calculated_score_norm: 90, + }, + }, + }), + ]); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/enrichments/index.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/enrichments/index.ts new file mode 100644 index 0000000000000..ad50f1d6a0bb3 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/enrichments/index.ts @@ -0,0 +1,82 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + createHostRiskEnrichments, + getIsHostRiskScoreAvailable, +} from './enrichment_by_type/host_risk'; + +import { + createUserRiskEnrichments, + getIsUserRiskScoreAvailable, +} from './enrichment_by_type/user_risk'; +import type { + EnrichEventsFunction, + EventsMapByEnrichments, + CreateEnrichEventsFunction, +} from './types'; +import { applyEnrichmentsToEvents } from './utils/transforms'; + +export const enrichEvents: EnrichEventsFunction = async ({ services, logger, events, spaceId }) => { + try { + const enrichments = []; + + logger.debug('Alert enrichments started'); + + const [isHostRiskScoreIndexExist, isUserRiskScoreIndexExist] = await Promise.all([ + getIsHostRiskScoreAvailable({ spaceId, services }), + getIsUserRiskScoreAvailable({ spaceId, services }), + ]); + + if (isHostRiskScoreIndexExist) { + enrichments.push( + createHostRiskEnrichments({ + services, + logger, + events, + spaceId, + }) + ); + } + + if (isUserRiskScoreIndexExist) { + enrichments.push( + createUserRiskEnrichments({ + services, + logger, + events, + spaceId, + }) + ); + } + + const allEnrichmentsResults = await Promise.allSettled(enrichments); + + const allFulfilledEnrichmentsResults = allEnrichmentsResults + .filter((result) => result.status === 'fulfilled') + .map((result) => (result as PromiseFulfilledResult)?.value); + + return applyEnrichmentsToEvents({ + events, + enrichmentsList: allFulfilledEnrichmentsResults, + logger, + }); + } catch (error) { + logger.error(`Enrichments failed ${error}`); + return events; + } +}; + +export const createEnrichEventsFunction: CreateEnrichEventsFunction = + ({ services, logger }) => + (events, { spaceId }: { spaceId: string }) => + enrichEvents({ + events, + services, + logger, + spaceId, + }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/enrichments/search_enrichments.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/enrichments/search_enrichments.ts new file mode 100644 index 0000000000000..75df49b4f6ef0 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/enrichments/search_enrichments.ts @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { getQueryFilter } from '../../../../../common/detection_engine/get_query_filter'; +import type { SearchEnrichments } from './types'; + +export const searchEnrichments: SearchEnrichments = async ({ index, services, query, fields }) => { + try { + const response = await services.scopedClusterClient.asCurrentUser.search({ + index, + body: { + _source: '', + fields, + query: getQueryFilter('', 'kuery', [query], index, []), + }, + track_total_hits: false, + }); + + return response.hits.hits; + } catch (e) { + return []; + } +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/enrichments/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/enrichments/types.ts new file mode 100644 index 0000000000000..ef0864c32cbd1 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/enrichments/types.ts @@ -0,0 +1,125 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import type { Filter } from '@kbn/es-query'; + +import type { + BaseFieldsLatest, + WrappedFieldsLatest, +} from '../../../../../common/detection_engine/schemas/alerts'; +import type { RuleServices } from '../types'; +import type { IRuleExecutionLogForExecutors } from '../../rule_monitoring'; + +export type EnrichmentType = estypes.SearchHit; + +export type EventsForEnrichment = Pick< + WrappedFieldsLatest, + '_id' | '_source' +>; + +export type EnrichmentFunction = ( + e: EventsForEnrichment +) => EventsForEnrichment; + +// +export interface EventsMapByEnrichments { + [id: string]: EnrichmentFunction[]; +} + +export type MergeEnrichments = ( + allEnrichmentsResults: EventsMapByEnrichments[] +) => EventsMapByEnrichments; + +export type ApplyEnrichmentsToEvents = (params: { + events: Array>; + enrichmentsList: EventsMapByEnrichments[]; + logger: IRuleExecutionLogForExecutors; +}) => Array>; + +interface BasedEnrichParamters { + services: RuleServices; + logger: IRuleExecutionLogForExecutors; + events: Array>; +} + +interface SingleMappingField { + eventField: string; + enrichmentField: string; +} + +export type GetEventValue = ( + events: EventsForEnrichment, + path: string +) => string | undefined; + +export type GetFieldValue = (events: EnrichmentType, path: string) => string | undefined; + +export type MakeSingleFieldMatchQuery = (params: { + values: string[]; + searchByField: string; +}) => Filter; + +export type SearchEnrichments = (params: { + index: string[]; + services: RuleServices; + logger: IRuleExecutionLogForExecutors; + query: Filter; + fields: string[]; +}) => Promise; + +export type GetIsRiskScoreAvailable = (params: { + spaceId: string; + services: RuleServices; +}) => Promise; + +export type CreateRiskEnrichment = ( + params: BasedEnrichParamters & { + spaceId: string; + } +) => Promise; + +export type CreateFieldsMatchEnrichment = ( + params: BasedEnrichParamters & { + name: string; + index: string[]; + mappingField: SingleMappingField; + enrichmentResponseFields: string[]; + createEnrichmentFunction: (enrichmentDoc: EnrichmentType) => EnrichmentFunction; + } +) => Promise; + +export type EnrichEventsFunction = ( + params: BasedEnrichParamters & { + spaceId: string; + } +) => Promise>>; + +export type CreateEnrichEventsFunction = (params: { + services: RuleServices; + logger: IRuleExecutionLogForExecutors; +}) => EnrichEvents; + +export type EnrichEvents = ( + alerts: Array>, + params: { spaceId: string } +) => Promise>>; + +interface Risk { + calculated_level: string; + calculated_score_norm: string; +} +export interface RiskEnrichmentFields { + host?: { + name?: string; + risk?: Risk; + }; + user?: { + name?: string; + risk?: Risk; + }; +} diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/enrichments/utils/events.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/enrichments/utils/events.test.ts new file mode 100644 index 0000000000000..e195a38647d46 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/enrichments/utils/events.test.ts @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { getEventValue } from './events'; +import { createAlert } from '../__mocks__/alerts'; + +describe('getEventValue', () => { + it('return value if field present in event in object notation', () => { + expect( + getEventValue( + createAlert('1', { + host: { + name: 'host name 1', + }, + }), + 'host.name' + ) + ).toEqual('host name 1'); + }); + + it('return value if field present in event in string notation', () => { + expect( + getEventValue( + createAlert('1', { + 'host.name': 'host name 2', + }), + 'host.name' + ) + ).toEqual('host name 2'); + }); + + it('return value from object if both notation presents', () => { + expect( + getEventValue( + createAlert('1', { + 'host.name': 'host name 2', + host: { + name: 'host name 1', + }, + }), + 'host.name' + ) + ).toEqual('host name 1'); + }); + + it('return first item if it is array', () => { + expect( + getEventValue( + createAlert('1', { + host: { + name: ['host name 1', 'host name 2'], + }, + }), + 'host.name' + ) + ).toEqual('host name 1'); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/enrichments/utils/events.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/enrichments/utils/events.ts new file mode 100644 index 0000000000000..54eea14c6e94a --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/enrichments/utils/events.ts @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { get } from 'lodash'; +import type { GetEventValue, GetFieldValue } from '../types'; + +export const getEventValue: GetEventValue = (event, path) => { + const value = get(event, `_source.${path}`) || event?._source?.[path]; + + if (Array.isArray(value)) { + return value[0]; + } + + return value; +}; + +export const getFieldValue: GetFieldValue = (event, path) => get(event?.fields, path)?.[0]; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/enrichments/utils/requests.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/enrichments/utils/requests.test.ts new file mode 100644 index 0000000000000..8cae81c8ef3b0 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/enrichments/utils/requests.test.ts @@ -0,0 +1,66 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { makeSingleFieldMatchQuery } from './requests'; + +describe('makeSingleFieldMatchQuery', () => { + it('return empty query if events are empty', () => { + expect( + makeSingleFieldMatchQuery({ values: [], searchByField: 'enrichment.host.name' }) + ).toEqual({ + meta: { + alias: null, + negate: false, + disabled: false, + }, + query: { + bool: { + should: [], + minimum_should_match: 1, + }, + }, + }); + }); + + it('return query to search for enrichments', () => { + expect( + makeSingleFieldMatchQuery({ + values: ['host name 1', 'host name 2'], + searchByField: 'enrichment.host.name', + }) + ).toEqual({ + meta: { + alias: null, + negate: false, + disabled: false, + }, + query: { + bool: { + should: [ + { + match: { + 'enrichment.host.name': { + query: 'host name 1', + minimum_should_match: 1, + }, + }, + }, + { + match: { + 'enrichment.host.name': { + query: 'host name 2', + minimum_should_match: 1, + }, + }, + }, + ], + minimum_should_match: 1, + }, + }, + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/enrichments/utils/requests.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/enrichments/utils/requests.ts new file mode 100644 index 0000000000000..b4567481691b6 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/enrichments/utils/requests.ts @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { MakeSingleFieldMatchQuery } from '../types'; + +export const makeSingleFieldMatchQuery: MakeSingleFieldMatchQuery = ({ values, searchByField }) => { + const shouldClauses = values.map((value) => ({ + match: { + [searchByField]: { + query: value, + minimum_should_match: 1, + }, + }, + })); + + return { + meta: { + alias: null, + negate: false, + disabled: false, + }, + query: { + bool: { + should: shouldClauses, + minimum_should_match: 1, + }, + }, + }; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/enrichments/utils/transforms.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/enrichments/utils/transforms.test.ts new file mode 100644 index 0000000000000..088916ef45598 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/enrichments/utils/transforms.test.ts @@ -0,0 +1,109 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { applyEnrichmentsToEvents, mergeEnrichments } from './transforms'; +import { ruleExecutionLogMock } from '../../../rule_monitoring/mocks'; +import { createAlert } from '../__mocks__/alerts'; +import type { EnrichmentFunction } from '../types'; +import { set } from 'lodash'; + +const createEnrichment = + (field: string, value: string): EnrichmentFunction => + (event) => { + set(event, field, value); + return event; + }; +describe('applyEnrichmentsToEvents', () => { + let ruleExecutionLogger: ReturnType; + + beforeEach(() => { + ruleExecutionLogger = ruleExecutionLogMock.forExecutors.create(); + }); + + it('return events if there no enrichments', () => { + expect( + applyEnrichmentsToEvents({ + events: [createAlert('1')], + enrichmentsList: [], + logger: ruleExecutionLogger, + }) + ).toEqual([createAlert('1')]); + }); + + it('return enriched events', () => { + expect( + applyEnrichmentsToEvents({ + events: [createAlert('1'), createAlert('2'), createAlert('3')], + enrichmentsList: [ + { + 1: [ + createEnrichment('host.risk.calculated_level', 'low'), + createEnrichment('enrichedField', '1'), + ], + 2: [createEnrichment('host.risk.calculated_level', '1')], + }, + { + 1: [ + createEnrichment('host.risk.other_field', '10'), + createEnrichment('enrichedField2', '2'), + ], + }, + ], + logger: ruleExecutionLogger, + }) + ).toEqual([ + { + ...createAlert('1'), + enrichedField: '1', + enrichedField2: '2', + host: { + risk: { + calculated_level: 'low', + other_field: '10', + }, + }, + }, + { + ...createAlert('2'), + host: { + risk: { + calculated_level: '1', + }, + }, + }, + createAlert('3'), + ]); + }); +}); + +describe('mergeEnrichments', () => { + it('return empty object, if enrichments are empty array', () => { + expect(mergeEnrichments([])).toEqual({}); + }); + + it('merge enrichemnts into single map', () => { + const fnA = createEnrichment('', ''); + const fnB = createEnrichment('', ''); + const fnC = createEnrichment('', ''); + expect( + mergeEnrichments([ + { + 1: [fnA, fnB], + 3: [fnC], + }, + { + 1: [fnA, fnC], + 2: [fnC], + }, + ]) + ).toEqual({ + 1: [fnA, fnB, fnA, fnC], + 2: [fnC], + 3: [fnC], + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/enrichments/utils/transforms.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/enrichments/utils/transforms.ts new file mode 100644 index 0000000000000..c1a0e6c2e37b4 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/enrichments/utils/transforms.ts @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { mergeWith, isArray } from 'lodash'; + +import type { ApplyEnrichmentsToEvents, MergeEnrichments } from '../types'; + +function customizer(objValue: T, srcValue: T) { + if (isArray(objValue)) { + return objValue.concat(srcValue); + } +} + +export const mergeEnrichments: MergeEnrichments = (enrichmentsList = []) => { + return enrichmentsList.reduce((acc, val) => mergeWith(acc, val, customizer), {}); +}; + +export const applyEnrichmentsToEvents: ApplyEnrichmentsToEvents = ({ + events, + enrichmentsList, + logger, +}) => { + const mergedEnrichments = mergeEnrichments(enrichmentsList); + logger.debug(`${Object.keys(mergedEnrichments).length} events ready to be enriched`); + const enrichedEvents = events.map((event) => { + const enrichFunctions = mergedEnrichments[event._id]; + + if (!enrichFunctions) return event; + + return enrichFunctions.reduce((acc, enrichFunction) => enrichFunction(acc), event); + }); + + return enrichedEvents; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/eql.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/eql.ts index 1244905f01e10..fa6f6d91f25b4 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/eql.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/eql.ts @@ -16,6 +16,7 @@ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { buildEqlSearchRequest } from '../build_events_query'; import { hasLargeValueItem } from '../../../../../common/detection_engine/utils'; +import { createEnrichEventsFunction } from '../enrichments'; import type { BulkCreate, @@ -115,7 +116,15 @@ export const eqlExecutor = async ({ } if (newSignals?.length) { - const createResult = await bulkCreate(newSignals); + const createResult = await bulkCreate( + newSignals, + undefined, + createEnrichEventsFunction({ + services, + logger: ruleExecutionLogger, + }) + ); + addToSearchAfterReturn({ current: result, next: createResult }); } return result; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.ts index 934cb7094d971..d00060b9bd5c5 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.ts @@ -141,6 +141,7 @@ export const thresholdExecutor = async ({ }); // Build and index new alerts + const createResult = await bulkCreateThresholdSignals({ buckets, completeRule, @@ -153,6 +154,7 @@ export const thresholdExecutor = async ({ signalHistory, bulkCreate, wrapHits, + ruleExecutionLogger, }); addToSearchAfterReturn({ current: result, next: createResult }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts index eb0bc7e740b2c..e57e6603cb067 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts @@ -22,6 +22,7 @@ import { } from './utils'; import type { SearchAfterAndBulkCreateParams, SearchAfterAndBulkCreateReturnType } from './types'; import { withSecuritySpan } from '../../../utils/with_security_span'; +import { createEnrichEventsFunction } from './enrichments'; // search_after through documents and re-index using bulk endpoint. export const searchAfterAndBulkCreate = async ({ @@ -143,7 +144,14 @@ export const searchAfterAndBulkCreate = async ({ const enrichedEvents = await enrichment(limitedEvents); const wrappedDocs = wrapHits(enrichedEvents, buildReasonMessage); - const bulkCreateResult = await bulkCreate(wrappedDocs); + const bulkCreateResult = await bulkCreate( + wrappedDocs, + undefined, + createEnrichEventsFunction({ + services, + logger: ruleExecutionLogger, + }) + ); addToSearchAfterReturn({ current: toReturn, next: bulkCreateResult }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/bulk_create_threshold_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/bulk_create_threshold_signals.ts index a4b7eb9cdcedb..758e69553bb4b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/bulk_create_threshold_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/bulk_create_threshold_signals.ts @@ -20,6 +20,8 @@ import type { ThresholdSignalHistory, BulkCreate, WrapHits } from '../types'; import type { CompleteRule, ThresholdRuleParams } from '../../schemas/rule_schemas'; import type { BaseFieldsLatest } from '../../../../../common/detection_engine/schemas/alerts'; import type { ThresholdBucket } from './types'; +import { createEnrichEventsFunction } from '../enrichments'; +import type { IRuleExecutionLogForExecutors } from '../../rule_monitoring'; interface BulkCreateThresholdSignalsParams { buckets: ThresholdBucket[]; @@ -33,6 +35,7 @@ interface BulkCreateThresholdSignalsParams { signalHistory: ThresholdSignalHistory; bulkCreate: BulkCreate; wrapHits: WrapHits; + ruleExecutionLogger: IRuleExecutionLogForExecutors; } export const getTransformedHits = ( @@ -89,5 +92,12 @@ export const bulkCreateThresholdSignals = async ( ruleParams.ruleId ); - return params.bulkCreate(params.wrapHits(ecsResults, buildReasonMessageForThresholdAlert)); + return params.bulkCreate( + params.wrapHits(ecsResults, buildReasonMessageForThresholdAlert), + undefined, + createEnrichEventsFunction({ + services: params.services, + logger: params.ruleExecutionLogger, + }) + ); }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts index 1e2b6db4b4c81..93ddadf826d73 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts @@ -36,6 +36,7 @@ import type { } from '../../../../common/detection_engine/schemas/alerts'; import type { IRuleExecutionLogForExecutors } from '../rule_monitoring'; import type { FullResponseSchema } from '../../../../common/detection_engine/schemas/request'; +import type { EnrichEvents } from './enrichments/types'; export interface ThresholdResult { terms?: Array<{ @@ -235,7 +236,8 @@ export type SignalsEnrichment = (signals: SignalSourceHit[]) => Promise( docs: Array>, - maxAlerts?: number + maxAlerts?: number, + enrichEvents?: EnrichEvents ) => Promise>; export type SimpleHit = BaseHit<{ '@timestamp'?: string }>; @@ -250,13 +252,18 @@ export type WrapSequences = ( buildReasonMessage: BuildReasonMessage ) => Array>; +export type RuleServices = RuleExecutorServices< + AlertInstanceState, + AlertInstanceContext, + 'default' +>; export interface SearchAfterAndBulkCreateParams { tuple: { to: moment.Moment; from: moment.Moment; maxSignals: number; }; - services: RuleExecutorServices; + services: RuleServices; listClient: ListClient; exceptionsList: ExceptionListItemSchema[]; ruleExecutionLogger: IRuleExecutionLogForExecutors; diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/group1/create_ml.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/group1/create_ml.ts index 8363aa0f26401..c94f73713e6a4 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/group1/create_ml.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/group1/create_ml.ts @@ -243,5 +243,25 @@ export default ({ getService }: FtrProviderContext) => { expect(signalsOpen.hits.hits.length).toBe(0); }); }); + + describe('alerts should be be enriched', () => { + before(async () => { + await esArchiver.load('x-pack/test/functional/es_archives/entity/host_risk'); + }); + + after(async () => { + await esArchiver.unload('x-pack/test/functional/es_archives/entity/host_risk'); + }); + + it('should be enriched with host risk score', async () => { + const createdRule = await createRule(supertest, log, testRule); + const signalsOpen = await getOpenSignals(supertest, log, es, createdRule); + expect(signalsOpen.hits.hits.length).toBe(1); + const fullSignal = signalsOpen.hits.hits[0]._source; + + expect(fullSignal?.host?.risk?.calculated_level).toBe('Low'); + expect(fullSignal?.host?.risk?.calculated_score_norm).toBe(1); + }); + }); }); }; diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/group1/create_new_terms.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/group1/create_new_terms.ts index 0a9073d23015b..0c879602af700 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/group1/create_new_terms.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/group1/create_new_terms.ts @@ -434,5 +434,37 @@ export default ({ getService }: FtrProviderContext) => { .sort(); expect(processPids[0]).eql([1]); }); + + describe('alerts should be be enriched', () => { + before(async () => { + await esArchiver.load('x-pack/test/functional/es_archives/entity/host_risk'); + }); + + after(async () => { + await esArchiver.unload('x-pack/test/functional/es_archives/entity/host_risk'); + }); + + it('should be enriched with host risk score', async () => { + const rule: NewTermsCreateSchema = { + ...getCreateNewTermsRulesSchemaMock('rule-1', true), + new_terms_fields: ['host.name'], + from: '2019-02-19T20:42:00.000Z', + history_window_start: '2019-01-19T20:42:00.000Z', + }; + + const createdRule = await createRule(supertest, log, rule); + + await waitForRuleSuccessOrStatus( + supertest, + log, + createdRule.id, + RuleExecutionStatus.succeeded + ); + + const signalsOpen = await getOpenSignals(supertest, log, es, createdRule); + expect(signalsOpen.hits.hits[0]?._source?.host?.risk?.calculated_level).to.eql('Low'); + expect(signalsOpen.hits.hits[0]?._source?.host?.risk?.calculated_score_norm).to.eql(23); + }); + }); }); }; diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/group1/create_threat_matching.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/group1/create_threat_matching.ts index a472e75582481..8c746f3113c3d 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/group1/create_threat_matching.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/group1/create_threat_matching.ts @@ -1358,6 +1358,63 @@ export default ({ getService }: FtrProviderContext) => { ]); }); }); + + describe('alerts should be be enriched', () => { + before(async () => { + await esArchiver.load('x-pack/test/functional/es_archives/entity/host_risk'); + }); + + after(async () => { + await esArchiver.unload('x-pack/test/functional/es_archives/entity/host_risk'); + }); + + it('should be enriched with host risk score', async () => { + const rule: CreateRulesSchema = { + description: 'Detecting root and admin users', + name: 'Query with a rule id', + severity: 'high', + index: ['auditbeat-*'], + type: 'threat_match', + risk_score: 55, + language: 'kuery', + rule_id: 'rule-1', + from: '1900-01-01T00:00:00.000Z', + query: '*:*', + threat_query: 'source.ip: "188.166.120.93"', // narrow things down with a query to a specific source ip + threat_index: ['auditbeat-*'], // We use auditbeat as both the matching index and the threat list for simplicity + threat_mapping: [ + // We match host.name against host.name + { + entries: [ + { + field: 'host.name', + value: 'host.name', + type: 'mapping', + }, + ], + }, + ], + threat_filters: [], + }; + + const createdRule = await createRule(supertest, log, rule); + await waitForRuleSuccessOrStatus(supertest, log, createdRule.id); + await waitForSignalsToBePresent(supertest, log, 10, [createdRule.id]); + const signalsOpen = await getSignalsByIds(supertest, log, [createdRule.id]); + expect(signalsOpen.hits.hits.length).equal(10); + const fullSource = signalsOpen.hits.hits.find( + (signal) => + (signal._source?.[ALERT_ANCESTORS] as Ancestor[])[0].id === '7yJ-B2kBR346wHgnhlMn' + ); + const fullSignal = fullSource?._source; + if (!fullSignal) { + return expect(fullSignal).to.be.ok(); + } + + expect(fullSignal?.host?.risk?.calculated_level).to.eql('Critical'); + expect(fullSignal?.host?.risk?.calculated_score_norm).to.eql(70); + }); + }); }); }); }; diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/group1/generating_signals.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/group1/generating_signals.ts index 9ff764ac7bb40..b71cd23063550 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/group1/generating_signals.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/group1/generating_signals.ts @@ -797,6 +797,34 @@ export default ({ getService }: FtrProviderContext) => { const signals = await getSignalsByIds(supertest, log, [id]); expect(signals.hits.hits.length).eql(2); }); + + describe('EQL alerts should be be enriched', () => { + before(async () => { + await esArchiver.load('x-pack/test/functional/es_archives/entity/host_risk'); + }); + + after(async () => { + await esArchiver.unload('x-pack/test/functional/es_archives/entity/host_risk'); + }); + + it('should be enriched with host risk score', async () => { + const rule: EqlCreateSchema = { + ...getEqlRuleForSignalTesting(['auditbeat-*']), + query: 'configuration where agent.id=="a1d7b39c-f898-4dbe-a761-efb61939302d"', + }; + const { id } = await createRule(supertest, log, rule); + await waitForRuleSuccessOrStatus(supertest, log, id); + await waitForSignalsToBePresent(supertest, log, 1, [id]); + const signals = await getSignalsByIds(supertest, log, [id]); + expect(signals.hits.hits.length).eql(1); + const fullSignal = signals.hits.hits[0]._source; + if (!fullSignal) { + return expect(fullSignal).to.be.ok(); + } + expect(fullSignal?.host?.risk?.calculated_level).to.eql('Critical'); + expect(fullSignal?.host?.risk?.calculated_score_norm).to.eql(96); + }); + }); }); describe('Threshold Rules', () => { @@ -1117,6 +1145,124 @@ export default ({ getService }: FtrProviderContext) => { } }); }); + + describe('Threshold alerts should be be enriched', () => { + before(async () => { + await esArchiver.load('x-pack/test/functional/es_archives/entity/host_risk'); + }); + + after(async () => { + await esArchiver.unload('x-pack/test/functional/es_archives/entity/host_risk'); + }); + + it('should be enriched with host risk score', async () => { + const rule: ThresholdCreateSchema = { + ...getThresholdRuleForSignalTesting(['auditbeat-*']), + threshold: { + field: 'host.name', + value: 100, + }, + }; + const { id } = await createRule(supertest, log, rule); + await waitForRuleSuccessOrStatus(supertest, log, id); + await waitForSignalsToBePresent(supertest, log, 2, [id]); + const signalsOpen = await getSignalsByIds(supertest, log, [id]); + + expect(signalsOpen.hits.hits[0]?._source?.host?.risk?.calculated_level).to.eql('Low'); + expect(signalsOpen.hits.hits[0]?._source?.host?.risk?.calculated_score_norm).to.eql(20); + expect(signalsOpen.hits.hits[1]?._source?.host?.risk?.calculated_level).to.eql( + 'Critical' + ); + expect(signalsOpen.hits.hits[1]?._source?.host?.risk?.calculated_score_norm).to.eql(96); + }); + }); + }); + + describe('Enrich alerts: query rule', () => { + describe('without index avalable', () => { + it('should do not have risk score fields', async () => { + const rule: QueryCreateSchema = { + ...getRuleForSignalTesting(['auditbeat-*']), + query: `_id:${ID}`, + }; + const { id } = await createRule(supertest, log, rule); + await waitForRuleSuccessOrStatus(supertest, log, id); + await waitForSignalsToBePresent(supertest, log, 1, [id]); + const signalsOpen = await getSignalsByIds(supertest, log, [id]); + expect(signalsOpen.hits.hits[0]?._source?.host?.risk).to.eql(undefined); + expect(signalsOpen.hits.hits[0]?._source?.user?.risk).to.eql(undefined); + }); + }); + + describe('with host risk score', () => { + before(async () => { + await esArchiver.load('x-pack/test/functional/es_archives/auditbeat/hosts'); + await esArchiver.load('x-pack/test/functional/es_archives/entity/host_risk'); + }); + + after(async () => { + await esArchiver.unload('x-pack/test/functional/es_archives/auditbeat/hosts'); + await esArchiver.unload('x-pack/test/functional/es_archives/entity/host_risk'); + }); + + it('should host have risk score field and do not have user risk score', async () => { + const rule: QueryCreateSchema = { + ...getRuleForSignalTesting(['auditbeat-*']), + query: `_id:${ID} or _id:GBbXBmkBR346wHgn5_eR or _id:x10zJ2oE9v5HJNSHhyxi`, + }; + const { id } = await createRule(supertest, log, rule); + await waitForRuleSuccessOrStatus(supertest, log, id); + await waitForSignalsToBePresent(supertest, log, 1, [id]); + const signalsOpen = await getSignalsByIds(supertest, log, [id]); + + const alerts = signalsOpen.hits.hits ?? []; + const firstAlert = alerts.find( + (alert) => alert?._source?.host?.name === 'suricata-zeek-sensor-toronto' + ); + const secondAlert = alerts.find( + (alert) => alert?._source?.host?.name === 'suricata-sensor-london' + ); + const thirdAlert = alerts.find((alert) => alert?._source?.host?.name === 'IE11WIN8_1'); + + expect(firstAlert?._source?.host?.risk?.calculated_level).to.eql('Critical'); + expect(firstAlert?._source?.host?.risk?.calculated_score_norm).to.eql(96); + expect(firstAlert?._source?.user?.risk).to.eql(undefined); + expect(secondAlert?._source?.host?.risk?.calculated_level).to.eql('Low'); + expect(secondAlert?._source?.host?.risk?.calculated_score_norm).to.eql(20); + expect(thirdAlert?._source?.host?.risk).to.eql(undefined); + }); + }); + + describe('with host and risk score and user risk score', () => { + before(async () => { + await esArchiver.load('x-pack/test/functional/es_archives/auditbeat/hosts'); + await esArchiver.load('x-pack/test/functional/es_archives/entity/host_risk'); + await esArchiver.load('x-pack/test/functional/es_archives/entity/user_risk'); + }); + + after(async () => { + await esArchiver.unload('x-pack/test/functional/es_archives/auditbeat/hosts'); + await esArchiver.unload('x-pack/test/functional/es_archives/entity/host_risk'); + await esArchiver.unload('x-pack/test/functional/es_archives/entity/user_risk'); + }); + + it('should have host and user risk score fields', async () => { + const rule: QueryCreateSchema = { + ...getRuleForSignalTesting(['auditbeat-*']), + query: `_id:${ID}`, + }; + const { id } = await createRule(supertest, log, rule); + await waitForRuleSuccessOrStatus(supertest, log, id); + await waitForSignalsToBePresent(supertest, log, 1, [id]); + const signalsOpen = await getSignalsByIds(supertest, log, [id]); + expect(signalsOpen.hits.hits[0]?._source?.host?.risk?.calculated_level).to.eql( + 'Critical' + ); + expect(signalsOpen.hits.hits[0]?._source?.host?.risk?.calculated_score_norm).to.eql(96); + expect(signalsOpen.hits.hits[0]?._source?.user?.risk?.calculated_level).to.eql('Low'); + expect(signalsOpen.hits.hits[0]?._source?.user?.risk?.calculated_score_norm).to.eql(11); + }); + }); }); }); diff --git a/x-pack/test/detection_engine_api_integration/utils/get_signals_by_ids.ts b/x-pack/test/detection_engine_api_integration/utils/get_signals_by_ids.ts index b534925b60118..faea391b18602 100644 --- a/x-pack/test/detection_engine_api_integration/utils/get_signals_by_ids.ts +++ b/x-pack/test/detection_engine_api_integration/utils/get_signals_by_ids.ts @@ -9,6 +9,7 @@ import { SearchResponse } from '@elastic/elasticsearch/lib/api/typesWithBodyKey' import type { ToolingLog } from '@kbn/tooling-log'; import type SuperTest from 'supertest'; import type { DetectionAlert } from '@kbn/security-solution-plugin/common/detection_engine/schemas/alerts'; +import type { RiskEnrichmentFields } from '@kbn/security-solution-plugin/server/lib/detection_engine/signals/enrichments/types'; import { DETECTION_ENGINE_QUERY_SIGNALS_URL } from '@kbn/security-solution-plugin/common/constants'; import { countDownTest } from './count_down_test'; @@ -25,8 +26,8 @@ export const getSignalsByIds = async ( log: ToolingLog, ids: string[], size?: number -): Promise> => { - const signalsOpen = await countDownTest>( +): Promise> => { + const signalsOpen = await countDownTest>( async () => { const response = await supertest .post(DETECTION_ENGINE_QUERY_SIGNALS_URL) diff --git a/x-pack/test/functional/es_archives/entity/host_risk/data.json b/x-pack/test/functional/es_archives/entity/host_risk/data.json new file mode 100644 index 0000000000000..c6e609cbcaf7b --- /dev/null +++ b/x-pack/test/functional/es_archives/entity/host_risk/data.json @@ -0,0 +1,100 @@ +{ + "type": "doc", + "value": { + "index": "ml_host_risk_score_latest_default", + "id": "1", + "source": { + "host": { + "name": "suricata-zeek-sensor-toronto", + "risk": { + "calculated_score_norm": 96, + "calculated_level": "Critical" + } + }, + "ingest_timestamp": "2022-08-15T16:32:16.142561766Z", + "@timestamp": "2022-08-12T14:45:36.171Z" + }, + "type": "_doc" + } +} + +{ + "type": "doc", + "value": { + "id": "2", + "index": "ml_host_risk_score_latest_default", + "source": { + "host": { + "name": "suricata-sensor-london", + "risk": { + "calculated_score_norm": 20, + "calculated_level": "Low" + } + }, + "ingest_timestamp": "2022-08-15T16:32:16.142561766Z", + "@timestamp": "2022-08-12T14:45:36.171Z" + }, + "type": "_doc" + } +} + +{ + "type": "doc", + "value": { + "id": "3", + "index": "ml_host_risk_score_latest_default", + "source": { + "host": { + "name": "zeek-newyork-sha-aa8df15", + "risk": { + "calculated_score_norm": 23, + "calculated_level": "Low" + } + }, + "ingest_timestamp": "2022-08-15T16:32:16.142561766Z", + "@timestamp": "2022-08-12T14:45:36.171Z" + }, + "type": "_doc" + } +} + +{ + "type": "doc", + "value": { + "id": "4", + "index": "ml_host_risk_score_latest_default", + "source": { + "host": { + "name": "zeek-sensor-amsterdam", + "risk": { + "calculated_score_norm": 70, + "calculated_level": "Critical" + } + }, + "ingest_timestamp": "2022-08-15T16:32:16.142561766Z", + "@timestamp": "2022-08-12T14:45:36.171Z" + }, + "type": "_doc" + } +} + +{ + "type": "doc", + "value": { + "id": "5", + "index": "ml_host_risk_score_latest_default", + "source": { + "host": { + "name": "mothra", + "risk": { + "calculated_score_norm": 1, + "calculated_level": "Low" + } + }, + "ingest_timestamp": "2022-08-15T16:32:16.142561766Z", + "@timestamp": "2022-08-12T14:45:36.171Z" + }, + "type": "_doc" + } +} + diff --git a/x-pack/test/functional/es_archives/entity/host_risk/mappings.json b/x-pack/test/functional/es_archives/entity/host_risk/mappings.json new file mode 100644 index 0000000000000..623ac12b2229a --- /dev/null +++ b/x-pack/test/functional/es_archives/entity/host_risk/mappings.json @@ -0,0 +1,35 @@ +{ + + "type": "index", + "value": { + "index": "ml_host_risk_score_latest_default", + "mappings": { + "properties": { + "host": { + "properties": { + "name": { + "type": "keyword" + }, + "risk": { + "properties": { + "calculated_level": { + "type": "keyword" + }, + "calculated_score_norm": { + "type": "float" + } + } + } + } + } + } + }, + "settings": { + "index": { + "auto_expand_replicas": "0-1", + "number_of_replicas": "0", + "number_of_shards": "1" + } + } + } +} diff --git a/x-pack/test/functional/es_archives/entity/user_risk/data.json b/x-pack/test/functional/es_archives/entity/user_risk/data.json new file mode 100644 index 0000000000000..39b403deddc69 --- /dev/null +++ b/x-pack/test/functional/es_archives/entity/user_risk/data.json @@ -0,0 +1,39 @@ +{ + "type": "doc", + "value": { + "index": "ml_user_risk_score_latest_default", + "id": "1", + "source": { + "user": { + "name": "root", + "risk": { + "calculated_score_norm": 11, + "calculated_level": "Low" + } + }, + "ingest_timestamp": "2022-08-15T16:32:16.142561766Z", + "@timestamp": "2022-08-12T14:45:36.171Z" + }, + "type": "_doc" + } +} + +{ + "type": "doc", + "value": { + "id": "2", + "index": "ml_user_risk_score_latest_default", + "source": { + "host": { + "name": "User name 1", + "risk": { + "calculated_score_norm": 20, + "calculated_level": "Low" + } + }, + "ingest_timestamp": "2022-08-15T16:32:16.142561766Z", + "@timestamp": "2022-08-12T14:45:36.171Z" + }, + "type": "_doc" + } +} diff --git a/x-pack/test/functional/es_archives/entity/user_risk/mappings.json b/x-pack/test/functional/es_archives/entity/user_risk/mappings.json new file mode 100644 index 0000000000000..22518c9c455fc --- /dev/null +++ b/x-pack/test/functional/es_archives/entity/user_risk/mappings.json @@ -0,0 +1,35 @@ +{ + + "type": "index", + "value": { + "index": "ml_user_risk_score_latest_default", + "mappings": { + "properties": { + "user": { + "properties": { + "name": { + "type": "keyword" + }, + "risk": { + "properties": { + "calculated_level": { + "type": "keyword" + }, + "calculated_score_norm": { + "type": "float" + } + } + } + } + } + } + }, + "settings": { + "index": { + "auto_expand_replicas": "0-1", + "number_of_replicas": "0", + "number_of_shards": "1" + } + } + } +} diff --git a/x-pack/test/security_solution_cypress/es_archives/risky_hosts_updated/data.json b/x-pack/test/security_solution_cypress/es_archives/risky_hosts_updated/data.json new file mode 100644 index 0000000000000..89ffd442b4786 --- /dev/null +++ b/x-pack/test/security_solution_cypress/es_archives/risky_hosts_updated/data.json @@ -0,0 +1,24 @@ +{ + "type": "doc", + "value": { + "id": "a4cf452c1e0375c3d4412cb550bd1783358468a3b3b777da4829d72c7d6fb74f", + "index": "ml_host_risk_score_latest_default", + "source": { + "@timestamp": "2021-03-10T14:51:05.766Z", + "host": { + "name": "siem-kibana", + "risk": { + "calculated_level": "Critical", + "calculated_score_norm": 90, + "rule_risks": [ + { + "rule_name": "Unusual Linux Username", + "rule_risk": 42 + } + ] + } + }, + "ingest_timestamp": "2021-03-09T18:02:08.319296053Z" + } + } +} diff --git a/x-pack/test/security_solution_cypress/es_archives/risky_hosts_updated/mappings.json b/x-pack/test/security_solution_cypress/es_archives/risky_hosts_updated/mappings.json new file mode 100644 index 0000000000000..3e1b52cb22f5e --- /dev/null +++ b/x-pack/test/security_solution_cypress/es_archives/risky_hosts_updated/mappings.json @@ -0,0 +1,102 @@ +{ + "type": "index", + "value": { + "index": "ml_host_risk_score_latest_default", + "mappings": { + "properties": { + "@timestamp": { + "type": "date" + }, + "host": { + "properties": { + "name": { + "type": "keyword" + }, + "risk": { + "properties": { + "calculated_level": { + "type": "keyword" + }, + "calculated_score_norm": { + "type": "long" + } + } + } + } + }, + "ingest_timestamp": { + "type": "date" + } + } + }, + "settings": { + "index": { + "lifecycle": { + "name": "ml_host_risk_score_latest_default", + "rollover_alias": "ml_host_risk_score_latest_default" + }, + "mapping": { + "total_fields": { + "limit": "10000" + } + }, + "max_docvalue_fields_search": "200", + "number_of_replicas": "1", + "number_of_shards": "1", + "refresh_interval": "5s" + } + } + } +} + + +{ + "type": "index", + "value": { + "index": "ml_host_risk_score_default", + "mappings": { + "properties": { + "@timestamp": { + "type": "date" + }, + "host": { + "properties": { + "name": { + "type": "keyword" + }, + "risk": { + "properties": { + "calculated_level": { + "type": "keyword" + }, + "calculated_score_norm": { + "type": "long" + } + } + } + } + }, + "ingest_timestamp": { + "type": "date" + } + } + }, + "settings": { + "index": { + "lifecycle": { + "name": "ml_host_risk_score_default", + "rollover_alias": "ml_host_risk_score_default" + }, + "mapping": { + "total_fields": { + "limit": "10000" + } + }, + "max_docvalue_fields_search": "200", + "number_of_replicas": "1", + "number_of_shards": "1", + "refresh_interval": "5s" + } + } + } +} diff --git a/x-pack/test/security_solution_cypress/es_archives/risky_users/data.json b/x-pack/test/security_solution_cypress/es_archives/risky_users/data.json index 5cb0404a9d0d5..dc182e631df99 100644 --- a/x-pack/test/security_solution_cypress/es_archives/risky_users/data.json +++ b/x-pack/test/security_solution_cypress/es_archives/risky_users/data.json @@ -171,4 +171,29 @@ "ingest_timestamp": "2021-03-09T18:02:08.319296053Z" } } +} + +{ + "type": "doc", + "value": { + "id": "a4cf452c1e0375c3d4412cb550bd1783358468b3123314829d72c7df6fb74", + "index": "ml_user_risk_score_latest_default", + "source": { + "@timestamp": "2021-03-10T14:51:05.766Z", + "user": { + "name": "test", + "risk": { + "calculated_score_norm": 21, + "calculated_level": "Low", + "rule_risks": [ + { + "rule_name": "Unusual Linux Username", + "rule_risk": 42 + } + ] + } + }, + "ingest_timestamp": "2021-03-09T18:02:08.319296053Z" + } + } } \ No newline at end of file