Alert Enrichments at ingest time (#139478)

* Add threat indicator enrichemnt * [CI] Auto-commit changed files from 'node scripts/eslint --no-cache --fix' * [CI] Auto-commit changed files from 'node scripts/eslint --no-cache --fix' * some temp logs * Add 5 enrichments * some temp logs * Add listClient * Add value list functionalityu * [CI] Auto-commit changed files from 'node scripts/precommit_hook.js --ref HEAD~1..HEAD --fix' * 10 enrichment * [CI] Auto-commit changed files from 'node scripts/eslint --no-cache --fix' * 1 enrichment - 10 idnex * Usage of enrichments * Add host and user risk score enrichments * remove unused loger * check that risk exist on enrichment * typos * sucesfully proceed if some enrichment fails * [CI] Auto-commit changed files from 'node scripts/precommit_hook.js --ref HEAD~1..HEAD --fix' * remove throwing error * Add try catch for enrichAlerts * Add enrichmenst for other rule types * [CI] Auto-commit changed files from 'node scripts/precommit_hook.js --ref HEAD~1..HEAD --fix' * [CI] Auto-commit changed files from 'node scripts/eslint --no-cache --fix' * Add some logging * remove user risk score enablament * [CI] Auto-commit changed files from 'node scripts/precommit_hook.js --ref HEAD~1..HEAD --fix' * [CI] Auto-commit changed files from 'node scripts/eslint --no-cache --fix' * Fix for threshold * chaneg log message * Fix wrong build * [CI] Auto-commit changed files from 'node scripts/eslint --no-cache --fix' * Remove wrong merges * add ecs mapping to alerts * Add default columns * Add score_norm to enrichment * Add host risks UI * fix some types * [CI] Auto-commit changed files from 'node scripts/eslint --no-cache --fix' * Unit tests and refactoring * Add integrations tests * Remove unused tpes * Add some unit tests * Do chunk if there more than 1000 values * Add cypress tests * Change search enrichments to field * Fix translations * Fix types * [CI] Auto-commit changed files from 'node scripts/eslint --no-cache --fix' * remove types * Remove types * Change index name back * Fix types * fix user risk score cypress data * Fix entity tests * Add license check for show the columns * fix typo * Fix tests issue * Add try catch for enrichment * [CI] Auto-commit changed files from 'node scripts/eslint --no-cache --fix' * Remove it after rebase * Add user rusk score support for flyout * Fix typos * Try to fix test * skip enrichment test for now Co-authored-by: kibanamachine <[email protected]>
elastic · Sep 19, 2022 · 6cebf21 · 6cebf21
1 parent c6177ae
commit 6cebf21
Show file tree

Hide file tree

Showing 53 changed files with 2,457 additions and 199 deletions.
diff --git a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts
@@ -22,7 +22,7 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper
           ...options,
           services: {
             ...options.services,
-            alertWithPersistence: async (alerts, refresh, maxAlerts = undefined) => {
+            alertWithPersistence: async (alerts, refresh, maxAlerts = undefined, enrichAlerts) => {
               const numAlerts = alerts.length;
               logger.debug(`Found ${numAlerts} alerts.`);
 
@@ -85,13 +85,25 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper
                   return { createdAlerts: [], errors: {}, alertsWereTruncated: false };
                 }
 
+                let enrichedAlerts = filteredAlerts;
+
+                if (enrichAlerts) {
+                  try {
+                    enrichedAlerts = await enrichAlerts(filteredAlerts, {
+                      spaceId: options.spaceId,
+                    });
+                  } catch (e) {
+                    logger.debug('Enrichemnts failed');
+                  }
+                }
+
                 let alertsWereTruncated = false;
-                if (maxAlerts && filteredAlerts.length > maxAlerts) {
-                  filteredAlerts.length = maxAlerts;
+                if (maxAlerts && enrichedAlerts.length > maxAlerts) {
+                  enrichedAlerts.length = maxAlerts;
                   alertsWereTruncated = true;
                 }
 
-                const augmentedAlerts = filteredAlerts.map((alert) => {
+                const augmentedAlerts = enrichedAlerts.map((alert) => {
                   return {
                     ...alert,
                     _source: {

diff --git a/x-pack/plugins/rule_registry/server/utils/persistence_types.ts b/x-pack/plugins/rule_registry/server/utils/persistence_types.ts
@@ -25,7 +25,19 @@ export type PersistenceAlertService = <T>(
     _source: T;
   }>,
   refresh: boolean | 'wait_for',
-  maxAlerts?: number
+  maxAlerts?: number,
+  enrichAlerts?: (
+    alerts: Array<{
+      _id: string;
+      _source: T;
+    }>,
+    params: { spaceId: string }
+  ) => Promise<
+    Array<{
+      _id: string;
+      _source: T;
+    }>
+  >
 ) => Promise<PersistenceAlertServiceResult<T>>;
 
 export interface PersistenceAlertServiceResult<T> {

diff --git a/x-pack/plugins/security_solution/cypress/e2e/dashboards/entity_analytics.cy.ts b/x-pack/plugins/security_solution/cypress/e2e/dashboards/entity_analytics.cy.ts
@@ -83,7 +83,7 @@ describe('Entity Analytics Dashboard', () => {
     });
 
     it('renders donut chart', () => {
-      cy.get(USERS_DONUT_CHART).should('include.text', '6Total');
+      cy.get(USERS_DONUT_CHART).should('include.text', '7Total');
     });
 
     it('renders table', () => {
@@ -94,8 +94,8 @@ describe('Entity Analytics Dashboard', () => {
     it('filters by risk classification', () => {
       openRiskTableFilterAndSelectTheLowOption();
 
-      cy.get(USERS_DONUT_CHART).should('include.text', '1Total');
-      cy.get(USERS_TABLE_ROWS).should('have.length', 1);
+      cy.get(USERS_DONUT_CHART).should('include.text', '2Total');
+      cy.get(USERS_TABLE_ROWS).should('have.length', 2);
     });
   });
 

diff --git a/x-pack/plugins/security_solution/cypress/e2e/detection_alerts/enrichments.cy.ts b/x-pack/plugins/security_solution/cypress/e2e/detection_alerts/enrichments.cy.ts
@@ -0,0 +1,79 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { getNewRule } from '../../objects/rule';
+import {
+  NUMBER_OF_ALERTS,
+  HOST_RISK_HEADER_COLIMN,
+  USER_RISK_HEADER_COLIMN,
+  HOST_RISK_COLUMN,
+  USER_RISK_COLUMN,
+  ACTION_COLUMN,
+} from '../../screens/alerts';
+import { ENRICHED_DATA_ROW } from '../../screens/alerts_details';
+import { esArchiverLoad, esArchiverUnload } from '../../tasks/es_archiver';
+
+import { createCustomRuleEnabled } from '../../tasks/api_calls/rules';
+import { cleanKibana, deleteAlertsAndRules } from '../../tasks/common';
+import { waitForAlertsToPopulate } from '../../tasks/create_new_rule';
+import {
+  expandFirstAlert,
+  scrollAlertTableColumnIntoView,
+  closeAlertFlyout,
+} from '../../tasks/alerts';
+
+import { login, visit } from '../../tasks/login';
+
+import { ALERTS_URL } from '../../urls/navigation';
+
+describe.skip('Enrichment', () => {
+  before(() => {
+    cleanKibana();
+    esArchiverLoad('risky_hosts');
+    esArchiverLoad('risky_users');
+    login();
+  });
+
+  after(() => {
+    esArchiverUnload('risky_hosts');
+    esArchiverUnload('risky_users');
+    esArchiverUnload('risky_hosts_updated');
+  });
+  describe('Custom query rule', () => {
+    beforeEach(() => {
+      deleteAlertsAndRules();
+      createCustomRuleEnabled(getNewRule(), 'rule1');
+      visit(ALERTS_URL);
+      waitForAlertsToPopulate();
+    });
+
+    it('Should has enrichment fields', function () {
+      cy.get(NUMBER_OF_ALERTS)
+        .invoke('text')
+        .should('match', /^[1-9].+$/); // Any number of alerts
+      cy.get(HOST_RISK_HEADER_COLIMN).contains('host.risk.calculated_level');
+      cy.get(USER_RISK_HEADER_COLIMN).contains('user.risk.calculated_level');
+      scrollAlertTableColumnIntoView(HOST_RISK_COLUMN);
+      cy.get(HOST_RISK_COLUMN).contains('Low');
+      scrollAlertTableColumnIntoView(USER_RISK_COLUMN);
+      cy.get(USER_RISK_COLUMN).contains('Low');
+      scrollAlertTableColumnIntoView(ACTION_COLUMN);
+      expandFirstAlert();
+      cy.get(ENRICHED_DATA_ROW).contains('Low');
+      cy.get(ENRICHED_DATA_ROW).contains('Current host risk classification');
+      cy.get(ENRICHED_DATA_ROW).contains('Critical').should('not.exist');
+      cy.get(ENRICHED_DATA_ROW).contains('Original host risk classification').should('not.exist');
+
+      closeAlertFlyout();
+      esArchiverUnload('risky_hosts');
+      esArchiverLoad('risky_hosts_updated');
+      expandFirstAlert();
+      cy.get(ENRICHED_DATA_ROW).contains('Critical');
+      cy.get(ENRICHED_DATA_ROW).contains('Original host risk classification');
+    });
+  });
+});
diff --git a/x-pack/plugins/security_solution/cypress/screens/alerts.ts b/x-pack/plugins/security_solution/cypress/screens/alerts.ts
@@ -44,6 +44,8 @@ export const EMPTY_ALERT_TABLE = '[data-test-subj="tGridEmptyState"]';
 
 export const EXPAND_ALERT_BTN = '[data-test-subj="expand-event"]';
 
+export const CLOSE_FLYOUT = '[data-test-subj="euiFlyoutCloseButton"]';
+
 export const GROUP_BY_TOP_INPUT = '[data-test-subj="groupByTop"] [data-test-subj="comboBoxInput"]';
 
 export const HOST_NAME = '[data-test-subj^=formatted-field][data-test-subj$=host\\.name]';
@@ -92,3 +94,15 @@ export const USER_NAME = '[data-test-subj^=formatted-field][data-test-subj$=user
 export const ATTACH_ALERT_TO_CASE_BUTTON = '[data-test-subj="add-to-existing-case-action"]';
 
 export const USER_COLUMN = '[data-gridcell-column-id="user.name"]';
+
+export const HOST_RISK_HEADER_COLIMN =
+  '[data-test-subj="dataGridHeaderCell-host.risk.calculated_level"]';
+
+export const HOST_RISK_COLUMN = '[data-gridcell-column-id="host.risk.calculated_level"]';
+
+export const USER_RISK_HEADER_COLIMN =
+  '[data-test-subj="dataGridHeaderCell-user.risk.calculated_level"]';
+
+export const USER_RISK_COLUMN = '[data-gridcell-column-id="user.risk.calculated_level"]';
+
+export const ACTION_COLUMN = '[data-gridcell-column-id="default-timeline-control-column"]';
diff --git a/x-pack/plugins/security_solution/cypress/screens/alerts_details.ts b/x-pack/plugins/security_solution/cypress/screens/alerts_details.ts
@@ -81,3 +81,5 @@ export const INSIGHTS_INVESTIGATE_IN_TIMELINE_BUTTON = `${INSIGHTS_RELATED_ALERT
 export const INSIGHTS_RELATED_ALERTS_BY_ANCESTRY = `[data-test-subj='related-alerts-by-ancestry']`;
 
 export const INSIGHTS_INVESTIGATE_ANCESTRY_ALERTS_IN_TIMELINE_BUTTON = `[data-test-subj='investigate-ancestry-in-timeline']`;
+
+export const ENRICHED_DATA_ROW = `[data-test-subj='EnrichedDataRow']`;
diff --git a/x-pack/plugins/security_solution/cypress/tasks/alerts.ts b/x-pack/plugins/security_solution/cypress/tasks/alerts.ts
@@ -24,6 +24,7 @@ import {
   SELECT_TABLE,
   TAKE_ACTION_POPOVER_BTN,
   TIMELINE_CONTEXT_MENU_BTN,
+  CLOSE_FLYOUT,
 } from '../screens/alerts';
 import { REFRESH_BUTTON } from '../screens/security_header';
 import {
@@ -82,6 +83,8 @@ export const expandFirstAlert = () => {
     .pipe(($el) => $el.trigger('click'));
 };
 
+export const closeAlertFlyout = () => cy.get(CLOSE_FLYOUT).click();
+
 export const viewThreatIntelTab = () => cy.get(THREAT_INTEL_TAB).click();
 
 export const setEnrichmentDates = (from?: string, to?: string) => {

diff --git a/...ecurity_solution/public/common/components/event_details/cti_details/host_risk_summary.tsx b/...ecurity_solution/public/common/components/event_details/cti_details/host_risk_summary.tsx
@@ -11,56 +11,66 @@ import { FormattedMessage } from '@kbn/i18n-react';
 import * as i18n from './translations';
 import { EnrichedDataRow, ThreatSummaryPanelHeader } from './threat_summary_view';
 import { RiskScore } from '../../severity/common';
+import type { RiskSeverity } from '../../../../../common/search_strategy';
 import type { HostRisk } from '../../../../risk_score/containers';
 import { getEmptyValue } from '../../empty_value';
 import { RISKY_HOSTS_DOC_LINK } from '../../../../../common/constants';
 
 const HostRiskSummaryComponent: React.FC<{
   hostRisk: HostRisk;
-}> = ({ hostRisk }) => (
-  <>
-    <EuiPanel hasBorder paddingSize="s" grow={false}>
-      <ThreatSummaryPanelHeader
-        title={i18n.HOST_RISK_DATA_TITLE}
-        toolTipContent={
-          <FormattedMessage
-            id="xpack.securitySolution.alertDetails.overview.hostDataTooltipContent"
-            defaultMessage="Risk classification is displayed only when available for a host. Ensure {hostRiskScoreDocumentationLink} is enabled within your environment."
-            values={{
-              hostRiskScoreDocumentationLink: (
-                <EuiLink href={RISKY_HOSTS_DOC_LINK} target="_blank">
-                  <FormattedMessage
-                    id="xpack.securitySolution.alertDetails.overview.hostRiskScoreLink"
-                    defaultMessage="Host Risk Score"
-                  />
-                </EuiLink>
-              ),
-            }}
-          />
-        }
-      />
+  originalHostRisk?: RiskSeverity | undefined;
+}> = ({ hostRisk, originalHostRisk }) => {
+  const currentHostRiskScore = hostRisk?.result?.[0]?.host?.risk?.calculated_level;
+  return (
+    <>
+      <EuiPanel hasBorder paddingSize="s" grow={false}>
+        <ThreatSummaryPanelHeader
+          title={i18n.HOST_RISK_DATA_TITLE}
+          toolTipContent={
+            <FormattedMessage
+              id="xpack.securitySolution.alertDetails.overview.hostDataTooltipContent"
+              defaultMessage="Risk classification is displayed only when available for a host. Ensure {hostRiskScoreDocumentationLink} is enabled within your environment."
+              values={{
+                hostRiskScoreDocumentationLink: (
+                  <EuiLink href={RISKY_HOSTS_DOC_LINK} target="_blank">
+                    <FormattedMessage
+                      id="xpack.securitySolution.alertDetails.overview.hostRiskScoreLink"
+                      defaultMessage="Host Risk Score"
+                    />
+                  </EuiLink>
+                ),
+              }}
+            />
+          }
+        />
 
-      {hostRisk.loading && <EuiLoadingSpinner data-test-subj="loading" />}
+        {hostRisk.loading && <EuiLoadingSpinner data-test-subj="loading" />}
 
-      {!hostRisk.loading && (
-        <>
-          <EnrichedDataRow
-            field={i18n.HOST_RISK_CLASSIFICATION}
-            value={
-              hostRisk.result && hostRisk.result.length > 0 ? (
-                <RiskScore
-                  severity={hostRisk.result[0].host.risk.calculated_level}
-                  hideBackgroundColor
-                />
-              ) : (
-                getEmptyValue()
-              )
-            }
-          />
-        </>
-      )}
-    </EuiPanel>
-  </>
-);
+        {!hostRisk.loading && (
+          <>
+            <EnrichedDataRow
+              field={i18n.CURRENT_HOST_RISK_CLASSIFICATION}
+              value={
+                currentHostRiskScore ? (
+                  <RiskScore severity={currentHostRiskScore} hideBackgroundColor />
+                ) : (
+                  getEmptyValue()
+                )
+              }
+            />
 
+            {originalHostRisk && currentHostRiskScore !== originalHostRisk && (
+              <>
+                <EnrichedDataRow
+                  field={i18n.ORIGINAL_HOST_RISK_CLASSIFICATION}
+                  value={<RiskScore severity={originalHostRisk} hideBackgroundColor />}
+                />
+              </>
+            )}
+          </>
+        )}
+      </EuiPanel>
+    </>
+  );
+};
 export const HostRiskSummary = React.memo(HostRiskSummaryComponent);
diff --git a/...urity_solution/public/common/components/event_details/cti_details/threat_summary_view.tsx b/...urity_solution/public/common/components/event_details/cti_details/threat_summary_view.tsx
@@ -24,6 +24,7 @@ import type { CtiEnrichment } from '../../../../../common/search_strategy/securi
 import type {
   BrowserFields,
   TimelineEventsDetailsItem,
+  RiskSeverity,
 } from '../../../../../common/search_strategy';
 import { HostRiskSummary } from './host_risk_summary';
 import { UserRiskSummary } from './user_risk_summary';
@@ -141,6 +142,14 @@ const ThreatSummaryViewComponent: React.FC<{
   isDraggable,
   isReadOnly,
 }) => {
+  const originalHostRisk = data?.find(
+    (eventDetail) => eventDetail?.field === 'host.risk.calculated_level'
+  )?.values?.[0] as RiskSeverity | undefined;
+
+  const originalUserRisk = data?.find(
+    (eventDetail) => eventDetail?.field === 'user.risk.calculated_level'
+  )?.values?.[0] as RiskSeverity | undefined;
+
   return (
     <>
       <EuiHorizontalRule />
@@ -152,11 +161,11 @@ const ThreatSummaryViewComponent: React.FC<{
 
       <EuiFlexGroup direction="column" gutterSize="m" style={{ flexGrow: 0 }}>
         <EuiFlexItem grow={false}>
-          <HostRiskSummary hostRisk={hostRisk} />
+          <HostRiskSummary hostRisk={hostRisk} originalHostRisk={originalHostRisk} />
         </EuiFlexItem>
 
         <EuiFlexItem grow={false}>
-          <UserRiskSummary userRisk={userRisk} />
+          <UserRiskSummary userRisk={userRisk} originalUserRisk={originalUserRisk} />
         </EuiFlexItem>
 
         <EnrichmentSummary

diff --git a/...gins/security_solution/public/common/components/event_details/cti_details/translations.ts b/...gins/security_solution/public/common/components/event_details/cti_details/translations.ts
@@ -99,10 +99,17 @@ export const ENRICHED_DATA = i18n.translate(
   }
 );
 
-export const HOST_RISK_CLASSIFICATION = i18n.translate(
+export const CURRENT_HOST_RISK_CLASSIFICATION = i18n.translate(
   'xpack.securitySolution.alertDetails.overview.hostRiskClassification',
   {
-    defaultMessage: 'Host risk classification',
+    defaultMessage: 'Current host risk classification',
+  }
+);
+
+export const ORIGINAL_HOST_RISK_CLASSIFICATION = i18n.translate(
+  'xpack.securitySolution.alertDetails.overview.originalHostRiskClassification',
+  {
+    defaultMessage: 'Original host risk classification',
   }
 );
 
@@ -120,9 +127,16 @@ export const USER_RISK_DATA_TITLE = i18n.translate(
   }
 );
 
-export const USER_RISK_CLASSIFICATION = i18n.translate(
-  'xpack.securitySolution.alertDetails.overview.userRiskClassification',
+export const ORIGINAL_USER_RISK_CLASSIFICATION = i18n.translate(
+  'xpack.securitySolution.alertDetails.overview.originalUserRiskClassification',
+  {
+    defaultMessage: 'Original risk classification',
+  }
+);
+
+export const CURRENT_USER_RISK_CLASSIFICATION = i18n.translate(
+  'xpack.securitySolution.alertDetails.overview.currentUserRiskClassification',
   {
-    defaultMessage: 'User risk classification',
+    defaultMessage: 'Current user risk classification',
   }
 );
Original file line number	Diff line number	Diff line change
Expand Up		@@ -81,3 +81,5 @@ export const INSIGHTS_INVESTIGATE_IN_TIMELINE_BUTTON = `${INSIGHTS_RELATED_ALERT
		export const INSIGHTS_RELATED_ALERTS_BY_ANCESTRY = `[data-test-subj='related-alerts-by-ancestry']`;

		export const INSIGHTS_INVESTIGATE_ANCESTRY_ALERTS_IN_TIMELINE_BUTTON = `[data-test-subj='investigate-ancestry-in-timeline']`;

		export const ENRICHED_DATA_ROW = `[data-test-subj='EnrichedDataRow']`;