From 89c103d3e7eb0ded45b3815b5dff1609001c8984 Mon Sep 17 00:00:00 2001 From: Dima Arnautov Date: Wed, 16 Mar 2022 17:21:18 +0100 Subject: [PATCH 1/3] [ML] Utilize ML memory stats endpoint for the memory overview chart (#127751) * replace memory_overview_service with ES API call * fix attributes type * marker for ml_max_in_bytes * update text * update jest tests --- .../plugins/ml/common/types/trained_models.ts | 4 +- .../nodes_overview/memory_preview_chart.tsx | 20 +- .../nodes_overview/nodes_list.tsx | 1 + .../ml/server/lib/ml_client/ml_client.ts | 3 + .../models_provider.test.ts | 201 +++++++++--------- .../data_frame_analytics/models_provider.ts | 86 ++++---- .../ml/server/models/memory_overview/index.ts | 8 - .../memory_overview_service.ts | 85 -------- .../ml/server/routes/trained_models.ts | 8 +- 9 files changed, 174 insertions(+), 242 deletions(-) delete mode 100644 x-pack/plugins/ml/server/models/memory_overview/index.ts delete mode 100644 x-pack/plugins/ml/server/models/memory_overview/memory_overview_service.ts diff --git a/x-pack/plugins/ml/common/types/trained_models.ts b/x-pack/plugins/ml/common/types/trained_models.ts index d6eda37f99465..3de759d9dbc87 100644 --- a/x-pack/plugins/ml/common/types/trained_models.ts +++ b/x-pack/plugins/ml/common/types/trained_models.ts @@ -205,6 +205,8 @@ export interface NodeDeploymentStatsResponse { total: number; jvm: number; }; + /** Max amount of memory available for ML */ + ml_max_in_bytes: number; /** Open anomaly detection jobs + hardcoded overhead */ anomaly_detection: { /** Total size in bytes */ @@ -226,6 +228,6 @@ export interface NodeDeploymentStatsResponse { } export interface NodesOverviewResponse { - count: number; + _nodes: { total: number; failed: number; successful: number }; nodes: NodeDeploymentStatsResponse[]; } diff --git a/x-pack/plugins/ml/public/application/trained_models/nodes_overview/memory_preview_chart.tsx b/x-pack/plugins/ml/public/application/trained_models/nodes_overview/memory_preview_chart.tsx index 8a8f7a1dd7e81..e355bc26f7fa7 100644 --- a/x-pack/plugins/ml/public/application/trained_models/nodes_overview/memory_preview_chart.tsx +++ b/x-pack/plugins/ml/public/application/trained_models/nodes_overview/memory_preview_chart.tsx @@ -15,8 +15,10 @@ import { ScaleType, SeriesColorAccessor, Settings, + LineAnnotation, + AnnotationDomainType, } from '@elastic/charts'; -import { euiPaletteGray } from '@elastic/eui'; +import { EuiIcon, euiPaletteGray } from '@elastic/eui'; import { NodeDeploymentStatsResponse } from '../../../../common/types/trained_models'; import { useFieldFormatter } from '../../contexts/kibana/use_field_formatter'; import { useCurrentEuiTheme } from '../../components/color_range_legend'; @@ -126,6 +128,22 @@ export const MemoryPreviewChart: FC = ({ memoryOverview tickFormat={(d: number) => bytesFormatter(d)} /> + } + markerPosition={Position.Top} + /> + = ({ compactView = false }) => { name: i18n.translate('xpack.ml.trainedModels.nodesList.nodeNameHeader', { defaultMessage: 'Name', }), + width: '200px', sortable: true, truncateText: true, 'data-test-subj': 'mlNodesTableColumnName', diff --git a/x-pack/plugins/ml/server/lib/ml_client/ml_client.ts b/x-pack/plugins/ml/server/lib/ml_client/ml_client.ts index 11d5de7c45ed8..4ff555445f2c8 100644 --- a/x-pack/plugins/ml/server/lib/ml_client/ml_client.ts +++ b/x-pack/plugins/ml/server/lib/ml_client/ml_client.ts @@ -636,6 +636,9 @@ export function getMlClient( async validateDetector(...p: Parameters) { return mlClient.validateDetector(...p); }, + async getMemoryStats(...p: Parameters) { + return mlClient.getMemoryStats(...p); + }, ...searchProvider(client, jobSavedObjectService), } as MlClient; diff --git a/x-pack/plugins/ml/server/models/data_frame_analytics/models_provider.test.ts b/x-pack/plugins/ml/server/models/data_frame_analytics/models_provider.test.ts index 22e803540f0d2..99919794001c9 100644 --- a/x-pack/plugins/ml/server/models/data_frame_analytics/models_provider.test.ts +++ b/x-pack/plugins/ml/server/models/data_frame_analytics/models_provider.test.ts @@ -5,129 +5,129 @@ * 2.0. */ -import { ModelService, modelsProvider } from './models_provider'; +import { MemoryStatsResponse, ModelService, modelsProvider } from './models_provider'; import { IScopedClusterClient } from 'kibana/server'; import { MlClient } from '../../lib/ml_client'; import mockResponse from './__mocks__/mock_deployment_response.json'; -import { MemoryOverviewService } from '../memory_overview/memory_overview_service'; describe('Model service', () => { const client = { - asInternalUser: { - nodes: { - stats: jest.fn(() => { - return Promise.resolve({ - _nodes: { - total: 3, - successful: 3, - failed: 0, - }, - cluster_name: 'test_cluster', - nodes: { - '3qIoLFnbSi-DwVrYioUCdw': { - timestamp: 1635167166946, - name: 'node3', - transport_address: '10.10.10.2:9353', - host: '10.10.10.2', - ip: '10.10.10.2:9353', - roles: ['data', 'ingest', 'master', 'ml', 'transform'], - attributes: { - 'ml.machine_memory': '15599742976', - 'xpack.installed': 'true', - 'ml.max_jvm_size': '1073741824', - }, - os: { - mem: { - total_in_bytes: 15599742976, - adjusted_total_in_bytes: 15599742976, - free_in_bytes: 376324096, - used_in_bytes: 15223418880, - free_percent: 2, - used_percent: 98, - }, - }, - }, - 'DpCy7SOBQla3pu0Dq-tnYw': { - timestamp: 1635167166946, - name: 'node2', - transport_address: '10.10.10.2:9352', - host: '10.10.10.2', - ip: '10.10.10.2:9352', - roles: ['data', 'master', 'ml', 'transform'], - attributes: { - 'ml.machine_memory': '15599742976', - 'xpack.installed': 'true', - 'ml.max_jvm_size': '1073741824', - }, - os: { - timestamp: 1635167166959, - mem: { - total_in_bytes: 15599742976, - adjusted_total_in_bytes: 15599742976, - free_in_bytes: 376324096, - used_in_bytes: 15223418880, - free_percent: 2, - used_percent: 98, - }, - }, - }, - 'pt7s6lKHQJaP4QHKtU-Q0Q': { - timestamp: 1635167166945, - name: 'node1', - transport_address: '10.10.10.2:9351', - host: '10.10.10.2', - ip: '10.10.10.2:9351', - roles: ['data', 'master', 'ml'], - attributes: { - 'ml.machine_memory': '15599742976', - 'xpack.installed': 'true', - 'ml.max_jvm_size': '1073741824', - }, - os: { - timestamp: 1635167166959, - mem: { - total_in_bytes: 15599742976, - adjusted_total_in_bytes: 15599742976, - free_in_bytes: 376324096, - used_in_bytes: 15223418880, - free_percent: 2, - used_percent: 98, - }, - }, - }, - }, - }); - }), - }, - }, + asInternalUser: {}, } as unknown as jest.Mocked; + const mlClient = { getTrainedModelsStats: jest.fn(() => { return Promise.resolve({ trained_model_stats: mockResponse, }); }), - } as unknown as jest.Mocked; - const memoryOverviewService = { - getDFAMemoryOverview: jest.fn(() => { - return Promise.resolve([{ job_id: '', node_id: '', model_size: 32165465 }]); - }), - getAnomalyDetectionMemoryOverview: jest.fn(() => { - return Promise.resolve([{ job_id: '', node_id: '', model_size: 32165465 }]); + getMemoryStats: jest.fn(() => { + return Promise.resolve({ + _nodes: { + total: 3, + successful: 3, + failed: 0, + }, + cluster_name: 'test_cluster', + nodes: { + '3qIoLFnbSi-DwVrYioUCdw': { + name: 'node3', + transport_address: '10.10.10.2:9353', + roles: ['data', 'ingest', 'master', 'ml', 'transform'], + attributes: { + 'ml.machine_memory': '15599742976', + 'ml.max_jvm_size': '1073741824', + }, + jvm: { + heap_max_in_bytes: 1073741824, + java_inference_in_bytes: 0, + java_inference_max_in_bytes: 0, + }, + mem: { + adjusted_total_in_bytes: 15599742976, + total_in_bytes: 15599742976, + ml: { + data_frame_analytics_in_bytes: 0, + native_code_overhead_in_bytes: 0, + max_in_bytes: 1073741824, + anomaly_detectors_in_bytes: 0, + native_inference_in_bytes: 1555161790, + }, + }, + ephemeral_id: '3qIoLFnbSi-DwVrYioUCdw', + }, + 'DpCy7SOBQla3pu0Dq-tnYw': { + name: 'node2', + transport_address: '10.10.10.2:9352', + roles: ['data', 'master', 'ml', 'transform'], + attributes: { + 'ml.machine_memory': '15599742976', + 'ml.max_jvm_size': '1073741824', + }, + jvm: { + heap_max_in_bytes: 1073741824, + java_inference_in_bytes: 0, + java_inference_max_in_bytes: 0, + }, + mem: { + adjusted_total_in_bytes: 15599742976, + total_in_bytes: 15599742976, + ml: { + data_frame_analytics_in_bytes: 0, + native_code_overhead_in_bytes: 0, + max_in_bytes: 1073741824, + anomaly_detectors_in_bytes: 0, + native_inference_in_bytes: 1555161790, + }, + }, + ephemeral_id: '3qIoLFnbSi-DwVrYioUCdw', + }, + 'pt7s6lKHQJaP4QHKtU-Q0Q': { + name: 'node1', + transport_address: '10.10.10.2:9351', + roles: ['data', 'master', 'ml'], + attributes: { + 'ml.machine_memory': '15599742976', + 'ml.max_jvm_size': '1073741824', + }, + jvm: { + heap_max_in_bytes: 1073741824, + java_inference_in_bytes: 0, + java_inference_max_in_bytes: 0, + }, + mem: { + adjusted_total_in_bytes: 15599742976, + total_in_bytes: 15599742976, + ml: { + data_frame_analytics_in_bytes: 0, + native_code_overhead_in_bytes: 0, + max_in_bytes: 1073741824, + anomaly_detectors_in_bytes: 0, + native_inference_in_bytes: 1555161790, + }, + }, + ephemeral_id: '3qIoLFnbSi-DwVrYioUCdw', + }, + }, + } as MemoryStatsResponse); }), - } as unknown as jest.Mocked; + } as unknown as jest.Mocked; let service: ModelService; beforeEach(() => { - service = modelsProvider(client, mlClient, memoryOverviewService); + service = modelsProvider(client, mlClient); }); afterEach(() => {}); it('extract nodes list correctly', async () => { expect(await service.getNodesOverview()).toEqual({ - count: 3, + _nodes: { + failed: 0, + successful: 3, + total: 3, + }, nodes: [ { name: 'node3', @@ -219,6 +219,7 @@ describe('Model service', () => { }, id: '3qIoLFnbSi-DwVrYioUCdw', memory_overview: { + ml_max_in_bytes: 1073741824, anomaly_detection: { total: 0, }, @@ -339,6 +340,7 @@ describe('Model service', () => { }, id: 'DpCy7SOBQla3pu0Dq-tnYw', memory_overview: { + ml_max_in_bytes: 1073741824, anomaly_detection: { total: 0, }, @@ -462,6 +464,7 @@ describe('Model service', () => { }, id: 'pt7s6lKHQJaP4QHKtU-Q0Q', memory_overview: { + ml_max_in_bytes: 1073741824, anomaly_detection: { total: 0, }, diff --git a/x-pack/plugins/ml/server/models/data_frame_analytics/models_provider.ts b/x-pack/plugins/ml/server/models/data_frame_analytics/models_provider.ts index 714d02704ff92..db0eee9ec757c 100644 --- a/x-pack/plugins/ml/server/models/data_frame_analytics/models_provider.ts +++ b/x-pack/plugins/ml/server/models/data_frame_analytics/models_provider.ts @@ -6,7 +6,7 @@ */ import type { IScopedClusterClient } from 'kibana/server'; -import { sumBy, pick } from 'lodash'; +import { pick } from 'lodash'; import { MlTrainedModelStats, NodesInfoNodeInfo, @@ -17,20 +17,15 @@ import type { NodesOverviewResponse, } from '../../../common/types/trained_models'; import type { MlClient } from '../../lib/ml_client'; -import { - MemoryOverviewService, - NATIVE_EXECUTABLE_CODE_OVERHEAD, -} from '../memory_overview/memory_overview_service'; import { TrainedModelDeploymentStatsResponse, TrainedModelModelSizeStats, } from '../../../common/types/trained_models'; import { isDefined } from '../../../common/types/guards'; -import { isPopulatedObject } from '../../../common'; export type ModelService = ReturnType; -const NODE_FIELDS = ['attributes', 'name', 'roles', 'version'] as const; +const NODE_FIELDS = ['attributes', 'name', 'roles'] as const; export type RequiredNodeFields = Pick; @@ -40,11 +35,38 @@ interface TrainedModelStatsResponse extends MlTrainedModelStats { model_size_stats?: TrainedModelModelSizeStats; } -export function modelsProvider( - client: IScopedClusterClient, - mlClient: MlClient, - memoryOverviewService?: MemoryOverviewService -) { +export interface MemoryStatsResponse { + _nodes: { total: number; failed: number; successful: number }; + cluster_name: string; + nodes: Record< + string, + { + jvm: { + heap_max_in_bytes: number; + java_inference_in_bytes: number; + java_inference_max_in_bytes: number; + }; + mem: { + adjusted_total_in_bytes: number; + total_in_bytes: number; + ml: { + data_frame_analytics_in_bytes: number; + native_code_overhead_in_bytes: number; + max_in_bytes: number; + anomaly_detectors_in_bytes: number; + native_inference_in_bytes: number; + }; + }; + transport_address: string; + roles: string[]; + name: string; + attributes: Record<`${'ml.'}${string}`, string>; + ephemeral_id: string; + } + >; +} + +export function modelsProvider(client: IScopedClusterClient, mlClient: MlClient) { return { /** * Retrieves the map of model ids and aliases with associated pipelines. @@ -89,30 +111,19 @@ export function modelsProvider( * Provides the ML nodes overview with allocated models. */ async getNodesOverview(): Promise { - if (!memoryOverviewService) { - throw new Error('Memory overview service is not provided'); - } + const response = (await mlClient.getMemoryStats()) as MemoryStatsResponse; const { trained_model_stats: trainedModelStats } = await mlClient.getTrainedModelsStats({ size: 10000, }); - const { nodes: clusterNodes } = await client.asInternalUser.nodes.stats(); - - const mlNodes = Object.entries(clusterNodes).filter(([, node]) => node.roles?.includes('ml')); - - const adMemoryReport = await memoryOverviewService.getAnomalyDetectionMemoryOverview(); - const dfaMemoryReport = await memoryOverviewService.getDFAMemoryOverview(); + const mlNodes = Object.entries(response.nodes); const nodeDeploymentStatsResponses: NodeDeploymentStatsResponse[] = mlNodes.map( ([nodeId, node]) => { const nodeFields = pick(node, NODE_FIELDS) as RequiredNodeFields; - nodeFields.attributes = isPopulatedObject(nodeFields.attributes) - ? Object.fromEntries( - Object.entries(nodeFields.attributes).filter(([id]) => id.startsWith('ml')) - ) - : nodeFields.attributes; + nodeFields.attributes = nodeFields.attributes; const allocatedModels = (trainedModelStats as TrainedModelStatsResponse[]) .filter( @@ -150,15 +161,9 @@ export function modelsProvider( }); const memoryRes = { - adTotalMemory: sumBy( - adMemoryReport.filter((ad) => ad.node_id === nodeId), - 'model_size' - ), - dfaTotalMemory: sumBy( - dfaMemoryReport.filter((dfa) => dfa.node_id === nodeId), - 'model_size' - ), - trainedModelsTotalMemory: sumBy(modelsMemoryUsage, 'model_size'), + adTotalMemory: node.mem.ml.anomaly_detectors_in_bytes, + dfaTotalMemory: node.mem.ml.data_frame_analytics_in_bytes, + trainedModelsTotalMemory: node.mem.ml.native_inference_in_bytes, }; for (const key of Object.keys(memoryRes)) { @@ -168,7 +173,7 @@ export function modelsProvider( * ML job to run on a given node will do this, and then subsequent ML jobs on the same node will reuse the * same already-loaded code. */ - memoryRes[key as keyof typeof memoryRes] += NATIVE_EXECUTABLE_CODE_OVERHEAD; + memoryRes[key as keyof typeof memoryRes] += node.mem.ml.native_code_overhead_in_bytes; break; } } @@ -179,10 +184,8 @@ export function modelsProvider( allocated_models: allocatedModels, memory_overview: { machine_memory: { - // TODO remove ts-ignore when elasticsearch client is updated - // @ts-ignore - total: Number(node.os?.mem.adjusted_total_in_bytes ?? node.os?.mem.total_in_bytes), - jvm: Number(node.attributes!['ml.max_jvm_size']), + total: node.mem.adjusted_total_in_bytes, + jvm: node.jvm.heap_max_in_bytes, }, anomaly_detection: { total: memoryRes.adTotalMemory, @@ -194,13 +197,14 @@ export function modelsProvider( total: memoryRes.trainedModelsTotalMemory, by_model: modelsMemoryUsage, }, + ml_max_in_bytes: node.mem.ml.max_in_bytes, }, }; } ); return { - count: nodeDeploymentStatsResponses.length, + _nodes: response._nodes, nodes: nodeDeploymentStatsResponses, }; }, diff --git a/x-pack/plugins/ml/server/models/memory_overview/index.ts b/x-pack/plugins/ml/server/models/memory_overview/index.ts deleted file mode 100644 index 038b1cd8d4b80..0000000000000 --- a/x-pack/plugins/ml/server/models/memory_overview/index.ts +++ /dev/null @@ -1,8 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -export { memoryOverviewServiceProvider } from './memory_overview_service'; diff --git a/x-pack/plugins/ml/server/models/memory_overview/memory_overview_service.ts b/x-pack/plugins/ml/server/models/memory_overview/memory_overview_service.ts deleted file mode 100644 index d7f6eb584f7fe..0000000000000 --- a/x-pack/plugins/ml/server/models/memory_overview/memory_overview_service.ts +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import numeral from '@elastic/numeral'; -import { keyBy } from 'lodash'; -import { MlClient } from '../../lib/ml_client'; - -export type MemoryOverviewService = ReturnType; - -export interface MlJobMemoryOverview { - job_id: string; - node_id: string; - model_size: number; -} - -const MB = Math.pow(2, 20); - -const AD_PROCESS_MEMORY_OVERHEAD = 10 * MB; -const DFA_PROCESS_MEMORY_OVERHEAD = 5 * MB; -export const NATIVE_EXECUTABLE_CODE_OVERHEAD = 30 * MB; - -/** - * Provides a service for memory overview across ML. - * @param mlClient - */ -export function memoryOverviewServiceProvider(mlClient: MlClient) { - return { - /** - * Retrieves memory consumed my started DFA jobs. - */ - async getDFAMemoryOverview(): Promise { - const { data_frame_analytics: dfaStats } = await mlClient.getDataFrameAnalyticsStats(); - - const dfaMemoryReport = dfaStats - .filter((dfa) => dfa.state === 'started') - .map((dfa) => { - return { - node_id: dfa.node?.id, - job_id: dfa.id, - }; - }) as MlJobMemoryOverview[]; - - if (dfaMemoryReport.length === 0) { - return []; - } - - const dfaMemoryKeyByJobId = keyBy(dfaMemoryReport, 'job_id'); - - const { data_frame_analytics: startedDfaJobs } = await mlClient.getDataFrameAnalytics({ - id: dfaMemoryReport.map((v) => v.job_id).join(','), - }); - - startedDfaJobs.forEach((dfa) => { - dfaMemoryKeyByJobId[dfa.id].model_size = - numeral( - dfa.model_memory_limit?.toUpperCase() - // @ts-ignore - ).value() + DFA_PROCESS_MEMORY_OVERHEAD; - }); - - return dfaMemoryReport; - }, - /** - * Retrieves memory consumed by opened Anomaly Detection jobs. - */ - async getAnomalyDetectionMemoryOverview(): Promise { - const { jobs: jobsStats } = await mlClient.getJobStats(); - - return jobsStats - .filter((v) => v.state === 'opened') - .map((jobStats) => { - return { - node_id: jobStats.node!.id, - // @ts-expect-error model_bytes can be string | number, cannot sum it with AD_PROCESS_MEMORY_OVERHEAD - model_size: jobStats.model_size_stats.model_bytes + AD_PROCESS_MEMORY_OVERHEAD, - job_id: jobStats.job_id, - }; - }); - }, - }; -} diff --git a/x-pack/plugins/ml/server/routes/trained_models.ts b/x-pack/plugins/ml/server/routes/trained_models.ts index 2cbf9a4dde763..887ad47f1ceb2 100644 --- a/x-pack/plugins/ml/server/routes/trained_models.ts +++ b/x-pack/plugins/ml/server/routes/trained_models.ts @@ -16,7 +16,6 @@ import { } from './schemas/inference_schema'; import { modelsProvider } from '../models/data_frame_analytics'; import { TrainedModelConfigResponse } from '../../common/types/trained_models'; -import { memoryOverviewServiceProvider } from '../models/memory_overview'; import { mlLog } from '../lib/log'; import { forceQuerySchema } from './schemas/anomaly_detectors_schema'; @@ -278,12 +277,7 @@ export function trainedModelsRoutes({ router, routeGuard }: RouteInitialization) }, routeGuard.fullLicenseAPIGuard(async ({ client, mlClient, request, response }) => { try { - const memoryOverviewService = memoryOverviewServiceProvider(mlClient); - const result = await modelsProvider( - client, - mlClient, - memoryOverviewService - ).getNodesOverview(); + const result = await modelsProvider(client, mlClient).getNodesOverview(); return response.ok({ body: result, }); From 93fc801b18eea76334400d890a2011805802020a Mon Sep 17 00:00:00 2001 From: "Joey F. Poon" Date: Wed, 16 Mar 2022 11:52:29 -0500 Subject: [PATCH 2/3] [Security Solution] blocklist create/edit form (#127098) --- .../src/path_validations/index.ts | 8 +- .../endpoint/schema/trusted_apps.test.ts | 6 +- .../common/endpoint/schema/trusted_apps.ts | 4 +- .../service/trusted_apps/validations.ts | 6 +- .../endpoint/types/exception_list_items.ts | 22 + .../common/endpoint/types/index.ts | 1 + .../common/endpoint/types/trusted_apps.ts | 8 +- .../common/utils/path_placeholder.ts | 8 +- .../utils/exception_list_items/index.ts | 12 + .../utils/exception_list_items/mappers.ts | 218 ++++++++ .../artifact_list_page/artifact_list_page.tsx | 10 +- .../components/artifact_flyout.test.tsx | 2 + .../components/artifact_flyout.tsx | 7 + .../components/artifact_list_page/types.ts | 4 + .../services/blocklists_api_client.ts | 45 +- .../pages/blocklist/translations.ts | 119 +++++ .../pages/blocklist/view/blocklist.tsx | 28 +- .../view/components/blocklist_form.tsx | 490 ++++++++++++++++++ .../pages/trusted_apps/service/mappers.ts | 153 ++---- .../pages/trusted_apps/state/type_guards.ts | 6 +- .../pages/trusted_apps/store/builders.ts | 4 +- .../condition_entry_input/index.test.tsx | 6 +- .../condition_entry_input/index.tsx | 13 +- .../view/components/condition_group/index.tsx | 6 +- .../components/create_trusted_app_form.tsx | 4 +- .../exceptions_list_api_client.ts | 81 ++- .../server/endpoint/lib/artifacts/lists.ts | 2 +- .../validators/trusted_app_validator.ts | 2 +- 28 files changed, 1064 insertions(+), 211 deletions(-) create mode 100644 x-pack/plugins/security_solution/common/endpoint/types/exception_list_items.ts create mode 100644 x-pack/plugins/security_solution/public/common/utils/exception_list_items/index.ts create mode 100644 x-pack/plugins/security_solution/public/common/utils/exception_list_items/mappers.ts create mode 100644 x-pack/plugins/security_solution/public/management/pages/blocklist/translations.ts create mode 100644 x-pack/plugins/security_solution/public/management/pages/blocklist/view/components/blocklist_form.tsx diff --git a/packages/kbn-securitysolution-utils/src/path_validations/index.ts b/packages/kbn-securitysolution-utils/src/path_validations/index.ts index 82d2cc3151b90..27d2f31d76d9a 100644 --- a/packages/kbn-securitysolution-utils/src/path_validations/index.ts +++ b/packages/kbn-securitysolution-utils/src/path_validations/index.ts @@ -28,7 +28,9 @@ export const enum OperatingSystem { WINDOWS = 'windows', } -export type TrustedAppEntryTypes = 'match' | 'wildcard'; +export type EntryTypes = 'match' | 'wildcard' | 'match_any'; +export type TrustedAppEntryTypes = Extract; + /* * regex to match executable names * starts matching from the eol of the path @@ -82,7 +84,7 @@ export const hasSimpleExecutableName = ({ value, }: { os: OperatingSystem; - type: TrustedAppEntryTypes; + type: EntryTypes; value: string; }): boolean => { if (type === 'wildcard') { @@ -99,7 +101,7 @@ export const isPathValid = ({ }: { os: OperatingSystem; field: ConditionEntryField | 'file.path.text'; - type: TrustedAppEntryTypes; + type: EntryTypes; value: string; }): boolean => { if (field === ConditionEntryField.PATH || field === 'file.path.text') { diff --git a/x-pack/plugins/security_solution/common/endpoint/schema/trusted_apps.test.ts b/x-pack/plugins/security_solution/common/endpoint/schema/trusted_apps.test.ts index ee5e064fb8666..7c8d6dd6138aa 100644 --- a/x-pack/plugins/security_solution/common/endpoint/schema/trusted_apps.test.ts +++ b/x-pack/plugins/security_solution/common/endpoint/schema/trusted_apps.test.ts @@ -12,7 +12,7 @@ import { PutTrustedAppUpdateRequestSchema, } from './trusted_apps'; import { ConditionEntryField, OperatingSystem } from '@kbn/securitysolution-utils'; -import { ConditionEntry, NewTrustedApp, PutTrustedAppsRequestParams } from '../types'; +import { TrustedAppConditionEntry, NewTrustedApp, PutTrustedAppsRequestParams } from '../types'; describe('When invoking Trusted Apps Schema', () => { describe('for GET List', () => { @@ -94,7 +94,7 @@ describe('When invoking Trusted Apps Schema', () => { }); describe('for POST Create', () => { - const createConditionEntry = (data?: T): ConditionEntry => ({ + const createConditionEntry = (data?: T): TrustedAppConditionEntry => ({ field: ConditionEntryField.PATH, type: 'match', operator: 'included', @@ -378,7 +378,7 @@ describe('When invoking Trusted Apps Schema', () => { }); describe('for PUT Update', () => { - const createConditionEntry = (data?: T): ConditionEntry => ({ + const createConditionEntry = (data?: T): TrustedAppConditionEntry => ({ field: ConditionEntryField.PATH, type: 'match', operator: 'included', diff --git a/x-pack/plugins/security_solution/common/endpoint/schema/trusted_apps.ts b/x-pack/plugins/security_solution/common/endpoint/schema/trusted_apps.ts index 88ac65768e163..a5476159a03ac 100644 --- a/x-pack/plugins/security_solution/common/endpoint/schema/trusted_apps.ts +++ b/x-pack/plugins/security_solution/common/endpoint/schema/trusted_apps.ts @@ -7,7 +7,7 @@ import { schema } from '@kbn/config-schema'; import { ConditionEntryField, OperatingSystem } from '@kbn/securitysolution-utils'; -import { ConditionEntry } from '../types'; +import { TrustedAppConditionEntry } from '../types'; import { getDuplicateFields, isValidHash } from '../service/trusted_apps/validations'; export const DeleteTrustedAppsRequestSchema = { @@ -96,7 +96,7 @@ const MacEntrySchema = schema.object({ const entriesSchemaOptions = { minSize: 1, - validate(entries: ConditionEntry[]) { + validate(entries: TrustedAppConditionEntry[]) { return ( getDuplicateFields(entries) .map((field) => `duplicatedEntry.${field}`) diff --git a/x-pack/plugins/security_solution/common/endpoint/service/trusted_apps/validations.ts b/x-pack/plugins/security_solution/common/endpoint/service/trusted_apps/validations.ts index 2d2c50572a8bc..df9ff68e5ef3a 100644 --- a/x-pack/plugins/security_solution/common/endpoint/service/trusted_apps/validations.ts +++ b/x-pack/plugins/security_solution/common/endpoint/service/trusted_apps/validations.ts @@ -6,7 +6,7 @@ */ import { ConditionEntryField } from '@kbn/securitysolution-utils'; -import { ConditionEntry } from '../../types'; +import { TrustedAppConditionEntry } from '../../types'; const HASH_LENGTHS: readonly number[] = [ 32, // MD5 @@ -18,8 +18,8 @@ const INVALID_CHARACTERS_PATTERN = /[^0-9a-f]/i; export const isValidHash = (value: string) => HASH_LENGTHS.includes(value.length) && !INVALID_CHARACTERS_PATTERN.test(value); -export const getDuplicateFields = (entries: ConditionEntry[]) => { - const groupedFields = new Map(); +export const getDuplicateFields = (entries: TrustedAppConditionEntry[]) => { + const groupedFields = new Map(); entries.forEach((entry) => { // With the move to the Exception Lists api, the server side now validates individual diff --git a/x-pack/plugins/security_solution/common/endpoint/types/exception_list_items.ts b/x-pack/plugins/security_solution/common/endpoint/types/exception_list_items.ts new file mode 100644 index 0000000000000..efdbe42465a5a --- /dev/null +++ b/x-pack/plugins/security_solution/common/endpoint/types/exception_list_items.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ConditionEntryField, EntryTypes } from '@kbn/securitysolution-utils'; + +export type ConditionEntriesMap = { + [K in ConditionEntryField]?: T; +}; + +export interface ConditionEntry< + F extends ConditionEntryField = ConditionEntryField, + T extends EntryTypes = EntryTypes +> { + field: F; + type: T; + operator: 'included'; + value: string | string[]; +} diff --git a/x-pack/plugins/security_solution/common/endpoint/types/index.ts b/x-pack/plugins/security_solution/common/endpoint/types/index.ts index 9e19cbd6c99c5..cbbf3010ef7b2 100644 --- a/x-pack/plugins/security_solution/common/endpoint/types/index.ts +++ b/x-pack/plugins/security_solution/common/endpoint/types/index.ts @@ -12,6 +12,7 @@ import { ManifestSchema } from '../schema/manifest'; export * from './actions'; export * from './os'; export * from './trusted_apps'; +export type { ConditionEntriesMap, ConditionEntry } from './exception_list_items'; /** * Supported React-Router state for the Policy Details page diff --git a/x-pack/plugins/security_solution/common/endpoint/types/trusted_apps.ts b/x-pack/plugins/security_solution/common/endpoint/types/trusted_apps.ts index 3872df8d10247..ab56d35d79f99 100644 --- a/x-pack/plugins/security_solution/common/endpoint/types/trusted_apps.ts +++ b/x-pack/plugins/security_solution/common/endpoint/types/trusted_apps.ts @@ -19,6 +19,7 @@ import { PutTrustedAppUpdateRequestSchema, GetTrustedAppsSummaryRequestSchema, } from '../schema/trusted_apps'; +import { ConditionEntry } from './exception_list_items'; /** API request params for deleting Trusted App entry */ export type DeleteTrustedAppsRequestParams = TypeOf; @@ -75,17 +76,18 @@ export enum OperatorFieldIds { matches = 'matches', } -export interface ConditionEntry { +export interface TrustedAppConditionEntry + extends ConditionEntry { field: T; type: TrustedAppEntryTypes; operator: 'included'; value: string; } -export type MacosLinuxConditionEntry = ConditionEntry< +export type MacosLinuxConditionEntry = TrustedAppConditionEntry< ConditionEntryField.HASH | ConditionEntryField.PATH >; -export type WindowsConditionEntry = ConditionEntry< +export type WindowsConditionEntry = TrustedAppConditionEntry< ConditionEntryField.HASH | ConditionEntryField.PATH | ConditionEntryField.SIGNER >; diff --git a/x-pack/plugins/security_solution/common/utils/path_placeholder.ts b/x-pack/plugins/security_solution/common/utils/path_placeholder.ts index 328df398dd576..b915c390b2011 100644 --- a/x-pack/plugins/security_solution/common/utils/path_placeholder.ts +++ b/x-pack/plugins/security_solution/common/utils/path_placeholder.ts @@ -4,11 +4,7 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import { - ConditionEntryField, - OperatingSystem, - TrustedAppEntryTypes, -} from '@kbn/securitysolution-utils'; +import { ConditionEntryField, OperatingSystem, EntryTypes } from '@kbn/securitysolution-utils'; export const getPlaceholderText = () => ({ windows: { @@ -28,7 +24,7 @@ export const getPlaceholderTextByOSType = ({ }: { os: OperatingSystem; field: ConditionEntryField; - type: TrustedAppEntryTypes; + type: EntryTypes; }): string | undefined => { if (field === ConditionEntryField.PATH) { if (os === OperatingSystem.WINDOWS) { diff --git a/x-pack/plugins/security_solution/public/common/utils/exception_list_items/index.ts b/x-pack/plugins/security_solution/public/common/utils/exception_list_items/index.ts new file mode 100644 index 0000000000000..1dc32749bc6d5 --- /dev/null +++ b/x-pack/plugins/security_solution/public/common/utils/exception_list_items/index.ts @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { + conditionEntriesToEntries, + entriesToConditionEntriesMap, + entriesToConditionEntries, +} from './mappers'; diff --git a/x-pack/plugins/security_solution/public/common/utils/exception_list_items/mappers.ts b/x-pack/plugins/security_solution/public/common/utils/exception_list_items/mappers.ts new file mode 100644 index 0000000000000..e04d059a515d4 --- /dev/null +++ b/x-pack/plugins/security_solution/public/common/utils/exception_list_items/mappers.ts @@ -0,0 +1,218 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + EntriesArray, + EntryMatch, + EntryMatchAny, + EntryMatchWildcard, + EntryNested, + NestedEntriesArray, +} from '@kbn/securitysolution-io-ts-list-types'; +import { ConditionEntryField, EntryTypes } from '@kbn/securitysolution-utils'; + +import { ConditionEntriesMap, ConditionEntry } from '../../../../common/endpoint/types'; + +const OPERATOR_VALUE = 'included'; + +const hashType = (hash: string): 'md5' | 'sha256' | 'sha1' | undefined => { + switch (hash.length) { + case 32: + return 'md5'; + case 40: + return 'sha1'; + case 64: + return 'sha256'; + } +}; + +const createEntryMatch = (field: string, value: string): EntryMatch => { + return { field, value, type: 'match', operator: OPERATOR_VALUE }; +}; + +const createEntryMatchAny = (field: string, value: string[]): EntryMatchAny => { + return { field, value, type: 'match_any', operator: OPERATOR_VALUE }; +}; + +const createEntryMatchWildcard = (field: string, value: string): EntryMatchWildcard => { + return { field, value, type: 'wildcard', operator: OPERATOR_VALUE }; +}; + +const createEntryNested = (field: string, entries: NestedEntriesArray): EntryNested => { + return { field, entries, type: 'nested' }; +}; + +function groupHashEntry(conditionEntry: ConditionEntry): EntriesArray { + const entriesArray: EntriesArray = []; + + if (!Array.isArray(conditionEntry.value)) { + const entry = createEntryMatch( + `process.hash.${hashType(conditionEntry.value)}`, + conditionEntry.value.toLowerCase() + ); + entriesArray.push(entry); + return entriesArray; + } + + const hashTypeGroups: { md5: string[]; sha1: string[]; sha256: string[] } = + conditionEntry.value.reduce( + (memo, val) => { + const type = hashType(val); + if (!type) return memo; + + return { + ...memo, + [type]: [...memo[type], val], + }; + }, + { + md5: [], + sha1: [], + sha256: [], + } as { md5: string[]; sha1: string[]; sha256: string[] } + ); + Object.entries(hashTypeGroups).forEach(([type, values]) => { + if (!values.length) { + return; + } + + const entry = createEntryMatchAny(`process.hash.${type}`, values); + entriesArray.push(entry); + }); + + return entriesArray; +} + +function createNestedSignatureEntry( + value: string | string[], + isTrustedApp: boolean = false +): EntryNested { + const subjectNameMatch = Array.isArray(value) + ? createEntryMatchAny('subject_name', value) + : createEntryMatch('subject_name', value); + const nestedEntries: EntryNested['entries'] = []; + if (isTrustedApp) nestedEntries.push(createEntryMatch('trusted', 'true')); + nestedEntries.push(subjectNameMatch); + return createEntryNested('process.Ext.code_signature', nestedEntries); +} + +function createWildcardPathEntry(value: string | string[]): EntryMatchWildcard | EntryMatchAny { + return Array.isArray(value) + ? createEntryMatchAny('process.executable.caseless', value) + : createEntryMatchWildcard('process.executable.caseless', value); +} + +function createPathEntry(value: string | string[]): EntryMatch | EntryMatchAny { + return Array.isArray(value) + ? createEntryMatchAny('process.executable.caseless', value) + : createEntryMatch('process.executable.caseless', value); +} + +export const conditionEntriesToEntries = ( + conditionEntries: ConditionEntry[], + isTrustedApp: boolean = false +): EntriesArray => { + const entriesArray: EntriesArray = []; + + conditionEntries.forEach((conditionEntry) => { + if (conditionEntry.field === ConditionEntryField.HASH) { + groupHashEntry(conditionEntry).forEach((entry) => entriesArray.push(entry)); + } else if (conditionEntry.field === ConditionEntryField.SIGNER) { + const entry = createNestedSignatureEntry(conditionEntry.value, isTrustedApp); + entriesArray.push(entry); + } else if ( + conditionEntry.field === ConditionEntryField.PATH && + conditionEntry.type === 'wildcard' + ) { + const entry = createWildcardPathEntry(conditionEntry.value); + entriesArray.push(entry); + } else { + const entry = createPathEntry(conditionEntry.value); + entriesArray.push(entry); + } + }); + + return entriesArray; +}; + +const createConditionEntry = ( + field: ConditionEntryField, + type: EntryTypes, + value: string | string[] +): ConditionEntry => { + return { field, value, type, operator: OPERATOR_VALUE }; +}; + +export const entriesToConditionEntriesMap = ( + entries: EntriesArray +): ConditionEntriesMap => { + return entries.reduce((memo: ConditionEntriesMap, entry) => { + if (entry.field.startsWith('process.hash') && entry.type === 'match') { + return { + ...memo, + [ConditionEntryField.HASH]: createConditionEntry( + ConditionEntryField.HASH, + entry.type, + entry.value + ), + } as ConditionEntriesMap; + } else if (entry.field.startsWith('process.hash') && entry.type === 'match_any') { + const currentValues = (memo[ConditionEntryField.HASH]?.value as string[]) ?? []; + + return { + ...memo, + [ConditionEntryField.HASH]: createConditionEntry(ConditionEntryField.HASH, entry.type, [ + ...currentValues, + ...entry.value, + ]), + } as ConditionEntriesMap; + } else if ( + entry.field === ConditionEntryField.PATH && + (entry.type === 'match' || entry.type === 'match_any' || entry.type === 'wildcard') + ) { + return { + ...memo, + [ConditionEntryField.PATH]: createConditionEntry( + ConditionEntryField.PATH, + entry.type, + entry.value + ), + } as ConditionEntriesMap; + } else if (entry.field === ConditionEntryField.SIGNER && entry.type === 'nested') { + const subjectNameCondition = entry.entries.find((subEntry): subEntry is EntryMatch => { + return ( + subEntry.field === 'subject_name' && + (subEntry.type === 'match' || subEntry.type === 'match_any') + ); + }); + + if (subjectNameCondition) { + return { + ...memo, + [ConditionEntryField.SIGNER]: createConditionEntry( + ConditionEntryField.SIGNER, + subjectNameCondition.type, + subjectNameCondition.value + ), + } as ConditionEntriesMap; + } + } + + return memo; + }, {} as ConditionEntriesMap); +}; + +export const entriesToConditionEntries = ( + entries: EntriesArray +): ConditionEntry[] => { + const conditionEntriesMap: ConditionEntriesMap = entriesToConditionEntriesMap(entries); + + return Object.values(conditionEntriesMap).reduce((memo, entry) => { + if (!entry) return memo; + return [...memo, entry]; + }, [] as ConditionEntry[]); +}; diff --git a/x-pack/plugins/security_solution/public/management/components/artifact_list_page/artifact_list_page.tsx b/x-pack/plugins/security_solution/public/management/components/artifact_list_page/artifact_list_page.tsx index 87e3b2bb00519..19f3a810f76d4 100644 --- a/x-pack/plugins/security_solution/public/management/components/artifact_list_page/artifact_list_page.tsx +++ b/x-pack/plugins/security_solution/public/management/components/artifact_list_page/artifact_list_page.tsx @@ -214,15 +214,15 @@ export const ArtifactListPage = memo( setSelectedItemForDelete(undefined); }, []); - const handleArtifactFlyoutOnClose = useCallback(() => { - setSelectedItemForEdit(undefined); - }, []); - const handleArtifactFlyoutOnSuccess = useCallback(() => { setSelectedItemForEdit(undefined); refetchListData(); }, [refetchListData]); + const handleArtifactFlyoutOnClose = useCallback(() => { + setSelectedItemForEdit(undefined); + }, []); + if (isPageInitializing) { return ; } @@ -255,6 +255,8 @@ export const ArtifactListPage = memo( labels={labels} size={flyoutSize} submitHandler={onFormSubmit} + policies={policiesRequest.data?.items || []} + policiesIsLoading={policiesRequest.isLoading} data-test-subj={getTestId('flyout')} /> )} diff --git a/x-pack/plugins/security_solution/public/management/components/artifact_list_page/components/artifact_flyout.test.tsx b/x-pack/plugins/security_solution/public/management/components/artifact_list_page/components/artifact_flyout.test.tsx index 743f71bfead05..ba37f1ab167e2 100644 --- a/x-pack/plugins/security_solution/public/management/components/artifact_list_page/components/artifact_flyout.test.tsx +++ b/x-pack/plugins/security_solution/public/management/components/artifact_list_page/components/artifact_flyout.test.tsx @@ -107,6 +107,8 @@ describe('When the flyout is opened in the ArtifactListPage component', () => { }, mode: 'create', onChange: expect.any(Function), + policies: expect.any(Array), + policiesIsLoading: false, }, expect.anything() ); diff --git a/x-pack/plugins/security_solution/public/management/components/artifact_list_page/components/artifact_flyout.tsx b/x-pack/plugins/security_solution/public/management/components/artifact_list_page/components/artifact_flyout.tsx index 63759df8d42cd..ab893b57f16e8 100644 --- a/x-pack/plugins/security_solution/public/management/components/artifact_list_page/components/artifact_flyout.tsx +++ b/x-pack/plugins/security_solution/public/management/components/artifact_list_page/components/artifact_flyout.tsx @@ -39,6 +39,7 @@ import { useWithArtifactSubmitData } from '../hooks/use_with_artifact_submit_dat import { useIsArtifactAllowedPerPolicyUsage } from '../hooks/use_is_artifact_allowed_per_policy_usage'; import { useIsMounted } from '../../hooks/use_is_mounted'; import { useGetArtifact } from '../../../hooks/artifacts'; +import type { PolicyData } from '../../../../../common/endpoint/types'; export const ARTIFACT_FLYOUT_LABELS = Object.freeze({ flyoutEditTitle: i18n.translate('xpack.securitySolution.artifactListPage.flyoutEditTitle', { @@ -151,6 +152,8 @@ const createFormInitialState = ( export interface ArtifactFlyoutProps { apiClient: ExceptionsListApiClient; FormComponent: React.ComponentType; + policies: PolicyData[]; + policiesIsLoading: boolean; onSuccess(): void; onClose(): void; submitHandler?: ( @@ -175,6 +178,8 @@ export const ArtifactFlyout = memo( ({ apiClient, item, + policies, + policiesIsLoading, FormComponent, onSuccess, onClose, @@ -373,6 +378,8 @@ export const ArtifactFlyout = memo( item={formState.item} error={submitError ?? undefined} mode={formMode} + policies={policies} + policiesIsLoading={policiesIsLoading} /> )} diff --git a/x-pack/plugins/security_solution/public/management/components/artifact_list_page/types.ts b/x-pack/plugins/security_solution/public/management/components/artifact_list_page/types.ts index fa63ebb863ce5..8ee4b50ba4671 100644 --- a/x-pack/plugins/security_solution/public/management/components/artifact_list_page/types.ts +++ b/x-pack/plugins/security_solution/public/management/components/artifact_list_page/types.ts @@ -10,6 +10,7 @@ import type { ExceptionListItemSchema, CreateExceptionListItemSchema, } from '@kbn/securitysolution-io-ts-list-types'; +import { PolicyData } from '../../../../common/endpoint/types'; export interface ArtifactListPageUrlParams { page?: number; @@ -30,6 +31,9 @@ export interface ArtifactFormComponentProps { /** Error will be set if the submission of the form to the api results in an API error. Form can use it to provide feedback to the user */ error: HttpFetchError | undefined; + policies: PolicyData[]; + policiesIsLoading: boolean; + /** reports the state of the form data and the current updated item */ onChange(formStatus: ArtifactFormComponentOnChangeCallbackProps): void; } diff --git a/x-pack/plugins/security_solution/public/management/pages/blocklist/services/blocklists_api_client.ts b/x-pack/plugins/security_solution/public/management/pages/blocklist/services/blocklists_api_client.ts index fa0451d9363ad..9c02729c68273 100644 --- a/x-pack/plugins/security_solution/public/management/pages/blocklist/services/blocklists_api_client.ts +++ b/x-pack/plugins/security_solution/public/management/pages/blocklist/services/blocklists_api_client.ts @@ -5,22 +5,61 @@ * 2.0. */ +import { + CreateExceptionListItemSchema, + ExceptionListItemSchema, + UpdateExceptionListItemSchema, +} from '@kbn/securitysolution-io-ts-list-types'; import { ENDPOINT_BLOCKLISTS_LIST_ID } from '@kbn/securitysolution-list-constants'; + import { HttpStart } from 'kibana/public'; +import { ConditionEntry } from '../../../../../common/endpoint/types'; +import { + conditionEntriesToEntries, + entriesToConditionEntries, +} from '../../../../common/utils/exception_list_items'; import { ExceptionsListApiClient } from '../../../services/exceptions_list/exceptions_list_api_client'; import { BLOCKLISTS_LIST_DEFINITION } from '../constants'; +function readTransform(item: ExceptionListItemSchema): ExceptionListItemSchema { + return { + ...item, + entries: entriesToConditionEntries(item.entries) as ExceptionListItemSchema['entries'], + }; +} + +function writeTransform( + item: T +): T { + return { + ...item, + entries: conditionEntriesToEntries(item.entries as ConditionEntry[]), + } as T; +} + /** * Blocklist exceptions Api client class using ExceptionsListApiClient as base class - * It follow the Singleton pattern. + * It follows the Singleton pattern. * Please, use the getInstance method instead of creating a new instance when using this implementation. */ export class BlocklistsApiClient extends ExceptionsListApiClient { constructor(http: HttpStart) { - super(http, ENDPOINT_BLOCKLISTS_LIST_ID, BLOCKLISTS_LIST_DEFINITION); + super( + http, + ENDPOINT_BLOCKLISTS_LIST_ID, + BLOCKLISTS_LIST_DEFINITION, + readTransform, + writeTransform + ); } public static getInstance(http: HttpStart): ExceptionsListApiClient { - return super.getInstance(http, ENDPOINT_BLOCKLISTS_LIST_ID, BLOCKLISTS_LIST_DEFINITION); + return super.getInstance( + http, + ENDPOINT_BLOCKLISTS_LIST_ID, + BLOCKLISTS_LIST_DEFINITION, + readTransform, + writeTransform + ); } } diff --git a/x-pack/plugins/security_solution/public/management/pages/blocklist/translations.ts b/x-pack/plugins/security_solution/public/management/pages/blocklist/translations.ts new file mode 100644 index 0000000000000..f7e4344cee23c --- /dev/null +++ b/x-pack/plugins/security_solution/public/management/pages/blocklist/translations.ts @@ -0,0 +1,119 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { i18n } from '@kbn/i18n'; +import { ConditionEntryField } from '@kbn/securitysolution-utils'; + +export const DETAILS_HEADER = i18n.translate('xpack.securitySolution.blocklists.details.header', { + defaultMessage: 'Details', +}); + +export const DETAILS_HEADER_DESCRIPTION = i18n.translate( + 'xpack.securitySolution.blocklists.details.header.description', + { + defaultMessage: 'Add a blocklist to prevent selected applications from running on your hosts.', + } +); + +export const NAME_LABEL = i18n.translate('xpack.securitySolution.blocklists.name.label', { + defaultMessage: 'Name', +}); + +export const DESCRIPTION_LABEL = i18n.translate( + 'xpack.securitySolution.blocklists.description.label', + { + defaultMessage: 'Description', + } +); + +export const CONDITIONS_HEADER = i18n.translate( + 'xpack.securitySolution.blocklists.conditions.header', + { + defaultMessage: 'Conditions', + } +); + +export const CONDITIONS_HEADER_DESCRIPTION = i18n.translate( + 'xpack.securitySolution.blocklists.conditions.header.description', + { + defaultMessage: + 'Select an operating system and add conditions. Availability of conditions may depend on your chosen OS.', + } +); + +export const SELECT_OS_LABEL = i18n.translate('xpack.securitySolution.blocklists.os.label', { + defaultMessage: 'Select operating system', +}); + +export const FIELD_LABEL = i18n.translate('xpack.securitySolution.blocklists.field.label', { + defaultMessage: 'Field', +}); + +export const OPERATOR_LABEL = i18n.translate('xpack.securitySolution.blocklists.operator.label', { + defaultMessage: 'Operator', +}); + +export const VALUE_LABEL = i18n.translate('xpack.securitySolution.blocklists.value.label', { + defaultMessage: 'Value', +}); + +export const CONDITION_FIELD_TITLE: { [K in ConditionEntryField]: string } = { + [ConditionEntryField.HASH]: i18n.translate('xpack.securitySolution.blocklists.entry.field.hash', { + defaultMessage: 'Hash', + }), + [ConditionEntryField.PATH]: i18n.translate('xpack.securitySolution.blocklists.entry.field.path', { + defaultMessage: 'Path', + }), + [ConditionEntryField.SIGNER]: i18n.translate( + 'xpack.securitySolution.blocklists.entry.field.signature', + { defaultMessage: 'Signature' } + ), +}; + +export const CONDITION_FIELD_DESCRIPTION: { [K in ConditionEntryField]: string } = { + [ConditionEntryField.HASH]: i18n.translate( + 'xpack.securitySolution.blocklists.entry.field.description.hash', + { defaultMessage: 'md5, sha1, or sha256' } + ), + [ConditionEntryField.PATH]: i18n.translate( + 'xpack.securitySolution.blocklists.entry.field.description.path', + { defaultMessage: 'The full path of the application' } + ), + [ConditionEntryField.SIGNER]: i18n.translate( + 'xpack.securitySolution.blocklists.entry.field.description.signature', + { defaultMessage: 'The signer of the application' } + ), +}; + +export const POLICY_SELECT_DESCRIPTION = i18n.translate( + 'xpack.securitySolution.blocklists.policyAssignmentSectionDescription', + { + defaultMessage: + 'Assign this blocklist globally across all policies, or assign it to specific policies.', + } +); + +export const ERRORS = { + NAME_REQUIRED: i18n.translate('xpack.securitySolution.blocklists.errors.name.required', { + defaultMessage: 'Name is required', + }), + VALUE_REQUIRED: i18n.translate('xpack.securitySolution.blocklists.errors.values.required', { + defaultMessage: 'Field entry must have a value', + }), + INVALID_HASH: i18n.translate('xpack.securitySolution.blocklists.errors.values.invalidHash', { + defaultMessage: 'Invalid hash value', + }), + INVALID_PATH: i18n.translate('xpack.securitySolution.blocklists.errors.values.invalidPath', { + defaultMessage: 'Path may be formed incorrectly; verify value', + }), + WILDCARD_PRESENT: i18n.translate( + 'xpack.securitySolution.blocklists.errors.values.wildcardPresent', + { + defaultMessage: "A wildcard in the filename will affect the endpoint's performance", + } + ), +}; diff --git a/x-pack/plugins/security_solution/public/management/pages/blocklist/view/blocklist.tsx b/x-pack/plugins/security_solution/public/management/pages/blocklist/view/blocklist.tsx index c016c10ad319f..45d76614ddce2 100644 --- a/x-pack/plugins/security_solution/public/management/pages/blocklist/view/blocklist.tsx +++ b/x-pack/plugins/security_solution/public/management/pages/blocklist/view/blocklist.tsx @@ -7,32 +7,11 @@ import React, { memo } from 'react'; import { i18n } from '@kbn/i18n'; + import { useHttp } from '../../../../common/lib/kibana'; import { ArtifactListPage, ArtifactListPageProps } from '../../../components/artifact_list_page'; import { BlocklistsApiClient } from '../services'; - -// FIXME:PT delete this when real component is implemented -const TempDevFormComponent: ArtifactListPageProps['ArtifactFormComponent'] = (props) => { - // For Dev. Delete once we implement this component - // @ts-ignore - if (!window._dev_artifact_form_props) { - // @ts-ignore - window._dev_artifact_form_props = []; - // @ts-ignore - window.console.log(window._dev_artifact_form_props); - } - // @ts-ignore - window._dev_artifact_form_props.push(props); - - return ( -
-
- {props.error ? props.error?.body?.message || props.error : ''} -
- {`TODO: ${props.mode} Form here`} -
- ); -}; +import { BlockListForm } from './components/blocklist_form'; const BLOCKLIST_PAGE_LABELS: ArtifactListPageProps['labels'] = { pageTitle: i18n.translate('xpack.securitySolution.blocklist.pageTitle', { @@ -123,9 +102,10 @@ export const Blocklist = memo(() => { return ( ); }); diff --git a/x-pack/plugins/security_solution/public/management/pages/blocklist/view/components/blocklist_form.tsx b/x-pack/plugins/security_solution/public/management/pages/blocklist/view/components/blocklist_form.tsx new file mode 100644 index 0000000000000..293ff2a80aec9 --- /dev/null +++ b/x-pack/plugins/security_solution/public/management/pages/blocklist/view/components/blocklist_form.tsx @@ -0,0 +1,490 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { useMemo, useState, useCallback, memo, useEffect, useRef } from 'react'; +import styled from 'styled-components'; +import { + EuiForm, + EuiFormRow, + EuiFieldText, + EuiTextArea, + EuiHorizontalRule, + EuiText, + EuiSpacer, + EuiSuperSelect, + EuiSuperSelectOption, + EuiComboBox, + EuiComboBoxOptionOption, + EuiTitle, +} from '@elastic/eui'; +import { + OperatingSystem, + ConditionEntryField, + isPathValid, + hasSimpleExecutableName, +} from '@kbn/securitysolution-utils'; +import { isOneOfOperator } from '@kbn/securitysolution-list-utils'; +import { ExceptionListItemSchema } from '@kbn/securitysolution-io-ts-list-types'; +import { uniq } from 'lodash'; + +import { OS_TITLES } from '../../../../common/translations'; +import { ArtifactFormComponentProps } from '../../../../components/artifact_list_page'; +import { + CONDITIONS_HEADER, + CONDITIONS_HEADER_DESCRIPTION, + CONDITION_FIELD_DESCRIPTION, + CONDITION_FIELD_TITLE, + DESCRIPTION_LABEL, + DETAILS_HEADER, + DETAILS_HEADER_DESCRIPTION, + FIELD_LABEL, + NAME_LABEL, + OPERATOR_LABEL, + POLICY_SELECT_DESCRIPTION, + SELECT_OS_LABEL, + VALUE_LABEL, + ERRORS, +} from '../../translations'; +import { + EffectedPolicySelect, + EffectedPolicySelection, +} from '../../../../components/effected_policy_select'; +import { + GLOBAL_ARTIFACT_TAG, + BY_POLICY_ARTIFACT_TAG_PREFIX, +} from '../../../../../../common/endpoint/service/artifacts/constants'; +import { useLicense } from '../../../../../common/hooks/use_license'; +import { isValidHash } from '../../../../../../common/endpoint/service/trusted_apps/validations'; +import { isArtifactGlobal } from '../../../../../../common/endpoint/service/artifacts'; +import type { PolicyData } from '../../../../../../common/endpoint/types'; + +interface BlocklistEntry { + field: ConditionEntryField; + operator: 'included'; + type: 'match_any'; + value: string[]; +} + +interface ItemValidation { + name?: React.ReactNode[]; + value?: React.ReactNode[]; +} + +function createValidationMessage(message: string): React.ReactNode { + return
{message}
; +} + +function getDropdownDisplay(field: ConditionEntryField): React.ReactNode { + return ( + <> + {CONDITION_FIELD_TITLE[field]} + + {CONDITION_FIELD_DESCRIPTION[field]} + + + ); +} + +function getMarginForGridArea(gridArea: string): string { + switch (gridArea) { + case 'field': + return '0 4px 0 0'; + case 'operator': + return '0 4px 0 4px'; + case 'value': + return '0 0 0 4px'; + } + + return '0'; +} + +function isValid(itemValidation: ItemValidation): boolean { + return !Object.values(itemValidation).some((error) => error.length); +} + +const InputGroup = styled.div` + display: grid; + grid-template-columns: 25% 25% 50%; + grid-template-areas: 'field operator value'; +`; + +const InputItem = styled.div<{ gridArea: string }>` + grid-area: ${({ gridArea }) => gridArea}; + align-self: center; + vertical-align: baseline; + margin: ${({ gridArea }) => getMarginForGridArea(gridArea)}; +`; + +export const BlockListForm = memo( + ({ item, policies, policiesIsLoading, onChange }: ArtifactFormComponentProps) => { + const [visited, setVisited] = useState<{ name: boolean; value: boolean }>({ + name: false, + value: false, + }); + const warningsRef = useRef({}); + const errorsRef = useRef({}); + const [selectedPolicies, setSelectedPolicies] = useState([]); + + // select policies if editing + useEffect(() => { + const policyIds = item.tags?.map((tag) => tag.split(':')[1]) ?? []; + if (!policyIds.length) return; + const policiesData = policies.filter((policy) => policyIds.includes(policy.id)); + setSelectedPolicies(policiesData); + }, [item.tags, policies]); + + const blocklistEntry = useMemo((): BlocklistEntry => { + if (!item.entries.length) { + return { + field: ConditionEntryField.HASH, + operator: 'included', + type: 'match_any', + value: [], + }; + } + return item.entries[0] as BlocklistEntry; + }, [item.entries]); + + const selectedOs = useMemo((): OperatingSystem => { + if (!item?.os_types?.length) { + return OperatingSystem.WINDOWS; + } + + return item.os_types[0] as OperatingSystem; + }, [item?.os_types]); + + const selectedValues = useMemo(() => { + return blocklistEntry.value.map((label) => ({ label })); + }, [blocklistEntry.value]); + + const osOptions: Array> = useMemo( + () => + [OperatingSystem.LINUX, OperatingSystem.MAC, OperatingSystem.WINDOWS].map((os) => ({ + value: os, + inputDisplay: OS_TITLES[os], + })), + [] + ); + + const fieldOptions: Array> = useMemo(() => { + const selectableFields: Array> = [ + ConditionEntryField.HASH, + ConditionEntryField.PATH, + ].map((field) => ({ + value: field, + inputDisplay: CONDITION_FIELD_TITLE[field], + dropdownDisplay: getDropdownDisplay(field), + })); + if (selectedOs === OperatingSystem.WINDOWS) { + selectableFields.push({ + value: ConditionEntryField.SIGNER, + inputDisplay: CONDITION_FIELD_TITLE[ConditionEntryField.SIGNER], + dropdownDisplay: getDropdownDisplay(ConditionEntryField.SIGNER), + }); + } + + return selectableFields; + }, [selectedOs]); + + const validateValues = useCallback((nextItem: ArtifactFormComponentProps['item']) => { + const os = ((nextItem.os_types ?? [])[0] as OperatingSystem) ?? OperatingSystem.WINDOWS; + const { + field = ConditionEntryField.HASH, + type = 'match_any', + value: values = [], + } = (nextItem.entries[0] ?? {}) as BlocklistEntry; + + const newValueWarnings: React.ReactNode[] = []; + const newNameErrors: React.ReactNode[] = []; + const newValueErrors: React.ReactNode[] = []; + + // error if name empty + if (!nextItem.name.trim()) { + newNameErrors.push(createValidationMessage(ERRORS.NAME_REQUIRED)); + } + + // error if no values + if (!values.length) { + newValueErrors.push(createValidationMessage(ERRORS.VALUE_REQUIRED)); + } + + // error if invalid hash + if (field === ConditionEntryField.HASH && values.some((value) => !isValidHash(value))) { + newValueErrors.push(createValidationMessage(ERRORS.INVALID_HASH)); + } + + const isInvalidPath = values.some((value) => !isPathValid({ os, field, type, value })); + + // warn if invalid path + if (field !== ConditionEntryField.HASH && isInvalidPath) { + newValueWarnings.push(createValidationMessage(ERRORS.INVALID_PATH)); + } + + // warn if wildcard + if ( + field !== ConditionEntryField.HASH && + !isInvalidPath && + values.some((value) => !hasSimpleExecutableName({ os, type, value })) + ) { + newValueWarnings.push(createValidationMessage(ERRORS.WILDCARD_PRESENT)); + } + + warningsRef.current = { ...warningsRef, value: newValueWarnings }; + errorsRef.current = { name: newNameErrors, value: newValueErrors }; + }, []); + + const handleOnNameBlur = useCallback(() => { + setVisited((prevVisited) => ({ ...prevVisited, name: true })); + validateValues(item); + }, [item, validateValues]); + + const handleOnValueFocus = useCallback(() => { + setVisited((prevVisited) => ({ ...prevVisited, value: true })); + validateValues(item); + }, [item, validateValues]); + + const handleOnNameChange = useCallback( + (event: React.ChangeEvent) => { + const nextItem = { + ...item, + name: event.target.value, + }; + + validateValues(nextItem); + onChange({ + isValid: isValid(errorsRef.current), + item: nextItem, + }); + }, + [validateValues, onChange, item] + ); + + const handleOnDescriptionChange = useCallback( + (event: React.ChangeEvent) => { + onChange({ + isValid: isValid(errorsRef.current), + item: { + ...item, + description: event.target.value, + }, + }); + }, + [onChange, item] + ); + + const handleOnOsChange = useCallback( + (os: OperatingSystem) => { + const nextItem = { + ...item, + os_types: [os], + entries: [ + { + ...blocklistEntry, + field: + os !== OperatingSystem.WINDOWS && + blocklistEntry.field === ConditionEntryField.SIGNER + ? ConditionEntryField.HASH + : blocklistEntry.field, + }, + ], + }; + + validateValues(nextItem); + onChange({ + isValid: isValid(errorsRef.current), + item: nextItem, + }); + }, + [validateValues, blocklistEntry, onChange, item] + ); + + const handleOnFieldChange = useCallback( + (field: ConditionEntryField) => { + const nextItem = { + ...item, + entries: [{ ...blocklistEntry, field }], + }; + + validateValues(nextItem); + onChange({ + isValid: isValid(errorsRef.current), + item: nextItem, + }); + }, + [validateValues, onChange, item, blocklistEntry] + ); + + // only triggered on remove / clear + const handleOnValueChange = useCallback( + (change: Array>) => { + const value = change.map((option) => option.label); + const nextItem = { + ...item, + entries: [{ ...blocklistEntry, value }], + }; + + validateValues(nextItem); + onChange({ + isValid: isValid(errorsRef.current), + item: nextItem, + }); + }, + [validateValues, onChange, item, blocklistEntry] + ); + + const handleOnValueAdd = useCallback( + (option: string) => { + const splitValues = option.split(',').filter((value) => value.trim()); + const value = uniq([...blocklistEntry.value, ...splitValues]); + + const nextItem = { + ...item, + entries: [{ ...blocklistEntry, value }], + }; + + validateValues(nextItem); + + onChange({ + isValid: isValid(errorsRef.current), + item: nextItem, + }); + }, + [validateValues, onChange, item, blocklistEntry] + ); + + const handleOnPolicyChange = useCallback( + (change: EffectedPolicySelection) => { + const tags = change.isGlobal + ? [GLOBAL_ARTIFACT_TAG] + : change.selected.map((policy) => `${BY_POLICY_ARTIFACT_TAG_PREFIX}${policy.id}`); + + setSelectedPolicies(change.selected); + onChange({ + isValid: isValid(errorsRef.current), + item: { + ...item, + tags, + }, + }); + }, + [onChange, item] + ); + + return ( + + +

{DETAILS_HEADER}

+ + + +

{DETAILS_HEADER_DESCRIPTION}

+ + + + + + + + + + + +

{CONDITIONS_HEADER}

+ + + +

{CONDITIONS_HEADER_DESCRIPTION}

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + <> + + + + + + + ); + } +); diff --git a/x-pack/plugins/security_solution/public/management/pages/trusted_apps/service/mappers.ts b/x-pack/plugins/security_solution/public/management/pages/trusted_apps/service/mappers.ts index f440a0a394631..fda0dde4c473a 100644 --- a/x-pack/plugins/security_solution/public/management/pages/trusted_apps/service/mappers.ts +++ b/x-pack/plugins/security_solution/public/management/pages/trusted_apps/service/mappers.ts @@ -7,32 +7,27 @@ import { CreateExceptionListItemSchema, - EntriesArray, - EntryMatch, - EntryMatchWildcard, - EntryNested, ExceptionListItemSchema, - NestedEntriesArray, OsType, UpdateExceptionListItemSchema, } from '@kbn/securitysolution-io-ts-list-types'; import { ENDPOINT_TRUSTED_APPS_LIST_ID } from '@kbn/securitysolution-list-constants'; +import { ConditionEntryField, OperatingSystem } from '@kbn/securitysolution-utils'; import { - ConditionEntryField, - OperatingSystem, - TrustedAppEntryTypes, -} from '@kbn/securitysolution-utils'; -import { - ConditionEntry, EffectScope, NewTrustedApp, TrustedApp, + TrustedAppConditionEntry, UpdateTrustedApp, + ConditionEntriesMap, } from '../../../../../common/endpoint/types'; import { tagsToEffectScope } from '../../../../../common/endpoint/service/trusted_apps/mapping'; import { BY_POLICY_ARTIFACT_TAG_PREFIX } from '../../../../../common/endpoint/service/artifacts/constants'; +import { + conditionEntriesToEntries, + entriesToConditionEntriesMap, +} from '../../../../common/utils/exception_list_items'; -type ConditionEntriesMap = { [K in ConditionEntryField]?: ConditionEntry }; type Mapping = { [K in T]: U }; const OS_TYPE_TO_OPERATING_SYSTEM: Mapping = { @@ -47,64 +42,10 @@ const OPERATING_SYSTEM_TO_OS_TYPE: Mapping = { [OperatingSystem.WINDOWS]: 'windows', }; -const OPERATOR_VALUE = 'included'; - const filterUndefined = (list: Array): T[] => { return list.filter((item: T | undefined): item is T => item !== undefined); }; -const createConditionEntry = ( - field: T, - type: TrustedAppEntryTypes, - value: string -): ConditionEntry => { - return { field, value, type, operator: OPERATOR_VALUE }; -}; - -const entriesToConditionEntriesMap = (entries: EntriesArray): ConditionEntriesMap => { - return entries.reduce((result, entry) => { - if (entry.field.startsWith('process.hash') && entry.type === 'match') { - return { - ...result, - [ConditionEntryField.HASH]: createConditionEntry( - ConditionEntryField.HASH, - entry.type, - entry.value - ), - }; - } else if ( - entry.field === 'process.executable.caseless' && - (entry.type === 'match' || entry.type === 'wildcard') - ) { - return { - ...result, - [ConditionEntryField.PATH]: createConditionEntry( - ConditionEntryField.PATH, - entry.type, - entry.value - ), - }; - } else if (entry.field === 'process.Ext.code_signature' && entry.type === 'nested') { - const subjectNameCondition = entry.entries.find((subEntry): subEntry is EntryMatch => { - return subEntry.field === 'subject_name' && subEntry.type === 'match'; - }); - - if (subjectNameCondition) { - return { - ...result, - [ConditionEntryField.SIGNER]: createConditionEntry( - ConditionEntryField.SIGNER, - subjectNameCondition.type, - subjectNameCondition.value - ), - }; - } - } - - return result; - }, {} as ConditionEntriesMap); -}; - /** * Map an ExceptionListItem to a TrustedApp item * @param exceptionListItem @@ -114,7 +55,19 @@ export const exceptionListItemToTrustedApp = ( ): TrustedApp => { if (exceptionListItem.os_types[0]) { const os = osFromExceptionItem(exceptionListItem); - const grouped = entriesToConditionEntriesMap(exceptionListItem.entries); + let groupedWin: ConditionEntriesMap = {}; + let groupedMacLinux: ConditionEntriesMap< + TrustedAppConditionEntry + > = {}; + if (os === OperatingSystem.WINDOWS) { + groupedWin = entriesToConditionEntriesMap( + exceptionListItem.entries + ); + } else { + groupedMacLinux = entriesToConditionEntriesMap< + TrustedAppConditionEntry + >(exceptionListItem.entries); + } return { id: exceptionListItem.item_id, @@ -129,17 +82,19 @@ export const exceptionListItemToTrustedApp = ( ...(os === OperatingSystem.LINUX || os === OperatingSystem.MAC ? { os, - entries: filterUndefined([ - grouped[ConditionEntryField.HASH], - grouped[ConditionEntryField.PATH], + entries: filterUndefined< + TrustedAppConditionEntry + >([ + groupedMacLinux[ConditionEntryField.HASH], + groupedMacLinux[ConditionEntryField.PATH], ]), } : { os, - entries: filterUndefined([ - grouped[ConditionEntryField.HASH], - grouped[ConditionEntryField.PATH], - grouped[ConditionEntryField.SIGNER], + entries: filterUndefined([ + groupedWin[ConditionEntryField.HASH], + groupedWin[ConditionEntryField.PATH], + groupedWin[ConditionEntryField.SIGNER], ]), }), }; @@ -152,29 +107,6 @@ const osFromExceptionItem = (exceptionListItem: ExceptionListItemSchema): Truste return OS_TYPE_TO_OPERATING_SYSTEM[exceptionListItem.os_types[0]]; }; -const hashType = (hash: string): 'md5' | 'sha256' | 'sha1' | undefined => { - switch (hash.length) { - case 32: - return 'md5'; - case 40: - return 'sha1'; - case 64: - return 'sha256'; - } -}; - -const createEntryMatch = (field: string, value: string): EntryMatch => { - return { field, value, type: 'match', operator: OPERATOR_VALUE }; -}; - -const createEntryMatchWildcard = (field: string, value: string): EntryMatchWildcard => { - return { field, value, type: 'wildcard', operator: OPERATOR_VALUE }; -}; - -const createEntryNested = (field: string, entries: NestedEntriesArray): EntryNested => { - return { field, entries, type: 'nested' }; -}; - const effectScopeToTags = (effectScope: EffectScope) => { if (effectScope.type === 'policy') { return effectScope.policies.map((policy) => `${BY_POLICY_ARTIFACT_TAG_PREFIX}${policy}`); @@ -183,29 +115,6 @@ const effectScopeToTags = (effectScope: EffectScope) => { } }; -const conditionEntriesToEntries = (conditionEntries: ConditionEntry[]): EntriesArray => { - return conditionEntries.map((conditionEntry) => { - if (conditionEntry.field === ConditionEntryField.HASH) { - return createEntryMatch( - `process.hash.${hashType(conditionEntry.value)}`, - conditionEntry.value.toLowerCase() - ); - } else if (conditionEntry.field === ConditionEntryField.SIGNER) { - return createEntryNested(`process.Ext.code_signature`, [ - createEntryMatch('trusted', 'true'), - createEntryMatch('subject_name', conditionEntry.value), - ]); - } else if ( - conditionEntry.field === ConditionEntryField.PATH && - conditionEntry.type === 'wildcard' - ) { - return createEntryMatchWildcard(`process.executable.caseless`, conditionEntry.value); - } else { - return createEntryMatch(`process.executable.caseless`, conditionEntry.value); - } - }); -}; - /** * Map NewTrustedApp to CreateExceptionListItemOptions. */ @@ -219,7 +128,7 @@ export const newTrustedAppToCreateExceptionListItem = ({ return { comments: [], description, - entries: conditionEntriesToEntries(entries), + entries: conditionEntriesToEntries(entries, true), list_id: ENDPOINT_TRUSTED_APPS_LIST_ID, meta: undefined, name, @@ -251,7 +160,7 @@ export const updatedTrustedAppToUpdateExceptionListItem = ( _version: version, name, description, - entries: conditionEntriesToEntries(entries), + entries: conditionEntriesToEntries(entries, true), os_types: [OPERATING_SYSTEM_TO_OS_TYPE[os]], tags: effectScopeToTags(effectScope), diff --git a/x-pack/plugins/security_solution/public/management/pages/trusted_apps/state/type_guards.ts b/x-pack/plugins/security_solution/public/management/pages/trusted_apps/state/type_guards.ts index 22aeedca7312c..1a28e6f3bfecf 100644 --- a/x-pack/plugins/security_solution/public/management/pages/trusted_apps/state/type_guards.ts +++ b/x-pack/plugins/security_solution/public/management/pages/trusted_apps/state/type_guards.ts @@ -7,7 +7,7 @@ import { ConditionEntryField } from '@kbn/securitysolution-utils'; import { - ConditionEntry, + TrustedAppConditionEntry, EffectScope, GlobalEffectScope, MacosLinuxConditionEntry, @@ -17,13 +17,13 @@ import { } from '../../../../../common/endpoint/types'; export const isWindowsTrustedAppCondition = ( - condition: ConditionEntry + condition: TrustedAppConditionEntry ): condition is WindowsConditionEntry => { return condition.field === ConditionEntryField.SIGNER || true; }; export const isMacosLinuxTrustedAppCondition = ( - condition: ConditionEntry + condition: TrustedAppConditionEntry ): condition is MacosLinuxConditionEntry => { return condition.field !== ConditionEntryField.SIGNER; }; diff --git a/x-pack/plugins/security_solution/public/management/pages/trusted_apps/store/builders.ts b/x-pack/plugins/security_solution/public/management/pages/trusted_apps/store/builders.ts index 431894274ee00..f0fab98188927 100644 --- a/x-pack/plugins/security_solution/public/management/pages/trusted_apps/store/builders.ts +++ b/x-pack/plugins/security_solution/public/management/pages/trusted_apps/store/builders.ts @@ -6,13 +6,13 @@ */ import { ConditionEntryField, OperatingSystem } from '@kbn/securitysolution-utils'; -import { ConditionEntry, NewTrustedApp } from '../../../../../common/endpoint/types'; +import { TrustedAppConditionEntry, NewTrustedApp } from '../../../../../common/endpoint/types'; import { MANAGEMENT_DEFAULT_PAGE, MANAGEMENT_DEFAULT_PAGE_SIZE } from '../../../common/constants'; import { TrustedAppsListPageState } from '../state'; -export const defaultConditionEntry = (): ConditionEntry => ({ +export const defaultConditionEntry = (): TrustedAppConditionEntry => ({ field: ConditionEntryField.HASH, operator: 'included', type: 'match', diff --git a/x-pack/plugins/security_solution/public/management/pages/trusted_apps/view/components/condition_entry_input/index.test.tsx b/x-pack/plugins/security_solution/public/management/pages/trusted_apps/view/components/condition_entry_input/index.test.tsx index 4ea42c896847c..8d20974249cce 100644 --- a/x-pack/plugins/security_solution/public/management/pages/trusted_apps/view/components/condition_entry_input/index.test.tsx +++ b/x-pack/plugins/security_solution/public/management/pages/trusted_apps/view/components/condition_entry_input/index.test.tsx @@ -9,7 +9,7 @@ import { shallow, mount } from 'enzyme'; import React from 'react'; import { keys } from 'lodash'; import { ConditionEntryField, OperatingSystem } from '@kbn/securitysolution-utils'; -import { ConditionEntry } from '../../../../../../../common/endpoint/types'; +import { TrustedAppConditionEntry } from '../../../../../../../common/endpoint/types'; import { ConditionEntryInput } from '.'; import { EuiSuperSelectProps } from '@elastic/eui'; @@ -18,7 +18,7 @@ let onRemoveMock: jest.Mock; let onChangeMock: jest.Mock; let onVisitedMock: jest.Mock; -const baseEntry: Readonly = { +const baseEntry: Readonly = { field: ConditionEntryField.HASH, type: 'match', operator: 'included', @@ -36,7 +36,7 @@ describe('Condition entry input', () => { subject: string, os: OperatingSystem = OperatingSystem.WINDOWS, isRemoveDisabled: boolean = false, - entry: ConditionEntry = baseEntry + entry: TrustedAppConditionEntry = baseEntry ) => ( void; - onChange: (newEntry: ConditionEntry, oldEntry: ConditionEntry) => void; + onRemove: (entry: TrustedAppConditionEntry) => void; + onChange: (newEntry: TrustedAppConditionEntry, oldEntry: TrustedAppConditionEntry) => void; /** * invoked when at least one field in the entry was visited (triggered when `onBlur` DOM event is dispatched) * For this component, that will be triggered only when the `value` field is visited, since that is the * only one needs user input. */ - onVisited?: (entry: ConditionEntry) => void; + onVisited?: (entry: TrustedAppConditionEntry) => void; 'data-test-subj'?: string; } diff --git a/x-pack/plugins/security_solution/public/management/pages/trusted_apps/view/components/condition_group/index.tsx b/x-pack/plugins/security_solution/public/management/pages/trusted_apps/view/components/condition_group/index.tsx index aed69128847f6..19c37498dacfc 100644 --- a/x-pack/plugins/security_solution/public/management/pages/trusted_apps/view/components/condition_group/index.tsx +++ b/x-pack/plugins/security_solution/public/management/pages/trusted_apps/view/components/condition_group/index.tsx @@ -10,7 +10,7 @@ import { EuiButton, EuiFlexGroup, EuiFlexItem, EuiHideFor, EuiSpacer } from '@el import styled from 'styled-components'; import { FormattedMessage } from '@kbn/i18n-react'; import { OperatingSystem } from '@kbn/securitysolution-utils'; -import { ConditionEntry } from '../../../../../../../common/endpoint/types'; +import { TrustedAppConditionEntry } from '../../../../../../../common/endpoint/types'; import { AndOrBadge } from '../../../../../../common/components/and_or_badge'; import { ConditionEntryInput, ConditionEntryInputProps } from '../condition_entry_input'; import { useTestIdGenerator } from '../../../../../components/hooks/use_test_id_generator'; @@ -45,7 +45,7 @@ const ConditionGroupFlexGroup = styled(EuiFlexGroup)` export interface ConditionGroupProps { os: OperatingSystem; - entries: ConditionEntry[]; + entries: TrustedAppConditionEntry[]; onEntryRemove: ConditionEntryInputProps['onRemove']; onEntryChange: ConditionEntryInputProps['onChange']; onAndClicked: () => void; @@ -82,7 +82,7 @@ export const ConditionGroup = memo( )}
- {(entries as ConditionEntry[]).map((entry, index) => ( + {(entries as TrustedAppConditionEntry[]).map((entry, index) => ( ): ValidationRe }) ); } else { - const duplicated = getDuplicateFields(values.entries as ConditionEntry[]); + const duplicated = getDuplicateFields(values.entries as TrustedAppConditionEntry[]); if (duplicated.length) { isValid = false; duplicated.forEach((field) => { diff --git a/x-pack/plugins/security_solution/public/management/services/exceptions_list/exceptions_list_api_client.ts b/x-pack/plugins/security_solution/public/management/services/exceptions_list/exceptions_list_api_client.ts index 81f9f20182bec..b8afc98b6f1e5 100644 --- a/x-pack/plugins/security_solution/public/management/services/exceptions_list/exceptions_list_api_client.ts +++ b/x-pack/plugins/security_solution/public/management/services/exceptions_list/exceptions_list_api_client.ts @@ -31,7 +31,13 @@ export class ExceptionsListApiClient { constructor( private readonly http: HttpStart, public readonly listId: ListId, - private readonly listDefinition: CreateExceptionListSchema + private readonly listDefinition: CreateExceptionListSchema, + private readonly readTransform?: (item: ExceptionListItemSchema) => ExceptionListItemSchema, + private readonly writeTransform?: < + T extends CreateExceptionListItemSchema | UpdateExceptionListItemSchema + >( + item: T + ) => T ) { this.ensureListExists = this.createExceptionList(); } @@ -94,7 +100,11 @@ export class ExceptionsListApiClient { public static getInstance( http: HttpStart, listId: string, - listDefinition: CreateExceptionListSchema + listDefinition: CreateExceptionListSchema, + readTransform?: (item: ExceptionListItemSchema) => ExceptionListItemSchema, + writeTransform?: ( + item: T + ) => T ): ExceptionsListApiClient { if ( !ExceptionsListApiClient.instance.has(listId) || @@ -102,14 +112,20 @@ export class ExceptionsListApiClient { ) { ExceptionsListApiClient.instance.set( listId, - new ExceptionsListApiClient(http, listId, listDefinition) + new ExceptionsListApiClient(http, listId, listDefinition, readTransform, writeTransform) ); } const currentInstance = ExceptionsListApiClient.instance.get(listId); if (currentInstance) { return currentInstance; } else { - return new ExceptionsListApiClient(http, listId, listDefinition); + return new ExceptionsListApiClient( + http, + listId, + listDefinition, + readTransform, + writeTransform + ); } } @@ -159,17 +175,26 @@ export class ExceptionsListApiClient { filter: string; }> = {}): Promise { await this.ensureListExists; - return this.http.get(`${EXCEPTION_LIST_ITEM_URL}/_find`, { - query: { - page, - per_page: perPage, - sort_field: sortField, - sort_order: sortOrder, - list_id: [this.listId], - namespace_type: ['agnostic'], - filter, - }, - }); + const result = await this.http.get( + `${EXCEPTION_LIST_ITEM_URL}/_find`, + { + query: { + page, + per_page: perPage, + sort_field: sortField, + sort_order: sortOrder, + list_id: [this.listId], + namespace_type: ['agnostic'], + filter, + }, + } + ); + + if (this.readTransform) { + result.data = result.data.map(this.readTransform); + } + + return result; } /** @@ -182,13 +207,19 @@ export class ExceptionsListApiClient { } await this.ensureListExists; - return this.http.get(EXCEPTION_LIST_ITEM_URL, { + let result = await this.http.get(EXCEPTION_LIST_ITEM_URL, { query: { id, item_id: itemId, namespace_type: 'agnostic', }, }); + + if (this.readTransform) { + result = this.readTransform(result); + } + + return result; } /** @@ -199,8 +230,14 @@ export class ExceptionsListApiClient { await this.ensureListExists; this.checkIfIsUsingTheRightInstance(exception.list_id); delete exception.meta; + + let transformedException = exception; + if (this.writeTransform) { + transformedException = this.writeTransform(exception); + } + return this.http.post(EXCEPTION_LIST_ITEM_URL, { - body: JSON.stringify(exception), + body: JSON.stringify(transformedException), }); } @@ -210,8 +247,16 @@ export class ExceptionsListApiClient { */ async update(exception: UpdateExceptionListItemSchema): Promise { await this.ensureListExists; + + let transformedException = exception; + if (this.writeTransform) { + transformedException = this.writeTransform(exception); + } + return this.http.put(EXCEPTION_LIST_ITEM_URL, { - body: JSON.stringify(ExceptionsListApiClient.cleanExceptionsBeforeUpdate(exception)), + body: JSON.stringify( + ExceptionsListApiClient.cleanExceptionsBeforeUpdate(transformedException) + ), }); } diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts index 7a36e2ef940e5..7521ccbf9df91 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts @@ -188,7 +188,7 @@ function getMatcherFunction({ os: ExceptionListItemSchema['os_types'][number]; }): TranslatedEntryMatcher { return matchAny - ? field.endsWith('.caseless') + ? field.endsWith('.caseless') && os !== 'linux' ? 'exact_caseless_any' : 'exact_cased_any' : field.endsWith('.caseless') diff --git a/x-pack/plugins/security_solution/server/lists_integration/endpoint/validators/trusted_app_validator.ts b/x-pack/plugins/security_solution/server/lists_integration/endpoint/validators/trusted_app_validator.ts index fc69153f0b21b..dc539e76e7946 100644 --- a/x-pack/plugins/security_solution/server/lists_integration/endpoint/validators/trusted_app_validator.ts +++ b/x-pack/plugins/security_solution/server/lists_integration/endpoint/validators/trusted_app_validator.ts @@ -15,7 +15,7 @@ import { CreateExceptionListItemOptions, UpdateExceptionListItemOptions, } from '../../../../../lists/server'; -import { ConditionEntry } from '../../../../common/endpoint/types'; +import { TrustedAppConditionEntry as ConditionEntry } from '../../../../common/endpoint/types'; import { getDuplicateFields, isValidHash, From c331caec0ac522982f0d62fd05b72423d575ab95 Mon Sep 17 00:00:00 2001 From: Marshall Main <55718608+marshallmain@users.noreply.github.com> Date: Wed, 16 Mar 2022 10:03:23 -0700 Subject: [PATCH 3/3] [Security Solution][Alerts] Update EQL rules to use EQL method of ES client (#127684) * Update EQL rules to use EQL method of ES client * Remove completed TODO comment * Fix EQL executor unit test * Fix buildEqlSearchRequest unit test Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../detection_engine/get_query_filter.test.ts | 12 +++---- .../detection_engine/get_query_filter.ts | 32 +++++++++---------- .../signals/executors/eql.test.ts | 4 +-- .../detection_engine/signals/executors/eql.ts | 13 +++----- .../lib/detection_engine/signals/types.ts | 3 -- .../tests/generating_signals.ts | 12 +++++++ 6 files changed, 39 insertions(+), 37 deletions(-) diff --git a/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.test.ts b/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.test.ts index 61e1ebb47464b..a97a13b8aba38 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.test.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.test.ts @@ -1125,8 +1125,8 @@ describe('get_filter', () => { undefined ); expect(request).toEqual({ - method: 'POST', - path: `/testindex1,testindex2/_eql/search?allow_no_indices=true`, + allow_no_indices: true, + index: ['testindex1', 'testindex2'], body: { size: 100, query: 'process where true', @@ -1171,8 +1171,8 @@ describe('get_filter', () => { 'event.other_category' ); expect(request).toEqual({ - method: 'POST', - path: `/testindex1,testindex2/_eql/search?allow_no_indices=true`, + allow_no_indices: true, + index: ['testindex1', 'testindex2'], body: { event_category_field: 'event.other_category', size: 100, @@ -1222,8 +1222,8 @@ describe('get_filter', () => { undefined ); expect(request).toEqual({ - method: 'POST', - path: `/testindex1,testindex2/_eql/search?allow_no_indices=true`, + allow_no_indices: true, + index: ['testindex1', 'testindex2'], body: { size: 100, query: 'process where true', diff --git a/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.ts b/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.ts index 42c10614975eb..e5fd1e8766ef7 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.ts @@ -12,6 +12,10 @@ import type { } from '@kbn/securitysolution-io-ts-list-types'; import { buildExceptionFilter } from '@kbn/securitysolution-list-utils'; import { Filter, EsQueryConfig, DataViewBase, buildEsQuery } from '@kbn/es-query'; +import { + EqlSearchRequest, + QueryDslQueryContainer, +} from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { ESBoolQuery } from '../typed_json'; import { Query, Index, TimestampOverrideOrUndefined } from './schemas/common/schemas'; @@ -58,12 +62,6 @@ export const getAllFilters = (filters: Filter[], exceptionFilter: Filter | undef } }; -interface EqlSearchRequest { - method: string; - path: string; - body: object; -} - export const buildEqlSearchRequest = ( query: string, index: string[], @@ -93,8 +91,7 @@ export const buildEqlSearchRequest = ( excludeExceptions: true, chunkSize: 1024, }); - const indexString = index.join(); - const requestFilter: unknown[] = [ + const requestFilter: QueryDslQueryContainer[] = [ { range: { [timestamp]: { @@ -114,9 +111,16 @@ export const buildEqlSearchRequest = ( }, }); } + const fields = [ + { + field: '*', + include_unmapped: true, + }, + ...docFields, + ]; return { - method: 'POST', - path: `/${indexString}/_eql/search?allow_no_indices=true`, + index, + allow_no_indices: true, body: { size, query, @@ -126,13 +130,7 @@ export const buildEqlSearchRequest = ( }, }, event_category_field: eventCategoryOverride, - fields: [ - { - field: '*', - include_unmapped: true, - }, - ...docFields, - ], + fields, }, }; }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/eql.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/eql.test.ts index 0d7ac2cea493a..eba22837cda5a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/eql.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/eql.test.ts @@ -36,9 +36,9 @@ describe('eql_executor', () => { beforeEach(() => { alertServices = alertsMock.createAlertServices(); logger = loggingSystemMock.createLogger(); - alertServices.scopedClusterClient.asCurrentUser.transport.request.mockResolvedValue({ + alertServices.scopedClusterClient.asCurrentUser.eql.search.mockResolvedValue({ hits: { - total: { value: 10 }, + total: { relation: 'eq', value: 10 }, events: [], }, }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/eql.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/eql.ts index ac44d1d6470a6..79a73e7c07ec0 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/eql.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/eql.ts @@ -24,10 +24,10 @@ import { BulkCreate, WrapHits, WrapSequences, - EqlSignalSearchResponse, RuleRangeTuple, SearchAfterAndBulkCreateReturnType, SimpleHit, + SignalSource, } from '../types'; import { createSearchAfterReturnType, makeFloatString } from '../utils'; import { ExperimentalFeatures } from '../../../../../common/experimental_features'; @@ -110,16 +110,11 @@ export const eqlExecutor = async ({ ); const eqlSignalSearchStart = performance.now(); - logger.debug( - `EQL query request path: ${request.path}, method: ${request.method}, body: ${JSON.stringify( - request.body - )}` - ); + logger.debug(`EQL query request: ${JSON.stringify(request)}`); - // TODO: fix this later - const response = (await services.scopedClusterClient.asCurrentUser.transport.request( + const response = await services.scopedClusterClient.asCurrentUser.eql.search( request - )) as EqlSignalSearchResponse; + ); const eqlSignalSearchEnd = performance.now(); const eqlSearchDuration = makeFloatString(eqlSignalSearchEnd - eqlSignalSearchStart); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts index 37ed4a78a61a6..a5803dc354040 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts @@ -21,7 +21,6 @@ import { } from '../../../../../alerting/server'; import { TermAggregationBucket } from '../../types'; import { - EqlSearchResponse, BaseHit, RuleAlertAction, SearchTypes, @@ -188,8 +187,6 @@ export type AlertSourceHit = estypes.SearchHit; export type WrappedSignalHit = BaseHit; export type BaseSignalHit = estypes.SearchHit; -export type EqlSignalSearchResponse = EqlSearchResponse; - export type RuleExecutorOptions = AlertExecutorOptions< RuleParams, AlertTypeState, diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/generating_signals.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/generating_signals.ts index 11c5af219de4c..197ea15947ab3 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/generating_signals.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/generating_signals.ts @@ -692,6 +692,18 @@ export default ({ getService }: FtrProviderContext) => { expect(shellSignals.length).eql(100); expect(buildingBlocks.length).eql(200); }); + + it('generates signals when an index name contains special characters to encode', async () => { + const rule: EqlCreateSchema = { + ...getEqlRuleForSignalTesting(['auditbeat-*', '']), + query: 'configuration where agent.id=="a1d7b39c-f898-4dbe-a761-efb61939302d"', + }; + const { id } = await createRule(supertest, log, rule); + await waitForRuleSuccessOrStatus(supertest, log, id); + await waitForSignalsToBePresent(supertest, log, 1, [id]); + const signals = await getSignalsByIds(supertest, log, [id]); + expect(signals.hits.hits.length).eql(1); + }); }); describe('Threshold Rules', () => {