diff --git a/x-pack/plugins/security_solution/common/endpoint/types/workflow_insights.ts b/x-pack/plugins/security_solution/common/endpoint/types/workflow_insights.ts new file mode 100644 index 0000000000000..11cbc1bfd7cd8 --- /dev/null +++ b/x-pack/plugins/security_solution/common/endpoint/types/workflow_insights.ts @@ -0,0 +1,78 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Moment } from 'moment'; + +import type { DefendInsightType } from '@kbn/elastic-assistant-common'; +import type { ExceptionListItemSchema } from '@kbn/securitysolution-io-ts-list-types'; + +export enum Category { + Endpoint = 'endpoint', +} + +export enum SourceType { + LlmConnector = 'llm-connector', +} + +export enum TargetType { + Endpoint = 'endpoint', +} + +export enum ActionType { + Refreshed = 'refreshed', // new or refreshed + Remediated = 'remediated', + Suppressed = 'suppressed', // temporarily supressed, can be refreshed + Dismissed = 'dismissed', // "permanently" dismissed, cannot be normally refreshed +} + +export type ExceptionListRemediationType = Pick< + ExceptionListItemSchema, + 'list_id' | 'name' | 'description' | 'entries' | 'tags' | 'os_types' +>; + +export interface SecurityWorkflowInsight { + id?: string; + '@timestamp': Moment; + message: string; + category: Category; + type: DefendInsightType; + source: { + type: SourceType; + id: string; + data_range_start: Moment; + data_range_end: Moment; + }; + target: { + type: TargetType; + ids: string[]; + }; + action: { + type: ActionType; + timestamp: Moment; + }; + value: string; + remediation: { + exception_list_items?: ExceptionListRemediationType[]; + }; + metadata: { + notes?: Record; + message_variables?: string[]; + }; +} + +export interface SearchParams { + size?: number; + from?: number; + ids?: string[]; + categories?: Category[]; + types?: DefendInsightType[]; + sourceTypes?: SourceType[]; + sourceIds?: string[]; + targetTypes?: TargetType[]; + targetIds?: string[]; + actionTypes: ActionType[]; +} diff --git a/x-pack/plugins/security_solution/server/endpoint/services/workflow_insights/constants.ts b/x-pack/plugins/security_solution/server/endpoint/services/workflow_insights/constants.ts index f0884f2214cb8..f8b97932289f5 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/workflow_insights/constants.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/workflow_insights/constants.ts @@ -5,7 +5,7 @@ * 2.0. */ -export const DATA_STREAM_PREFIX = '.security-workflow-insights'; +export const DATA_STREAM_PREFIX = '.edr-workflow-insights'; export const COMPONENT_TEMPLATE_NAME = `${DATA_STREAM_PREFIX}-component-template`; export const INDEX_TEMPLATE_NAME = `${DATA_STREAM_PREFIX}-index-template`; export const INGEST_PIPELINE_NAME = `${DATA_STREAM_PREFIX}-ingest-pipeline`; diff --git a/x-pack/plugins/security_solution/server/endpoint/services/workflow_insights/helpers.test.ts b/x-pack/plugins/security_solution/server/endpoint/services/workflow_insights/helpers.test.ts index 33f1851091167..119c1848f6a1a 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/workflow_insights/helpers.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/workflow_insights/helpers.test.ts @@ -6,11 +6,15 @@ */ import type { ElasticsearchClient } from '@kbn/core/server'; -import { elasticsearchServiceMock } from '@kbn/core-elasticsearch-server-mocks'; + import { DataStreamSpacesAdapter } from '@kbn/data-stream-adapter'; +import { DefendInsightType } from '@kbn/elastic-assistant-common'; +import { elasticsearchServiceMock } from '@kbn/core-elasticsearch-server-mocks'; import { kibanaPackageJson } from '@kbn/repo-info'; -import { createDatastream, createPipeline } from './helpers'; +import type { SearchParams } from '../../../../common/endpoint/types/workflow_insights'; + +import { buildEsQueryParams, createDatastream, createPipeline } from './helpers'; import { DATA_STREAM_PREFIX, COMPONENT_TEMPLATE_NAME, @@ -19,6 +23,12 @@ import { TOTAL_FIELDS_LIMIT, } from './constants'; import { securityWorkflowInsightsFieldMap } from './field_map_configurations'; +import { + ActionType, + Category, + SourceType, + TargetType, +} from '../../../../common/endpoint/types/workflow_insights'; jest.mock('@kbn/data-stream-adapter', () => ({ DataStreamSpacesAdapter: jest.fn().mockImplementation(() => ({ @@ -77,4 +87,64 @@ describe('helpers', () => { }); }); }); + + describe('buildEsQueryParams', () => { + it('should build es query correct', () => { + const searchParams: SearchParams = { + size: 50, + from: 50, + ids: ['id1', 'id2'], + categories: [Category.Endpoint], + types: [DefendInsightType.Enum.incompatible_antivirus], + sourceTypes: [SourceType.LlmConnector], + sourceIds: ['source-id1', 'source-id2'], + targetTypes: [TargetType.Endpoint], + targetIds: ['target-id1', 'target-id2'], + actionTypes: [ActionType.Refreshed, ActionType.Remediated], + }; + const result = buildEsQueryParams(searchParams); + expect(result).toEqual([ + { + terms: { + _id: ['id1', 'id2'], + }, + }, + { + terms: { + categories: ['endpoint'], + }, + }, + { + terms: { + types: ['incompatible_antivirus'], + }, + }, + { + terms: { + 'source.type': ['llm-connector'], + }, + }, + { + terms: { + 'source.id': ['source-id1', 'source-id2'], + }, + }, + { + terms: { + 'target.type': ['endpoint'], + }, + }, + { + terms: { + 'target.id': ['target-id1', 'target-id2'], + }, + }, + { + terms: { + 'action.type': ['refreshed', 'remediated'], + }, + }, + ]); + }); + }); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/services/workflow_insights/helpers.ts b/x-pack/plugins/security_solution/server/endpoint/services/workflow_insights/helpers.ts index 54b449edf86ff..f0057faed6aa6 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/workflow_insights/helpers.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/workflow_insights/helpers.ts @@ -5,10 +5,15 @@ * 2.0. */ +import { get as _get } from 'lodash'; + +import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types'; import type { ElasticsearchClient } from '@kbn/core/server'; import { DataStreamSpacesAdapter } from '@kbn/data-stream-adapter'; +import type { SearchParams } from '../../../../common/endpoint/types/workflow_insights'; + import { COMPONENT_TEMPLATE_NAME, DATA_STREAM_PREFIX, @@ -60,3 +65,34 @@ export async function createPipeline(esClient: ElasticsearchClient): Promise { + if (!validKeys.has(k)) { + return acc; + } + + const paramKey = _get(paramFieldMap, k, k); + const next = { terms: { [paramKey]: v } }; + + return [...acc, next]; + }, []); +} diff --git a/x-pack/plugins/security_solution/server/endpoint/services/workflow_insights/index.test.ts b/x-pack/plugins/security_solution/server/endpoint/services/workflow_insights/index.test.ts index 6271bd780dedd..792a7a9ecd949 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/workflow_insights/index.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/workflow_insights/index.test.ts @@ -5,22 +5,92 @@ * 2.0. */ +import { merge } from 'lodash'; +import moment from 'moment'; import { ReplaySubject } from 'rxjs'; import type { ElasticsearchClient, Logger } from '@kbn/core/server'; -import { elasticsearchServiceMock } from '@kbn/core-elasticsearch-server-mocks'; + import { DataStreamSpacesAdapter } from '@kbn/data-stream-adapter'; -import { loggerMock } from '@kbn/logging-mocks'; +import { DefendInsightType } from '@kbn/elastic-assistant-common'; +import { elasticsearchServiceMock } from '@kbn/core-elasticsearch-server-mocks'; import { kibanaPackageJson } from '@kbn/repo-info'; +import { loggerMock } from '@kbn/logging-mocks'; + +import type { + SearchParams, + SecurityWorkflowInsight, +} from '../../../../common/endpoint/types/workflow_insights'; +import { + Category, + SourceType, + TargetType, + ActionType, +} from '../../../../common/endpoint/types/workflow_insights'; import { createDatastream, createPipeline } from './helpers'; import { securityWorkflowInsightsService } from '.'; import { DATA_STREAM_NAME } from './constants'; -jest.mock('./helpers', () => ({ - createDatastream: jest.fn(), - createPipeline: jest.fn(), -})); +jest.mock('./helpers', () => { + const original = jest.requireActual('./helpers'); + return { + ...original, + createDatastream: jest.fn(), + createPipeline: jest.fn(), + }; +}); + +function getDefaultInsight(overrides?: Partial): SecurityWorkflowInsight { + const defaultInsight = { + '@timestamp': moment(), + message: 'This is a test message', + category: Category.Endpoint, + type: DefendInsightType.Enum.incompatible_antivirus, + source: { + type: SourceType.LlmConnector, + id: 'openai-connector-id', + data_range_start: moment(), + data_range_end: moment(), + }, + target: { + type: TargetType.Endpoint, + ids: ['endpoint-1', 'endpoint-2'], + }, + action: { + type: ActionType.Refreshed, + timestamp: moment(), + }, + value: 'unique-key', + remediation: { + exception_list_items: [ + { + list_id: 'example-list-id', + name: 'Example List Name', + description: 'Example description', + entries: [ + { + field: 'example-field', + operator: 'included', + type: 'match', + value: 'example-value', + }, + ], + tags: ['example-tag'], + os_types: ['windows', 'linux'], + }, + ], + }, + metadata: { + notes: { + key1: 'value1', + key2: 'value2', + }, + message_variables: ['variable1', 'variable2'], + }, + }; + return merge(defaultInsight, overrides); +} describe('SecurityWorkflowInsightsService', () => { let logger: Logger; @@ -126,38 +196,127 @@ describe('SecurityWorkflowInsightsService', () => { }); describe('create', () => { - it('should wait for initialization', async () => { + it('should index the doc correctly', async () => { const isInitializedSpy = jest .spyOn(securityWorkflowInsightsService, 'isInitialized', 'get') .mockResolvedValueOnce([undefined, undefined]); - await securityWorkflowInsightsService.create(); + await securityWorkflowInsightsService.start({ esClient }); + const insight = getDefaultInsight(); + await securityWorkflowInsightsService.create(insight); + // ensure it waits for initialization first expect(isInitializedSpy).toHaveBeenCalledTimes(1); + // indexes the doc + expect(esClient.index).toHaveBeenCalledTimes(1); + expect(esClient.index).toHaveBeenCalledWith({ + index: DATA_STREAM_NAME, + body: insight, + }); }); }); describe('update', () => { - it('should wait for initialization', async () => { + it('should update the doc correctly', async () => { const isInitializedSpy = jest .spyOn(securityWorkflowInsightsService, 'isInitialized', 'get') .mockResolvedValueOnce([undefined, undefined]); - await securityWorkflowInsightsService.update(); + await securityWorkflowInsightsService.start({ esClient }); + const insightId = 'some-insight-id'; + const insight = getDefaultInsight(); + await securityWorkflowInsightsService.update(insightId, insight); + // ensure it waits for initialization first expect(isInitializedSpy).toHaveBeenCalledTimes(1); + // updates the doc + expect(esClient.update).toHaveBeenCalledTimes(1); + expect(esClient.update).toHaveBeenCalledWith({ + index: DATA_STREAM_NAME, + id: insightId, + body: { doc: insight }, + }); }); }); describe('fetch', () => { - it('should wait for initialization', async () => { + it('should fetch the docs with the correct params', async () => { const isInitializedSpy = jest .spyOn(securityWorkflowInsightsService, 'isInitialized', 'get') .mockResolvedValueOnce([undefined, undefined]); - await securityWorkflowInsightsService.fetch(); + await securityWorkflowInsightsService.start({ esClient }); + const searchParams: SearchParams = { + size: 50, + from: 50, + ids: ['id1', 'id2'], + categories: [Category.Endpoint], + types: [DefendInsightType.Enum.incompatible_antivirus], + sourceTypes: [SourceType.LlmConnector], + sourceIds: ['source-id1', 'source-id2'], + targetTypes: [TargetType.Endpoint], + targetIds: ['target-id1', 'target-id2'], + actionTypes: [ActionType.Refreshed, ActionType.Remediated], + }; + await securityWorkflowInsightsService.fetch(searchParams); + // ensure it waits for initialization first expect(isInitializedSpy).toHaveBeenCalledTimes(1); + // fetches the doc + expect(esClient.search).toHaveBeenCalledTimes(1); + expect(esClient.search).toHaveBeenCalledWith({ + index: DATA_STREAM_NAME, + body: { + query: { + bool: { + must: [ + { + terms: { + _id: ['id1', 'id2'], + }, + }, + { + terms: { + categories: ['endpoint'], + }, + }, + { + terms: { + types: ['incompatible_antivirus'], + }, + }, + { + terms: { + 'source.type': ['llm-connector'], + }, + }, + { + terms: { + 'source.id': ['source-id1', 'source-id2'], + }, + }, + { + terms: { + 'target.type': ['endpoint'], + }, + }, + { + terms: { + 'target.id': ['target-id1', 'target-id2'], + }, + }, + { + terms: { + 'action.type': ['refreshed', 'remediated'], + }, + }, + ], + }, + }, + size: searchParams.size, + from: searchParams.from, + }, + }); }); }); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/services/workflow_insights/index.ts b/x-pack/plugins/security_solution/server/endpoint/services/workflow_insights/index.ts index 005be1b0398e1..0aa495dac0931 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/workflow_insights/index.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/workflow_insights/index.ts @@ -7,14 +7,25 @@ import { ReplaySubject, firstValueFrom, combineLatest } from 'rxjs'; +import type { + SearchHit, + UpdateResponse, + WriteResponseBase, +} from '@elastic/elasticsearch/lib/api/types'; import type { ElasticsearchClient, Logger } from '@kbn/core/server'; - import type { DataStreamSpacesAdapter } from '@kbn/data-stream-adapter'; +import type { + SearchParams, + SecurityWorkflowInsight, +} from '../../../../common/endpoint/types/workflow_insights'; + import { SecurityWorkflowInsightsFailedInitialized } from './errors'; -import { createDatastream, createPipeline } from './helpers'; +import { buildEsQueryParams, createDatastream, createPipeline } from './helpers'; import { DATA_STREAM_NAME } from './constants'; +const DEFAULT_PAGE_SIZE = 10; + interface SetupInterface { kibanaVersion: string; logger: Logger; @@ -30,7 +41,7 @@ class SecurityWorkflowInsightsService { private start$ = new ReplaySubject(1); private stop$ = new ReplaySubject(1); private ds: DataStreamSpacesAdapter | undefined; - // private _esClient: ElasticsearchClient | undefined; + private _esClient: ElasticsearchClient | undefined; private _logger: Logger | undefined; private _isInitialized: Promise<[void, void]> = firstValueFrom( combineLatest<[void, void]>([this.setup$, this.start$]) @@ -64,7 +75,7 @@ class SecurityWorkflowInsightsService { return; } - // this._esClient = esClient; + this._esClient = esClient; await firstValueFrom(this.setup$); try { @@ -97,26 +108,62 @@ class SecurityWorkflowInsightsService { this.stop$.complete(); } - public async create() { + public async create(insight: SecurityWorkflowInsight): Promise { await this.isInitialized; + + const response = await this.esClient.index({ + index: DATA_STREAM_NAME, + body: insight, + }); + + return response; } - public async update() { + public async update( + id: string, + insight: Partial + ): Promise { await this.isInitialized; + + const response = await this.esClient.update({ + index: DATA_STREAM_NAME, + id, + body: { doc: insight }, + }); + + return response; } - public async fetch() { + public async fetch(params?: SearchParams): Promise>> { await this.isInitialized; + + const size = params?.size ?? DEFAULT_PAGE_SIZE; + const from = params?.from ?? 0; + + const termFilters = params ? buildEsQueryParams(params) : []; + const response = await this.esClient.search({ + index: DATA_STREAM_NAME, + body: { + query: { + bool: { + must: termFilters, + }, + }, + size, + from, + }, + }); + + return response?.hits?.hits ?? []; } - // to be used in create/update/fetch above - // private get esClient(): ElasticsearchClient { - // if (!this._esClient) { - // throw new SecurityWorkflowInsightsFailedInitialized('no elasticsearch client found'); - // } + private get esClient(): ElasticsearchClient { + if (!this._esClient) { + throw new SecurityWorkflowInsightsFailedInitialized('no elasticsearch client found'); + } - // return this._esClient; - // } + return this._esClient; + } private get logger(): Logger { if (!this._logger) {