From e70d96bf97f32b8aea590ea166e273d5eb09a7fc Mon Sep 17 00:00:00 2001 From: Paul Tavares <56442535+paul-tavares@users.noreply.github.com> Date: Fri, 25 Oct 2024 15:18:12 -0400 Subject: [PATCH] [Security Solution][Endpoint] Ensure that DS indices for response actions are created prior to sending action to Endpoint (#196953) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary PR adds changes to Security Solution so that DOT indices (restricted in Serverless) are created in Kibana prior to Elastic Defend (Endpoint) attempting to stream documents to these indices. The indices that are now created in kibana are: - `.logs-endpoint.diagnostic.collection-` - `.logs-endpoint.action.responses-` - `.logs-endpoint.heartbeat-` _(⚠️ created only in serverless only)_ ### Fleet changes: - Added support for the following two server-side extension points: - `packagePolicyPostUpdate` : callbacks invoked after an integration policy has been updated successfully - `agentPolicyPostUpdate` : callbacks invoked after an agent policy has been updated successfully ### Security Solution: - Logic was added to the following Fleet server-side extension points that checks if the necessary indices exist and if not, it creates them: - After creating an Elastic Defend integration policy - After updating an Elastic Defend integration policy - After updating a Fleet Agent Policy that includes Elastic Defend integration policy (cherry picked from commit ae9c0d385015f3068a04af46678e18e2f00b519a) # Conflicts: # x-pack/plugins/fleet/server/services/agent_policy.ts --- x-pack/plugins/fleet/server/mocks/index.ts | 16 +- .../server/services/agent_policy.test.ts | 32 +++ .../fleet/server/services/agent_policy.ts | 16 +- .../server/services/package_policy.test.ts | 35 +++ .../fleet/server/services/package_policy.ts | 90 ++++---- .../server/services/package_policy_service.ts | 52 +++-- .../plugins/fleet/server/types/extensions.ts | 23 +- .../endpoint/endpoint_app_context_services.ts | 35 ++- .../clients => }/lib/simple_mem_cache.test.ts | 7 + .../clients => }/lib/simple_mem_cache.ts | 6 + .../server/endpoint/mocks/mocks.ts | 2 + .../lib/base_response_actions_client.ts | 8 +- .../endpoint_fleet_services_factory.mocks.ts | 22 +- .../endpoint_fleet_services_factory.test.ts | 181 +++++++++++++++ .../fleet/endpoint_fleet_services_factory.ts | 216 +++++++++++++++--- .../fleet_integration.test.ts | 54 ++++- .../fleet_integration/fleet_integration.ts | 91 +++++++- .../create_policy_datastreams.test.ts | 119 ++++++++++ .../handlers/create_policy_datastreams.ts | 159 +++++++++++++ 19 files changed, 1028 insertions(+), 136 deletions(-) rename x-pack/plugins/security_solution/server/endpoint/{services/actions/clients => }/lib/simple_mem_cache.test.ts (92%) rename x-pack/plugins/security_solution/server/endpoint/{services/actions/clients => }/lib/simple_mem_cache.ts (95%) create mode 100644 x-pack/plugins/security_solution/server/endpoint/services/fleet/endpoint_fleet_services_factory.test.ts create mode 100644 x-pack/plugins/security_solution/server/fleet_integration/handlers/create_policy_datastreams.test.ts create mode 100644 x-pack/plugins/security_solution/server/fleet_integration/handlers/create_policy_datastreams.ts diff --git a/x-pack/plugins/fleet/server/mocks/index.ts b/x-pack/plugins/fleet/server/mocks/index.ts index 43b113899072..ac806c1448a2 100644 --- a/x-pack/plugins/fleet/server/mocks/index.ts +++ b/x-pack/plugins/fleet/server/mocks/index.ts @@ -186,7 +186,7 @@ export const createPackagePolicyServiceMock = (): jest.Mocked => { return { - get: jest.fn(), - list: jest.fn(), - getFullAgentPolicy: jest.fn(), - getByIds: jest.fn(), - turnOffAgentTamperProtections: jest.fn(), - fetchAllAgentPolicies: jest.fn(), - fetchAllAgentPolicyIds: jest.fn(), + get: jest.fn().mockReturnValue(Promise.resolve()), + list: jest.fn().mockReturnValue(Promise.resolve()), + getFullAgentPolicy: jest.fn().mockReturnValue(Promise.resolve()), + getByIds: jest.fn().mockReturnValue(Promise.resolve()), + turnOffAgentTamperProtections: jest.fn().mockReturnValue(Promise.resolve()), + fetchAllAgentPolicies: jest.fn().mockReturnValue(Promise.resolve()), + fetchAllAgentPolicyIds: jest.fn().mockReturnValue(Promise.resolve()), }; }; diff --git a/x-pack/plugins/fleet/server/services/agent_policy.test.ts b/x-pack/plugins/fleet/server/services/agent_policy.test.ts index 00bc01aa1f2c..608b6f739fc2 100644 --- a/x-pack/plugins/fleet/server/services/agent_policy.test.ts +++ b/x-pack/plugins/fleet/server/services/agent_policy.test.ts @@ -1285,6 +1285,38 @@ describe('Agent policy', () => { }) ).resolves.not.toThrow(); }); + + it('should run external "agentPolicyPostUpdate" callbacks when update is successful', async () => { + const soClient = getAgentPolicyCreateMock(); + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + + const postUpdateCallback = jest.fn(async (policy) => policy); + mockedAppContextService.getExternalCallbacks.mockImplementation((type) => { + if (type === 'agentPolicyPostUpdate') { + return new Set([postUpdateCallback]); + } + }); + + soClient.get.mockResolvedValue({ + attributes: {}, + id: 'test-id', + type: 'mocked', + references: [], + }); + + await expect( + agentPolicyService.update(soClient, esClient, 'test-id', { + name: 'test', + namespace: 'default', + }) + ).resolves.not.toThrow(); + + expect(mockedAppContextService.getExternalCallbacks).toHaveBeenCalledWith( + 'agentPolicyPostUpdate' + ); + + expect(postUpdateCallback).toHaveBeenCalled(); + }); }); describe('deployPolicy', () => { diff --git a/x-pack/plugins/fleet/server/services/agent_policy.ts b/x-pack/plugins/fleet/server/services/agent_policy.ts index f377f95190df..1493f55cf690 100644 --- a/x-pack/plugins/fleet/server/services/agent_policy.ts +++ b/x-pack/plugins/fleet/server/services/agent_policy.ts @@ -61,6 +61,7 @@ import type { PostAgentPolicyCreateCallback, PostAgentPolicyUpdateCallback, PreconfiguredAgentPolicy, + PostAgentPolicyPostUpdateCallback, } from '../types'; import { AGENT_POLICY_INDEX, @@ -306,8 +307,8 @@ class AgentPolicyService { public async runExternalCallbacks( externalCallbackType: ExternalCallback[0], - agentPolicy: NewAgentPolicy | Partial - ): Promise> { + agentPolicy: NewAgentPolicy | Partial | AgentPolicy + ): Promise | AgentPolicy> { const logger = appContextService.getLogger(); logger.debug(`Running external callbacks for ${externalCallbackType}`); try { @@ -330,6 +331,12 @@ class AgentPolicyService { ); updatedNewAgentPolicy = result; } + if (externalCallbackType === 'agentPolicyPostUpdate') { + result = await (callback as PostAgentPolicyPostUpdateCallback)( + newAgentPolicy as AgentPolicy + ); + updatedNewAgentPolicy = result; + } } newAgentPolicy = updatedNewAgentPolicy; } @@ -738,6 +745,11 @@ class AgentPolicyService { bumpRevision: true, removeProtection: false, skipValidation: options?.skipValidation ?? false, + }).then((updatedAgentPolicy) => { + return this.runExternalCallbacks( + 'agentPolicyPostUpdate', + updatedAgentPolicy + ) as unknown as AgentPolicy; }); } diff --git a/x-pack/plugins/fleet/server/services/package_policy.test.ts b/x-pack/plugins/fleet/server/services/package_policy.test.ts index 8c322d997a61..30523448e721 100644 --- a/x-pack/plugins/fleet/server/services/package_policy.test.ts +++ b/x-pack/plugins/fleet/server/services/package_policy.test.ts @@ -1718,6 +1718,41 @@ describe('Package policy service', () => { }); }); + it('should run "packagePolicyPostUpdate" external callbacks', async () => { + const soClient = savedObjectsClientMock.create(); + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + const mockPackagePolicy = createPackagePolicyMock(); + const attributes = { + ...mockPackagePolicy, + inputs: [], + }; + + jest.spyOn(appContextService, 'getExternalCallbacks'); + + soClient.get.mockResolvedValue({ + id: 'test-package-policy', + type: LEGACY_PACKAGE_POLICY_SAVED_OBJECT_TYPE, + references: [], + attributes, + }); + + soClient.update.mockResolvedValue({ + id: 'test-package-policy', + type: LEGACY_PACKAGE_POLICY_SAVED_OBJECT_TYPE, + references: [], + attributes, + }); + + await packagePolicyService.update(soClient, esClient, 'test-package-policy', { + ...mockPackagePolicy, + inputs: [], + }); + + expect(appContextService.getExternalCallbacks).toHaveBeenCalledWith( + 'packagePolicyPostUpdate' + ); + }); + describe('remove protections', () => { beforeEach(() => { mockAgentPolicyService.bumpRevision.mockReset(); diff --git a/x-pack/plugins/fleet/server/services/package_policy.ts b/x-pack/plugins/fleet/server/services/package_policy.ts index ff1c4abcef4e..981799329ddb 100644 --- a/x-pack/plugins/fleet/server/services/package_policy.ts +++ b/x-pack/plugins/fleet/server/services/package_policy.ts @@ -94,6 +94,7 @@ import type { DryRunPackagePolicy, PostPackagePolicyCreateCallback, PostPackagePolicyPostCreateCallback, + PutPackagePolicyPostUpdateCallback, } from '../types'; import type { ExternalCallback } from '..'; @@ -127,6 +128,8 @@ import type { PackagePolicyClient, PackagePolicyClientFetchAllItemsOptions, PackagePolicyService, + RunExternalCallbacksPackagePolicyArgument, + RunExternalCallbacksPackagePolicyResponse, } from './package_policy_service'; import { installAssetsForInputPackagePolicy } from './epm/packages/install'; import { auditLoggingService } from './audit_logging'; @@ -1091,9 +1094,16 @@ class PackagePolicyClientImpl implements PackagePolicyClient { await Promise.all([...bumpPromises, assetRemovePromise, deleteSecretsPromise]); sendUpdatePackagePolicyTelemetryEvent(soClient, [packagePolicyUpdate], [oldPackagePolicy]); + logger.debug(`Package policy ${id} update completed`); - return newPolicy; + // Run external post-update callbacks and return + return packagePolicyService.runExternalCallbacks( + 'packagePolicyPostUpdate', + newPolicy, + soClient, + esClient + ); } public async bulkUpdate( @@ -1930,48 +1940,21 @@ class PackagePolicyClientImpl implements PackagePolicyClient { public async runExternalCallbacks( externalCallbackType: A, - packagePolicy: A extends 'packagePolicyDelete' - ? DeletePackagePoliciesResponse - : A extends 'packagePolicyPostDelete' - ? PostDeletePackagePoliciesResponse - : A extends 'packagePolicyPostCreate' - ? PackagePolicy - : A extends 'packagePolicyCreate' - ? NewPackagePolicy - : never, + packagePolicy: RunExternalCallbacksPackagePolicyArgument, soClient: SavedObjectsClientContract, esClient: ElasticsearchClient, context?: RequestHandlerContext, request?: KibanaRequest - ): Promise< - A extends 'packagePolicyDelete' - ? void - : A extends 'packagePolicyPostDelete' - ? void - : A extends 'packagePolicyPostCreate' - ? PackagePolicy - : A extends 'packagePolicyCreate' - ? NewPackagePolicy - : never - >; - public async runExternalCallbacks( - externalCallbackType: ExternalCallback[0], - packagePolicy: - | PackagePolicy - | NewPackagePolicy - | PostDeletePackagePoliciesResponse - | DeletePackagePoliciesResponse, - soClient: SavedObjectsClientContract, - esClient: ElasticsearchClient, - context?: RequestHandlerContext, - request?: KibanaRequest - ): Promise { + ): Promise> { const logger = appContextService.getLogger(); const numberOfCallbacks = appContextService.getExternalCallbacks(externalCallbackType)?.size; + let runResult: any; + logger.debug(`Running ${numberOfCallbacks} external callbacks for ${externalCallbackType}`); + try { if (externalCallbackType === 'packagePolicyPostDelete') { - return await this.runPostDeleteExternalCallbacks( + runResult = await this.runPostDeleteExternalCallbacks( packagePolicy as PostDeletePackagePoliciesResponse, soClient, esClient, @@ -1979,7 +1962,7 @@ class PackagePolicyClientImpl implements PackagePolicyClient { request ); } else if (externalCallbackType === 'packagePolicyDelete') { - return await this.runDeleteExternalCallbacks( + runResult = await this.runDeleteExternalCallbacks( packagePolicy as DeletePackagePoliciesResponse, soClient, esClient @@ -1988,21 +1971,33 @@ class PackagePolicyClientImpl implements PackagePolicyClient { if (!Array.isArray(packagePolicy)) { let newData = packagePolicy; const externalCallbacks = appContextService.getExternalCallbacks(externalCallbackType); + if (externalCallbacks && externalCallbacks.size > 0) { - let updatedNewData = newData; + let updatedNewData: any = newData; + for (const callback of externalCallbacks) { - let result; + let thisCallbackResponse; + if (externalCallbackType === 'packagePolicyPostCreate') { - result = await (callback as PostPackagePolicyPostCreateCallback)( + thisCallbackResponse = await (callback as PostPackagePolicyPostCreateCallback)( updatedNewData as PackagePolicy, soClient, esClient, context, request ); - updatedNewData = PackagePolicySchema.validate(result) as NewPackagePolicy; + updatedNewData = PackagePolicySchema.validate(thisCallbackResponse); + } else if (externalCallbackType === 'packagePolicyPostUpdate') { + thisCallbackResponse = await (callback as PutPackagePolicyPostUpdateCallback)( + updatedNewData as PackagePolicy, + soClient, + esClient, + context, + request + ); + updatedNewData = PackagePolicySchema.validate(thisCallbackResponse); } else { - result = await (callback as PostPackagePolicyCreateCallback)( + thisCallbackResponse = await (callback as PostPackagePolicyCreateCallback)( updatedNewData as NewPackagePolicy, soClient, esClient, @@ -2012,10 +2007,10 @@ class PackagePolicyClientImpl implements PackagePolicyClient { } if (externalCallbackType === 'packagePolicyCreate') { - updatedNewData = NewPackagePolicySchema.validate(result) as NewPackagePolicy; + updatedNewData = NewPackagePolicySchema.validate(thisCallbackResponse); } else if (externalCallbackType === 'packagePolicyUpdate') { const omitted = { - ...omit(result, [ + ...omit(thisCallbackResponse, [ 'id', 'spaceIds', 'version', @@ -2026,16 +2021,19 @@ class PackagePolicyClientImpl implements PackagePolicyClient { 'created_by', 'elasticsearch', ]), - inputs: result.inputs.map((input) => omit(input, ['compiled_input'])), + inputs: thisCallbackResponse.inputs.map((input) => + omit(input, ['compiled_input']) + ), }; - updatedNewData = UpdatePackagePolicySchema.validate(omitted) as PackagePolicy; + updatedNewData = UpdatePackagePolicySchema.validate(omitted); } } newData = updatedNewData; } - return newData; + + runResult = newData; } } } catch (error) { @@ -2043,6 +2041,8 @@ class PackagePolicyClientImpl implements PackagePolicyClient { logger.error(error); throw error; } + + return runResult as unknown as RunExternalCallbacksPackagePolicyResponse; } public async runPostDeleteExternalCallbacks( diff --git a/x-pack/plugins/fleet/server/services/package_policy_service.ts b/x-pack/plugins/fleet/server/services/package_policy_service.ts index f5cb879cef7c..967efb1055cf 100644 --- a/x-pack/plugins/fleet/server/services/package_policy_service.ts +++ b/x-pack/plugins/fleet/server/services/package_policy_service.ts @@ -40,6 +40,36 @@ export interface PackagePolicyService { get asInternalUser(): PackagePolicyClient; } +export type RunExternalCallbacksPackagePolicyArgument = + A extends 'packagePolicyDelete' + ? DeletePackagePoliciesResponse + : A extends 'packagePolicyPostDelete' + ? PostDeletePackagePoliciesResponse + : A extends 'packagePolicyCreate' + ? NewPackagePolicy + : A extends 'packagePolicyPostCreate' + ? PackagePolicy + : A extends 'packagePolicyUpdate' + ? UpdatePackagePolicy + : A extends 'packagePolicyPostUpdate' + ? PackagePolicy + : never; + +export type RunExternalCallbacksPackagePolicyResponse = + A extends 'packagePolicyDelete' + ? void + : A extends 'packagePolicyPostDelete' + ? void + : A extends 'packagePolicyCreate' + ? NewPackagePolicy + : A extends 'packagePolicyPostCreate' + ? PackagePolicy + : A extends 'packagePolicyUpdate' + ? UpdatePackagePolicy + : A extends 'packagePolicyPostUpdate' + ? PackagePolicy + : undefined; + export interface PackagePolicyClient { create( soClient: SavedObjectsClientContract, @@ -169,30 +199,12 @@ export interface PackagePolicyClient { runExternalCallbacks( externalCallbackType: A, - packagePolicy: A extends 'packagePolicyDelete' - ? DeletePackagePoliciesResponse - : A extends 'packagePolicyPostDelete' - ? PostDeletePackagePoliciesResponse - : A extends 'packagePolicyPostCreate' - ? PackagePolicy - : A extends 'packagePolicyUpdate' - ? UpdatePackagePolicy - : NewPackagePolicy, + packagePolicy: RunExternalCallbacksPackagePolicyArgument, soClient: SavedObjectsClientContract, esClient: ElasticsearchClient, context?: RequestHandlerContext, request?: KibanaRequest - ): Promise< - A extends 'packagePolicyDelete' - ? void - : A extends 'packagePolicyPostDelete' - ? void - : A extends 'packagePolicyPostCreate' - ? PackagePolicy - : A extends 'packagePolicyUpdate' - ? UpdatePackagePolicy - : NewPackagePolicy - >; + ): Promise>; runDeleteExternalCallbacks( deletedPackagePolicies: DeletePackagePoliciesResponse, diff --git a/x-pack/plugins/fleet/server/types/extensions.ts b/x-pack/plugins/fleet/server/types/extensions.ts index 594e16f61955..2293747b253e 100644 --- a/x-pack/plugins/fleet/server/types/extensions.ts +++ b/x-pack/plugins/fleet/server/types/extensions.ts @@ -60,6 +60,14 @@ export type PutPackagePolicyUpdateCallback = ( request?: KibanaRequest ) => Promise; +export type PutPackagePolicyPostUpdateCallback = ( + packagePolicy: PackagePolicy, + soClient: SavedObjectsClientContract, + esClient: ElasticsearchClient, + context?: RequestHandlerContext, + request?: KibanaRequest +) => Promise; + export type PostAgentPolicyCreateCallback = ( agentPolicy: NewAgentPolicy ) => Promise; @@ -68,6 +76,8 @@ export type PostAgentPolicyUpdateCallback = ( agentPolicy: Partial ) => Promise>; +export type PostAgentPolicyPostUpdateCallback = (agentPolicy: AgentPolicy) => Promise; + export type ExternalCallbackCreate = ['packagePolicyCreate', PostPackagePolicyCreateCallback]; export type ExternalCallbackPostCreate = [ 'packagePolicyPostCreate', @@ -79,7 +89,12 @@ export type ExternalCallbackPostDelete = [ 'packagePolicyPostDelete', PostPackagePolicyPostDeleteCallback ]; + export type ExternalCallbackUpdate = ['packagePolicyUpdate', PutPackagePolicyUpdateCallback]; +export type ExternalCallbackPostUpdate = [ + 'packagePolicyPostUpdate', + PutPackagePolicyPostUpdateCallback +]; export type ExternalCallbackAgentPolicyCreate = [ 'agentPolicyCreate', @@ -89,6 +104,10 @@ export type ExternalCallbackAgentPolicyUpdate = [ 'agentPolicyUpdate', PostAgentPolicyUpdateCallback ]; +export type ExternalCallbackAgentPolicyPostUpdate = [ + 'agentPolicyPostUpdate', + PostAgentPolicyPostUpdateCallback +]; /** * Callbacks supported by the Fleet plugin @@ -99,7 +118,9 @@ export type ExternalCallback = | ExternalCallbackDelete | ExternalCallbackPostDelete | ExternalCallbackUpdate + | ExternalCallbackPostUpdate | ExternalCallbackAgentPolicyCreate - | ExternalCallbackAgentPolicyUpdate; + | ExternalCallbackAgentPolicyUpdate + | ExternalCallbackAgentPolicyPostUpdate; export type ExternalCallbacksStorage = Map>; diff --git a/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.ts b/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.ts index 192fb6059325..1afa24ebbd52 100644 --- a/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.ts +++ b/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.ts @@ -31,10 +31,12 @@ import type { ResponseActionsClient } from './services'; import { getResponseActionsClient, NormalizedExternalConnectorClient } from './services'; import { getAgentPolicyCreateCallback, + getAgentPolicyPostUpdateCallback, getAgentPolicyUpdateCallback, getPackagePolicyCreateCallback, getPackagePolicyDeleteCallback, getPackagePolicyPostCreateCallback, + getPackagePolicyPostUpdateCallback, getPackagePolicyUpdateCallback, } from '../fleet_integration/fleet_integration'; import type { ManifestManager } from './services/artifacts'; @@ -117,7 +119,8 @@ export class EndpointAppContextService { this.savedObjectsFactoryService = savedObjectsFactory; this.fleetServicesFactory = new EndpointFleetServicesFactory( dependencies.fleetStartServices, - savedObjectsFactory + savedObjectsFactory, + this.createLogger('endpointFleetServices') ); this.registerFleetExtensions(); @@ -169,6 +172,8 @@ export class EndpointAppContextService { getAgentPolicyUpdateCallback(logger, productFeaturesService) ); + registerFleetCallback('agentPolicyPostUpdate', getAgentPolicyPostUpdateCallback(this)); + registerFleetCallback( 'packagePolicyCreate', getPackagePolicyCreateCallback( @@ -183,10 +188,7 @@ export class EndpointAppContextService { ) ); - registerFleetCallback( - 'packagePolicyPostCreate', - getPackagePolicyPostCreateCallback(logger, exceptionListsClient) - ); + registerFleetCallback('packagePolicyPostCreate', getPackagePolicyPostCreateCallback(this)); registerFleetCallback( 'packagePolicyUpdate', @@ -201,6 +203,8 @@ export class EndpointAppContextService { ) ); + registerFleetCallback('packagePolicyPostUpdate', getPackagePolicyPostUpdateCallback(this)); + registerFleetCallback( 'packagePolicyPostDelete', getPackagePolicyDeleteCallback(exceptionListsClient, soClient) @@ -218,6 +222,27 @@ export class EndpointAppContextService { return this.savedObjectsFactoryService; } + /** + * Is kibana running in serverless mode + */ + public isServerless(): boolean { + if (!this.setupDependencies) { + throw new EndpointAppContentServicesNotSetUpError(); + } + + // TODO:PT check what this returns when running locally with kibana in serverless emulation + + return Boolean(this.setupDependencies.cloud.isServerlessEnabled); + } + + public getInternalEsClient(): ElasticsearchClient { + if (!this.startDependencies?.esClient) { + throw new EndpointAppContentServicesNotStartedError(); + } + + return this.startDependencies.esClient; + } + private getFleetAuthzService(): FleetStartContract['authz'] { if (!this.startDependencies?.fleetStartServices) { throw new EndpointAppContentServicesNotStartedError(); diff --git a/x-pack/plugins/security_solution/server/endpoint/services/actions/clients/lib/simple_mem_cache.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/simple_mem_cache.test.ts similarity index 92% rename from x-pack/plugins/security_solution/server/endpoint/services/actions/clients/lib/simple_mem_cache.test.ts rename to x-pack/plugins/security_solution/server/endpoint/lib/simple_mem_cache.test.ts index f351e2e40d5b..27ba3bdf2394 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/actions/clients/lib/simple_mem_cache.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/simple_mem_cache.test.ts @@ -51,6 +51,13 @@ describe('SimpleMemCache class', () => { expect(cache.get(key)).toEqual(undefined); }); + it('should delete all entries from cache', () => { + cache.set(key, value); + cache.deleteAll(); + + expect(cache.get(key)).toEqual(undefined); + }); + it('should cleanup expired cache entries', () => { const key2 = 'myKey'; cache.set(key, value); // Default ttl of 10s diff --git a/x-pack/plugins/security_solution/server/endpoint/services/actions/clients/lib/simple_mem_cache.ts b/x-pack/plugins/security_solution/server/endpoint/lib/simple_mem_cache.ts similarity index 95% rename from x-pack/plugins/security_solution/server/endpoint/services/actions/clients/lib/simple_mem_cache.ts rename to x-pack/plugins/security_solution/server/endpoint/lib/simple_mem_cache.ts index fc355bf6c379..a65a1ee6be71 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/actions/clients/lib/simple_mem_cache.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/simple_mem_cache.ts @@ -19,6 +19,8 @@ export interface SimpleMemCacheInterface { get(key: any): TValue | undefined; /** Delete a piece of data from cache */ delete(key: any): void; + /** Delete all cached entries */ + deleteAll(): void; /** Clean up the cache by removing all expired entries */ cleanup(): void; } @@ -79,6 +81,10 @@ export class SimpleMemCache implements SimpleMemCacheInterface { this.cache.delete(key); } + public deleteAll(): void { + this.cache.clear(); + } + public cleanup(): void { for (const [cacheKey, cacheData] of this.cache.entries()) { if (this.isExpired(cacheData)) { diff --git a/x-pack/plugins/security_solution/server/endpoint/mocks/mocks.ts b/x-pack/plugins/security_solution/server/endpoint/mocks/mocks.ts index 5ab221b7bfc0..91a2bc40454b 100644 --- a/x-pack/plugins/security_solution/server/endpoint/mocks/mocks.ts +++ b/x-pack/plugins/security_solution/server/endpoint/mocks/mocks.ts @@ -144,6 +144,8 @@ export const createMockEndpointAppContextService = ( return responseActionsClientMock.create(); }), savedObjects: createSavedObjectsClientFactoryMock({ savedObjectsServiceStart }).service, + isServerless: jest.fn().mockReturnValue(false), + getInternalEsClient: jest.fn().mockReturnValue(esClient), } as unknown as jest.Mocked; }; diff --git a/x-pack/plugins/security_solution/server/endpoint/services/actions/clients/lib/base_response_actions_client.ts b/x-pack/plugins/security_solution/server/endpoint/services/actions/clients/lib/base_response_actions_client.ts index 0411e4a9c8f6..7a8f14b6e9a8 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/actions/clients/lib/base_response_actions_client.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/actions/clients/lib/base_response_actions_client.ts @@ -20,7 +20,7 @@ import { } from '../../../../../lib/telemetry/event_based/events'; import { NotFoundError } from '../../../../errors'; import { fetchActionRequestById } from '../../utils/fetch_action_request_by_id'; -import { SimpleMemCache } from './simple_mem_cache'; +import { SimpleMemCache } from '../../../../lib/simple_mem_cache'; import { fetchActionResponses, fetchEndpointActionResponses, @@ -581,6 +581,10 @@ export abstract class ResponseActionsClientImpl implements ResponseActionsClient >( options: ResponseActionsClientWriteActionResponseToEndpointIndexOptions ): Promise> { + // FIXME:PT need to ensure we use a index below that has the proper `namespace` when agent type is Endpoint + // Background: Endpoint responses require that the document be written to an index that has the + // correct `namespace` as defined by the Integration/Agent policy and that logic is not currently implemented. + const doc = this.buildActionResponseEsDoc(options); this.log.debug(() => `Writing response action response:\n${stringify(doc)}`); @@ -594,7 +598,7 @@ export abstract class ResponseActionsClientImpl implements ResponseActionsClient .catch((err) => { throw new ResponseActionsClientError( `Failed to create action response document: ${err.message}`, - err.statusCode ?? 500, + 500, err ); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/services/fleet/endpoint_fleet_services_factory.mocks.ts b/x-pack/plugins/security_solution/server/endpoint/services/fleet/endpoint_fleet_services_factory.mocks.ts index 91119ea3df5f..302528b024f7 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/fleet/endpoint_fleet_services_factory.mocks.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/fleet/endpoint_fleet_services_factory.mocks.ts @@ -8,6 +8,8 @@ import type { DeeplyMockedKeys } from '@kbn/utility-types-jest'; import type { FleetStartContract } from '@kbn/fleet-plugin/server'; import { createFleetStartContractMock } from '@kbn/fleet-plugin/server/mocks'; +import type { MockedLogger } from '@kbn/logging-mocks'; +import { loggingSystemMock } from '@kbn/core-logging-server-mocks'; import type { SavedObjectsClientFactory } from '../saved_objects'; import type { EndpointFleetServicesFactoryInterface, @@ -24,35 +26,41 @@ export interface EndpointFleetServicesFactoryInterfaceMocked asInternalUser: () => EndpointInternalFleetServicesInterfaceMocked; } -interface CreateEndpointFleetServicesFactoryMockOptions { +export interface CreateEndpointFleetServicesFactoryMockOptions { fleetDependencies: DeeplyMockedKeys; savedObjects: SavedObjectsClientFactory; + logger: MockedLogger; } -export const createEndpointFleetServicesFactoryMock = ( - dependencies: Partial = {} -): { +export interface CreateEndpointFleetServicesFactoryResponse { service: EndpointFleetServicesFactoryInterfaceMocked; dependencies: CreateEndpointFleetServicesFactoryMockOptions; -} => { +} + +export const createEndpointFleetServicesFactoryMock = ( + dependencies: Partial = {} +): CreateEndpointFleetServicesFactoryResponse => { const { fleetDependencies = createFleetStartContractMock(), savedObjects = createSavedObjectsClientFactoryMock().service, + logger = loggingSystemMock.createLogger(), } = dependencies; const serviceFactoryMock = new EndpointFleetServicesFactory( fleetDependencies, - savedObjects + savedObjects, + logger ) as unknown as EndpointFleetServicesFactoryInterfaceMocked; const fleetInternalServicesMocked = serviceFactoryMock.asInternalUser(); jest.spyOn(fleetInternalServicesMocked, 'ensureInCurrentSpace'); + jest.spyOn(fleetInternalServicesMocked, 'getPolicyNamespace'); const asInternalUserSpy = jest.spyOn(serviceFactoryMock, 'asInternalUser'); asInternalUserSpy.mockReturnValue(fleetInternalServicesMocked); return { service: serviceFactoryMock, - dependencies: { fleetDependencies, savedObjects }, + dependencies: { fleetDependencies, savedObjects, logger }, }; }; diff --git a/x-pack/plugins/security_solution/server/endpoint/services/fleet/endpoint_fleet_services_factory.test.ts b/x-pack/plugins/security_solution/server/endpoint/services/fleet/endpoint_fleet_services_factory.test.ts new file mode 100644 index 000000000000..c1f7ca004e03 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/services/fleet/endpoint_fleet_services_factory.test.ts @@ -0,0 +1,181 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { + CreateEndpointFleetServicesFactoryResponse, + EndpointInternalFleetServicesInterfaceMocked, +} from './endpoint_fleet_services_factory.mocks'; +import { createEndpointFleetServicesFactoryMock } from './endpoint_fleet_services_factory.mocks'; +import { AgentNotFoundError } from '@kbn/fleet-plugin/server'; +import { NotFoundError } from '../../errors'; +import type { AgentPolicy, PackagePolicy } from '@kbn/fleet-plugin/common'; +import { FleetAgentPolicyGenerator } from '../../../../common/endpoint/data_generators/fleet_agent_policy_generator'; +import { FleetPackagePolicyGenerator } from '../../../../common/endpoint/data_generators/fleet_package_policy_generator'; + +describe('EndpointServiceFactory', () => { + let fleetServicesMock: EndpointInternalFleetServicesInterfaceMocked; + let fleetServicesFactoryMock: CreateEndpointFleetServicesFactoryResponse; + + beforeEach(() => { + fleetServicesFactoryMock = createEndpointFleetServicesFactoryMock(); + fleetServicesMock = fleetServicesFactoryMock.service.asInternalUser(); + }); + + it('should return fleet services when `asInternalUser()` is invoked', () => { + expect(Object.keys(fleetServicesMock)).toEqual([ + 'agent', + 'agentPolicy', + 'packages', + 'packagePolicy', + 'savedObjects', + 'endpointPolicyKuery', + 'ensureInCurrentSpace', + 'getPolicyNamespace', + ]); + }); + + describe('#ensureInCurentSpace()', () => { + it('should check agent ids', async () => { + await expect( + fleetServicesMock.ensureInCurrentSpace({ agentIds: ['123'] }) + ).resolves.toBeUndefined(); + expect( + fleetServicesFactoryMock.dependencies.fleetDependencies.agentService.asInternalUser.getByIds + ).toHaveBeenCalledWith(['123']); + expect( + fleetServicesFactoryMock.dependencies.fleetDependencies.agentPolicyService.getByIds + ).not.toHaveBeenCalled(); + expect( + fleetServicesFactoryMock.dependencies.fleetDependencies.packagePolicyService.getByIDs + ).not.toHaveBeenCalled(); + }); + + it('should check integration policy ids', async () => { + await expect( + fleetServicesMock.ensureInCurrentSpace({ integrationPolicyIds: ['123'] }) + ).resolves.toBeUndefined(); + expect( + fleetServicesFactoryMock.dependencies.fleetDependencies.agentService.asInternalUser.getByIds + ).not.toHaveBeenCalled(); + expect( + fleetServicesFactoryMock.dependencies.fleetDependencies.agentPolicyService.getByIds + ).not.toHaveBeenCalled(); + expect( + fleetServicesFactoryMock.dependencies.fleetDependencies.packagePolicyService.getByIDs + ).toHaveBeenCalledWith(expect.anything(), ['123']); + }); + + it('should check agent policy ids', async () => { + await expect( + fleetServicesMock.ensureInCurrentSpace({ agentPolicyIds: ['123'] }) + ).resolves.toBeUndefined(); + expect( + fleetServicesFactoryMock.dependencies.fleetDependencies.agentService.asInternalUser.getByIds + ).not.toHaveBeenCalled(); + expect( + fleetServicesFactoryMock.dependencies.fleetDependencies.agentPolicyService.getByIds + ).toHaveBeenCalledWith(expect.anything(), ['123']); + expect( + fleetServicesFactoryMock.dependencies.fleetDependencies.packagePolicyService.getByIDs + ).not.toHaveBeenCalled(); + }); + + it('should check agent Ids, integration policy id and agent policy ids', async () => { + await expect( + fleetServicesMock.ensureInCurrentSpace({ + integrationPolicyIds: ['123'], + agentIds: ['123'], + agentPolicyIds: ['123'], + }) + ).resolves.toBeUndefined(); + expect( + fleetServicesFactoryMock.dependencies.fleetDependencies.agentService.asInternalUser.getByIds + ).toHaveBeenCalledWith(['123']); + expect( + fleetServicesFactoryMock.dependencies.fleetDependencies.agentPolicyService.getByIds + ).toHaveBeenCalledWith(expect.anything(), ['123']); + expect( + fleetServicesFactoryMock.dependencies.fleetDependencies.packagePolicyService.getByIDs + ).toHaveBeenCalledWith(expect.anything(), ['123']); + }); + + it('should throw error any of the data is not visible in current space', async () => { + fleetServicesFactoryMock.dependencies.fleetDependencies.agentService.asInternalUser.getByIds.mockImplementation( + async () => { + throw new AgentNotFoundError('not found mock'); + } + ); + await expect( + fleetServicesMock.ensureInCurrentSpace({ + integrationPolicyIds: ['123'], + agentIds: ['123'], + agentPolicyIds: ['123'], + }) + ).rejects.toThrowError(NotFoundError); + }); + }); + + describe('#getPolicyNamespace()', () => { + let integrationPolicy: PackagePolicy; + let agentPolicy1: AgentPolicy; + let agentPolicy2: AgentPolicy; + + beforeEach(() => { + const agentPolicyGenerator = new FleetAgentPolicyGenerator('seed'); + const integrationPolicyGenerator = new FleetPackagePolicyGenerator('seed'); + + agentPolicy1 = agentPolicyGenerator.generate({ namespace: 'foo1' }); + agentPolicy2 = agentPolicyGenerator.generate({ namespace: 'foo2' }); + integrationPolicy = integrationPolicyGenerator.generate({ + namespace: undefined, + policy_ids: [agentPolicy1.id, agentPolicy2.id], + }); + + fleetServicesFactoryMock.dependencies.fleetDependencies.packagePolicyService.getByIDs.mockResolvedValue( + [integrationPolicy] + ); + fleetServicesFactoryMock.dependencies.fleetDependencies.agentPolicyService.getByIds.mockResolvedValue( + [agentPolicy1, agentPolicy2] + ); + }); + + it('should return namespace from agent policies if integration policy does not have one defined', async () => { + await expect( + fleetServicesMock.getPolicyNamespace({ + integrationPolicies: [integrationPolicy.id], + }) + ).resolves.toEqual({ + integrationPolicy: { + [integrationPolicy.id]: ['foo1', 'foo2'], + }, + }); + expect( + fleetServicesFactoryMock.dependencies.fleetDependencies.agentPolicyService.getByIds + ).toHaveBeenCalledWith(expect.anything(), [agentPolicy1.id, agentPolicy2.id]); + }); + + it('should return namespace from integration policy if defined', async () => { + integrationPolicy.namespace = 'bar'; + + await expect( + fleetServicesMock.getPolicyNamespace({ + integrationPolicies: [integrationPolicy.id], + }) + ).resolves.toEqual({ + integrationPolicy: { + [integrationPolicy.id]: ['bar'], + }, + }); + + // The agentPolicy sevice should not have been called because the package policy has + // a namespace id, so no need. + expect( + fleetServicesFactoryMock.dependencies.fleetDependencies.agentPolicyService.getByIds + ).not.toHaveBeenCalled(); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/endpoint/services/fleet/endpoint_fleet_services_factory.ts b/x-pack/plugins/security_solution/server/endpoint/services/fleet/endpoint_fleet_services_factory.ts index 50e200627221..bbda061b3cef 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/fleet/endpoint_fleet_services_factory.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/fleet/endpoint_fleet_services_factory.ts @@ -13,12 +13,14 @@ import type { PackageClient, } from '@kbn/fleet-plugin/server'; import { AgentNotFoundError } from '@kbn/fleet-plugin/server'; -import { PACKAGE_POLICY_SAVED_OBJECT_TYPE } from '@kbn/fleet-plugin/common'; -import type { SavedObjectsClientContract } from '@kbn/core/server'; +import type { AgentPolicy } from '@kbn/fleet-plugin/common'; +import { PACKAGE_POLICY_SAVED_OBJECT_TYPE, type PackagePolicy } from '@kbn/fleet-plugin/common'; +import type { Logger, SavedObjectsClientContract } from '@kbn/core/server'; import { AgentPolicyNotFoundError, PackagePolicyNotFoundError, } from '@kbn/fleet-plugin/server/errors'; +import { stringify } from '../../utils/stringify'; import { NotFoundError } from '../../errors'; import type { SavedObjectsClientFactory } from '../saved_objects'; @@ -37,14 +39,21 @@ export interface EndpointFleetServicesInterface { * Will check the data provided to ensure it is visible for the current space. Supports * several types of data (ex. integration policies, agent policies, etc) */ - ensureInCurrentSpace(options: EnsureInCurrentSpaceOptions): Promise; -} + ensureInCurrentSpace( + options: Pick< + CheckInCurrentSpaceOptions, + 'agentIds' | 'integrationPolicyIds' | 'agentPolicyIds' + > + ): Promise; -type EnsureInCurrentSpaceOptions = Partial<{ - agentIds: string[]; - agentPolicyIds: string[]; - integrationPolicyIds: string[]; -}>; + /** + * Retrieves the `namespace` assigned to Endpoint Integration Policies + * @param options + */ + getPolicyNamespace( + options: Pick + ): Promise; +} export interface EndpointInternalFleetServicesInterface extends EndpointFleetServicesInterface { savedObjects: SavedObjectsClientFactory; @@ -60,7 +69,8 @@ export interface EndpointFleetServicesFactoryInterface { export class EndpointFleetServicesFactory implements EndpointFleetServicesFactoryInterface { constructor( private readonly fleetDependencies: FleetStartContract, - private readonly savedObjects: SavedObjectsClientFactory + private readonly savedObjects: SavedObjectsClientFactory, + private readonly logger: Logger ) {} asInternalUser(spaceId?: string): EndpointInternalFleetServicesInterface { @@ -85,31 +95,31 @@ export class EndpointFleetServicesFactory implements EndpointFleetServicesFactor if (!soClient) { soClient = this.savedObjects.createInternalScopedSoClient({ spaceId }); } + return checkInCurrentSpace({ + soClient, + agentService: agent, + agentPolicyService: agentPolicy, + packagePolicyService: packagePolicy, + integrationPolicyIds, + agentPolicyIds, + agentIds, + }); + }; + + const getPolicyNamespace: EndpointFleetServicesInterface['getPolicyNamespace'] = async ( + options + ) => { + if (!soClient) { + soClient = this.savedObjects.createInternalScopedSoClient({ spaceId }); + } - const handlePromiseErrors = (err: Error): never => { - // We wrap the error with our own Error class so that the API can property return a 404 - if ( - err instanceof AgentNotFoundError || - err instanceof AgentPolicyNotFoundError || - err instanceof PackagePolicyNotFoundError - ) { - throw new NotFoundError(err.message, err); - } - - throw err; - }; - - await Promise.all([ - agentIds.length ? agent.getByIds(agentIds).catch(handlePromiseErrors) : null, - - agentPolicyIds.length - ? agentPolicy.getByIds(soClient, agentPolicyIds).catch(handlePromiseErrors) - : null, - - integrationPolicyIds.length - ? packagePolicy.getByIDs(soClient, integrationPolicyIds).catch(handlePromiseErrors) - : null, - ]); + return fetchEndpointPolicyNamespace({ + ...options, + soClient, + logger: this.logger, + packagePolicyService: packagePolicy, + agentPolicyService: agentPolicy, + }); }; return { @@ -123,6 +133,144 @@ export class EndpointFleetServicesFactory implements EndpointFleetServicesFactor endpointPolicyKuery: `${PACKAGE_POLICY_SAVED_OBJECT_TYPE}.package.name: "endpoint"`, ensureInCurrentSpace, + getPolicyNamespace, }; } } + +interface CheckInCurrentSpaceOptions { + soClient: SavedObjectsClientContract; + agentService: AgentClient; + agentPolicyService: AgentPolicyServiceInterface; + packagePolicyService: PackagePolicyClient; + agentIds?: string[]; + agentPolicyIds?: string[]; + integrationPolicyIds?: string[]; +} + +/** + * Checks if data provided (integration policies, agent policies and/or agentIds) are visible in + * current space + * + * @param soClient + * @param agentService + * @param agentPolicyService + * @param packagePolicyService + * @param integrationPolicyIds + * @param agentPolicyIds + * @param agentIds + * + * @throws NotFoundError + */ +const checkInCurrentSpace = async ({ + soClient, + agentService, + agentPolicyService, + packagePolicyService, + integrationPolicyIds = [], + agentPolicyIds = [], + agentIds = [], +}: CheckInCurrentSpaceOptions): Promise => { + const handlePromiseErrors = (err: Error): never => { + // We wrap the error with our own Error class so that the API can property return a 404 + if ( + err instanceof AgentNotFoundError || + err instanceof AgentPolicyNotFoundError || + err instanceof PackagePolicyNotFoundError + ) { + throw new NotFoundError(err.message, err); + } + + throw err; + }; + + await Promise.all([ + agentIds.length ? agentService.getByIds(agentIds).catch(handlePromiseErrors) : null, + + agentPolicyIds.length + ? agentPolicyService.getByIds(soClient, agentPolicyIds).catch(handlePromiseErrors) + : null, + + integrationPolicyIds.length + ? packagePolicyService.getByIDs(soClient, integrationPolicyIds).catch(handlePromiseErrors) + : null, + ]); +}; + +interface FetchEndpointPolicyNamespaceOptions { + logger: Logger; + soClient: SavedObjectsClientContract; + packagePolicyService: PackagePolicyClient; + agentPolicyService: AgentPolicyServiceInterface; + /** A list of integration policies IDs */ + integrationPolicies: string[]; +} + +export interface FetchEndpointPolicyNamespaceResponse { + integrationPolicy: Record; +} + +const fetchEndpointPolicyNamespace = async ({ + logger, + soClient, + packagePolicyService, + agentPolicyService, + integrationPolicies, +}: FetchEndpointPolicyNamespaceOptions): Promise => { + const response: FetchEndpointPolicyNamespaceResponse = { + integrationPolicy: {}, + }; + const agentPolicyIdsToRetrieve = new Set(); + const retrievedIntegrationPolicies: Record = {}; + const retrievedAgentPolicies: Record = {}; + + if (integrationPolicies.length > 0) { + logger.debug( + () => `Retrieving package policies from fleet for:\n${stringify(integrationPolicies)}` + ); + const packagePolicies = + (await packagePolicyService.getByIDs(soClient, integrationPolicies)) ?? []; + + logger.trace(() => `Fleet package policies retrieved:\n${stringify(packagePolicies)}`); + + for (const packagePolicy of packagePolicies) { + retrievedIntegrationPolicies[packagePolicy.id] = packagePolicy; + + // Integration policy does not have an explicit namespace, which means it + // inherits it from the associated agent policies, so lets retrieve those + if (!packagePolicy.namespace) { + packagePolicy.policy_ids.forEach((agentPolicyId) => { + agentPolicyIdsToRetrieve.add(agentPolicyId); + }); + } + } + } + + if (agentPolicyIdsToRetrieve.size > 0) { + const ids = Array.from(agentPolicyIdsToRetrieve); + + logger.debug(() => `Retrieving agent policies from fleet for:\n${stringify(ids)}`); + + const agentPolicies = await agentPolicyService.getByIds(soClient, ids); + + logger.trace(() => `Fleet agent policies retrieved:\n${stringify(agentPolicies)}`); + + for (const agentPolicy of agentPolicies) { + retrievedAgentPolicies[agentPolicy.id] = agentPolicy; + } + } + + for (const integrationPolicyId of integrationPolicies) { + const integrationPolicyNamespace = retrievedIntegrationPolicies[integrationPolicyId].namespace; + + response.integrationPolicy[integrationPolicyId] = integrationPolicyNamespace + ? [integrationPolicyNamespace] + : retrievedIntegrationPolicies[integrationPolicyId].policy_ids.map((agentPolicyId) => { + return retrievedAgentPolicies[agentPolicyId].namespace; + }); + } + + logger.debug(() => `Policy namespaces:\n${stringify(response)}`); + + return response; +}; diff --git a/x-pack/plugins/security_solution/server/fleet_integration/fleet_integration.test.ts b/x-pack/plugins/security_solution/server/fleet_integration/fleet_integration.test.ts index 6accb29354ee..80337d1a927b 100644 --- a/x-pack/plugins/security_solution/server/fleet_integration/fleet_integration.test.ts +++ b/x-pack/plugins/security_solution/server/fleet_integration/fleet_integration.test.ts @@ -7,6 +7,7 @@ import type { ExceptionListSchema } from '@kbn/securitysolution-io-ts-list-types'; +import type { ElasticsearchClientMock } from '@kbn/core/server/mocks'; import { elasticsearchServiceMock, httpServerMock, @@ -38,7 +39,10 @@ import { import { requestContextMock } from '../lib/detection_engine/routes/__mocks__'; import { requestContextFactoryMock } from '../request_context_factory.mock'; import type { EndpointAppContextServiceStartContract } from '../endpoint/endpoint_app_context_services'; -import { createMockEndpointAppContextServiceStartContract } from '../endpoint/mocks'; +import { + createMockEndpointAppContextService, + createMockEndpointAppContextServiceStartContract, +} from '../endpoint/mocks'; import { licenseMock } from '@kbn/licensing-plugin/common/licensing.mock'; import { LicenseService } from '../../common/license'; import { Subject } from 'rxjs'; @@ -73,16 +77,30 @@ import { createProductFeaturesServiceMock } from '../lib/product_features_servic import * as moment from 'moment'; import type { PostAgentPolicyCreateCallback, + PostPackagePolicyPostCreateCallback, PutPackagePolicyUpdateCallback, } from '@kbn/fleet-plugin/server/types'; import type { EndpointMetadataService } from '../endpoint/services/metadata'; import { createEndpointMetadataServiceTestContextMock } from '../endpoint/services/metadata/mocks'; +import { createPolicyDataStreamsIfNeeded as _createPolicyDataStreamsIfNeeded } from './handlers/create_policy_datastreams'; jest.mock('uuid', () => ({ v4: (): string => 'NEW_UUID', })); -describe('ingest_integration tests ', () => { +jest.mock('./handlers/create_policy_datastreams', () => { + const actualModule = jest.requireActual('./handlers/create_policy_datastreams'); + + return { + ...actualModule, + createPolicyDataStreamsIfNeeded: jest.fn(async () => {}), + }; +}); + +const createPolicyDataStreamsIfNeededMock = + _createPolicyDataStreamsIfNeeded as unknown as jest.Mock; + +describe('Fleet integrations', () => { let endpointAppContextStartContract: EndpointAppContextServiceStartContract; let req: KibanaRequest; let ctx: ReturnType; @@ -350,10 +368,20 @@ describe('ingest_integration tests ', () => { }); describe('package policy post create callback', () => { - const soClient = savedObjectsClientMock.create(); - const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; - const callback = getPackagePolicyPostCreateCallback(logger, exceptionListClient); - const policyConfig = generator.generatePolicyPackagePolicy() as PackagePolicy; + let soClient: ReturnType; + let esClient: ElasticsearchClientMock; + let callback: PostPackagePolicyPostCreateCallback; + let policyConfig: PackagePolicy; + let endpointAppContextServiceMock: ReturnType; + + beforeEach(() => { + soClient = savedObjectsClientMock.create(); + esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + endpointAppContextServiceMock = createMockEndpointAppContextService(); + endpointAppContextServiceMock.getExceptionListsClient.mockReturnValue(exceptionListClient); + callback = getPackagePolicyPostCreateCallback(endpointAppContextServiceMock); + policyConfig = generator.generatePolicyPackagePolicy() as PackagePolicy; + }); it('should create the Endpoint Event Filters List and add the correct Event Filters List Item attached to the policy given nonInteractiveSession parameter on integration config eventFilters', async () => { const integrationConfig = { @@ -374,11 +402,11 @@ describe('ingest_integration tests ', () => { req ); - expect(await exceptionListClient.createExceptionList).toHaveBeenCalledWith( + expect(exceptionListClient.createExceptionList).toHaveBeenCalledWith( expect.objectContaining({ listId: ENDPOINT_EVENT_FILTERS_LIST_ID }) ); - expect(await exceptionListClient.createExceptionListItem).toHaveBeenCalledWith( + expect(exceptionListClient.createExceptionListItem).toHaveBeenCalledWith( expect.objectContaining({ listId: ENDPOINT_EVENT_FILTERS_LIST_ID, tags: [`policy:${postCreatedPolicyConfig.id}`], @@ -413,14 +441,20 @@ describe('ingest_integration tests ', () => { req ); - expect(await exceptionListClient.createExceptionList).not.toHaveBeenCalled(); + expect(exceptionListClient.createExceptionList).not.toHaveBeenCalled(); - expect(await exceptionListClient.createExceptionListItem).not.toHaveBeenCalled(); + expect(exceptionListClient.createExceptionListItem).not.toHaveBeenCalled(); expect(postCreatedPolicyConfig.inputs[0]!.config!.integration_config.value).toEqual( integrationConfig ); }); + + it('should call `createPolicyDatastreamsIfNeeded`', async () => { + await callback(policyConfig, soClient, esClient, requestContextMock.convertContext(ctx), req); + + expect(createPolicyDataStreamsIfNeededMock).toHaveBeenCalled(); + }); }); describe('agent policy update callback', () => { diff --git a/x-pack/plugins/security_solution/server/fleet_integration/fleet_integration.ts b/x-pack/plugins/security_solution/server/fleet_integration/fleet_integration.ts index 71c935e72051..54f1ce8cc7e0 100644 --- a/x-pack/plugins/security_solution/server/fleet_integration/fleet_integration.ts +++ b/x-pack/plugins/security_solution/server/fleet_integration/fleet_integration.ts @@ -27,8 +27,13 @@ import type { InfoResponse } from '@elastic/elasticsearch/lib/api/types'; import { ProductFeatureSecurityKey } from '@kbn/security-solution-features/keys'; import type { PostAgentPolicyCreateCallback, + PostAgentPolicyPostUpdateCallback, PostAgentPolicyUpdateCallback, + PutPackagePolicyPostUpdateCallback, } from '@kbn/fleet-plugin/server/types'; +import type { EndpointInternalFleetServicesInterface } from '../endpoint/services/fleet'; +import type { EndpointAppContextService } from '../endpoint/endpoint_app_context_services'; +import { createPolicyDataStreamsIfNeeded } from './handlers/create_policy_datastreams'; import { updateAntivirusRegistrationEnabled } from '../../common/endpoint/utils/update_antivirus_registration_enabled'; import { validatePolicyAgainstProductFeatures } from './handlers/validate_policy_against_product_features'; import { validateEndpointPackagePolicy } from './handlers/validate_endpoint_package_policy'; @@ -62,6 +67,32 @@ const isEndpointPackagePolicy = ( return packagePolicy.package?.name === 'endpoint'; }; +const getEndpointPolicyForAgentPolicy = async ( + fleetServices: EndpointInternalFleetServicesInterface, + agentPolicy: AgentPolicy +): Promise => { + let agentPolicyIntegrations: PackagePolicy[] | undefined = agentPolicy.package_policies; + + if (!agentPolicyIntegrations) { + const fullAgentPolicy = await fleetServices.agentPolicy.get( + fleetServices.savedObjects.createInternalScopedSoClient(), + agentPolicy.id, + true + ); + agentPolicyIntegrations = fullAgentPolicy?.package_policies ?? []; + } + + if (Array.isArray(agentPolicyIntegrations)) { + for (const integrationPolicy of agentPolicyIntegrations) { + if (isEndpointPackagePolicy(integrationPolicy)) { + return integrationPolicy; + } + } + } + + return undefined; +}; + const shouldUpdateMetaValues = ( endpointPackagePolicy: PolicyConfig, currentLicenseType: string, @@ -279,16 +310,47 @@ export const getPackagePolicyUpdateCallback = ( }; }; +export const getPackagePolicyPostUpdateCallback = ( + endpointServices: EndpointAppContextService +): PutPackagePolicyPostUpdateCallback => { + const logger = endpointServices.createLogger('endpointPackagePolicyPostUpdate'); + + return async (packagePolicy) => { + if (!isEndpointPackagePolicy(packagePolicy)) { + return packagePolicy; + } + + logger.debug(`Processing endpoint integration policy (post update): ${packagePolicy.id}`); + + // The check below will run in the background - we don't need to wait for it + createPolicyDataStreamsIfNeeded({ + endpointServices, + endpointPolicyIds: [packagePolicy.id], + }).catch(() => {}); // to silence @typescript-eslint/no-floating-promises + + return packagePolicy; + }; +}; + export const getPackagePolicyPostCreateCallback = ( - logger: Logger, - exceptionsClient: ExceptionListClient | undefined + endpointServices: EndpointAppContextService ): PostPackagePolicyPostCreateCallback => { + const logger = endpointServices.createLogger('endpointPolicyPostCreate'); + const exceptionsClient = endpointServices.getExceptionListsClient(); + return async (packagePolicy: PackagePolicy): Promise => { // We only care about Endpoint package policies if (!exceptionsClient || !isEndpointPackagePolicy(packagePolicy)) { return packagePolicy; } + // Check and create internal datastreams for this policy if needed. + // NOTE: we don't need for it to complete here, thus no `await`. + createPolicyDataStreamsIfNeeded({ + endpointServices, + endpointPolicyIds: [packagePolicy.id], + }).catch(() => {}); // to silence @typescript-eslint/no-floating-promises + const integrationConfig = packagePolicy?.inputs[0]?.config?.integration_config; if (integrationConfig && integrationConfig?.value?.eventFilters !== undefined) { @@ -353,6 +415,31 @@ export const getAgentPolicyUpdateCallback = ( }; }; +export const getAgentPolicyPostUpdateCallback = ( + endpointServices: EndpointAppContextService +): PostAgentPolicyPostUpdateCallback => { + const logger = endpointServices.createLogger('endpointPolicyPostUpdate'); + + return async (agentPolicy) => { + const fleetServices = endpointServices.getInternalFleetServices(); + const endpointPolicy = await getEndpointPolicyForAgentPolicy(fleetServices, agentPolicy); + + if (!endpointPolicy) { + return agentPolicy; + } + + logger.debug(`Processing post-update to Fleet agent policy: [${agentPolicy.id}]`); + + // We don't need to `await` for this function to execute. It can be done in the background + createPolicyDataStreamsIfNeeded({ + endpointServices, + endpointPolicyIds: [endpointPolicy.id], + }).catch(() => {}); // to silence @typescript-eslint/no-floating-promises + + return agentPolicy; + }; +}; + export const getPackagePolicyDeleteCallback = ( exceptionsClient: ExceptionListClient | undefined, savedObjectsClient: SavedObjectsClientContract | undefined diff --git a/x-pack/plugins/security_solution/server/fleet_integration/handlers/create_policy_datastreams.test.ts b/x-pack/plugins/security_solution/server/fleet_integration/handlers/create_policy_datastreams.test.ts new file mode 100644 index 000000000000..0efaa5516a6f --- /dev/null +++ b/x-pack/plugins/security_solution/server/fleet_integration/handlers/create_policy_datastreams.test.ts @@ -0,0 +1,119 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { createMockEndpointAppContextService } from '../../endpoint/mocks'; +import type { ElasticsearchClientMock } from '@kbn/core-elasticsearch-client-server-mocks'; +import type { FetchEndpointPolicyNamespaceResponse } from '../../endpoint/services/fleet'; +import { createPolicyDataStreamsIfNeeded } from './create_policy_datastreams'; + +describe('createPolicyDataStreamsIfNeeded()', () => { + let endpointServicesMock: ReturnType; + let esClientMock: ElasticsearchClientMock; + let policyNamespacesMock: FetchEndpointPolicyNamespaceResponse; + + beforeEach(() => { + endpointServicesMock = createMockEndpointAppContextService(); + + esClientMock = endpointServicesMock.getInternalEsClient() as ElasticsearchClientMock; + esClientMock.indices.exists.mockResolvedValue(false); + + policyNamespacesMock = { integrationPolicy: { '123': ['foo1', 'foo2'] } }; + ( + endpointServicesMock.getInternalFleetServices().getPolicyNamespace as jest.Mock + ).mockResolvedValue(policyNamespacesMock); + }); + + afterEach(() => { + createPolicyDataStreamsIfNeeded.cache.deleteAll(); + }); + + it('should create datastreams if they do not exist', async () => { + await createPolicyDataStreamsIfNeeded({ + endpointServices: endpointServicesMock, + endpointPolicyIds: ['123'], + }); + + expect(esClientMock.indices.createDataStream).toHaveBeenCalledTimes(4); + [ + '.logs-endpoint.diagnostic.collection-foo1', + '.logs-endpoint.diagnostic.collection-foo2', + '.logs-endpoint.action.responses-foo1', + '.logs-endpoint.action.responses-foo2', + ].forEach((indexName) => { + expect(esClientMock.indices.createDataStream).toHaveBeenCalledWith({ + name: indexName, + }); + }); + }); + + it('should not create datastream if they already exist', async () => { + esClientMock.indices.exists.mockImplementation(async (options) => { + return ( + options.index === '.logs-endpoint.action.responses-foo1' || + options.index === '.logs-endpoint.diagnostic.collection-foo1' + ); + }); + + await createPolicyDataStreamsIfNeeded({ + endpointServices: endpointServicesMock, + endpointPolicyIds: ['123'], + }); + + expect(esClientMock.indices.createDataStream).toHaveBeenCalledTimes(2); + ['.logs-endpoint.diagnostic.collection-foo2', '.logs-endpoint.action.responses-foo2'].forEach( + (indexName) => { + expect(esClientMock.indices.createDataStream).toHaveBeenCalledWith({ + name: indexName, + }); + } + ); + }); + + it('should create heartbeat index when running in serverless', async () => { + (endpointServicesMock.isServerless as jest.Mock).mockReturnValue(true); + await createPolicyDataStreamsIfNeeded({ + endpointServices: endpointServicesMock, + endpointPolicyIds: ['123'], + }); + + expect(esClientMock.indices.createDataStream).toHaveBeenCalledTimes(6); + [ + '.logs-endpoint.diagnostic.collection-foo1', + '.logs-endpoint.diagnostic.collection-foo2', + '.logs-endpoint.action.responses-foo1', + '.logs-endpoint.action.responses-foo2', + '.logs-endpoint.heartbeat-foo1', + '.logs-endpoint.heartbeat-foo2', + ].forEach((indexName) => { + expect(esClientMock.indices.createDataStream).toHaveBeenCalledWith({ + name: indexName, + }); + }); + }); + + it('should not call ES if index existence was already checked', async () => { + createPolicyDataStreamsIfNeeded.cache.set('.logs-endpoint.action.responses-foo1', true); + await createPolicyDataStreamsIfNeeded({ + endpointServices: endpointServicesMock, + endpointPolicyIds: ['123'], + }); + + expect(esClientMock.indices.exists).not.toHaveBeenCalledWith({ + index: '.logs-endpoint.action.responses-foo1', + }); + expect(esClientMock.indices.createDataStream).toHaveBeenCalledTimes(3); + [ + '.logs-endpoint.diagnostic.collection-foo1', + '.logs-endpoint.diagnostic.collection-foo2', + '.logs-endpoint.action.responses-foo2', + ].forEach((indexName) => { + expect(esClientMock.indices.createDataStream).toHaveBeenCalledWith({ + name: indexName, + }); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/fleet_integration/handlers/create_policy_datastreams.ts b/x-pack/plugins/security_solution/server/fleet_integration/handlers/create_policy_datastreams.ts new file mode 100644 index 000000000000..93fec3526a7b --- /dev/null +++ b/x-pack/plugins/security_solution/server/fleet_integration/handlers/create_policy_datastreams.ts @@ -0,0 +1,159 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import pMap from 'p-map'; +import type { EndpointAppContextService } from '../../endpoint/endpoint_app_context_services'; +import { catchAndWrapError } from '../../endpoint/utils'; +import type { SimpleMemCacheInterface } from '../../endpoint/lib/simple_mem_cache'; +import { SimpleMemCache } from '../../endpoint/lib/simple_mem_cache'; +import { + ENDPOINT_ACTION_RESPONSES_DS, + ENDPOINT_HEARTBEAT_INDEX_PATTERN, +} from '../../../common/endpoint/constants'; +import { DEFAULT_DIAGNOSTIC_INDEX } from '../../lib/telemetry/constants'; +import { stringify } from '../../endpoint/utils/stringify'; + +const buildIndexNameWithNamespace = ( + indexNamePrefixOrPattern: string, + namespace: string +): string => { + if (indexNamePrefixOrPattern.endsWith('*')) { + const hasDash = indexNamePrefixOrPattern.endsWith('-*'); + return `${indexNamePrefixOrPattern.substring(0, indexNamePrefixOrPattern.length - 1)}${ + hasDash ? '' : '-' + }${namespace}`; + } + + return `${indexNamePrefixOrPattern}${ + indexNamePrefixOrPattern.endsWith('-') ? '' : '-' + }${namespace}`; +}; + +const cache = new SimpleMemCache({ + // Cache of created Datastreams last for 12h, at which point it is checked again. + // This is just a safeguard case (for whatever reason) the index is deleted + // 1.8e+7 === hours + ttl: 1.8e7, +}); + +interface PolicyDataStreamsCreator { + (options: CreatePolicyDataStreamsOptions): Promise; + cache: SimpleMemCacheInterface; +} + +export interface CreatePolicyDataStreamsOptions { + endpointServices: EndpointAppContextService; + endpointPolicyIds: string[]; +} + +/** + * Ensures that the DOT index Datastreams necessary to support Elastic Defend are crated (prior to + * endpoint writing data to them) + */ +export const createPolicyDataStreamsIfNeeded: PolicyDataStreamsCreator = async ({ + endpointServices, + endpointPolicyIds, +}: CreatePolicyDataStreamsOptions): Promise => { + const logger = endpointServices.createLogger('endpointPolicyDatastreamCreator'); + const esClient = endpointServices.getInternalEsClient(); + + logger.debug( + () => + `Checking if datastreams need to be created for Endpoint integration policy [${endpointPolicyIds.join( + ', ' + )}]` + ); + + // FIXME:PT Need to ensure that the datastreams are created in all associated space ids that the policy is shared with + // This can be deferred to activity around support of Spaces - team issue: 8199 (epic) + // We might need to do much here other than to ensure we can access all policies across all spaces in order to get the namespace value + + const fleetServices = endpointServices.getInternalFleetServices(); + const policyNamespaces = await fleetServices.getPolicyNamespace({ + integrationPolicies: endpointPolicyIds, + }); + const indexesCreated: string[] = []; + const createErrors: string[] = []; + const indicesToCreate: string[] = Object.values(policyNamespaces.integrationPolicy).reduce< + string[] + >((acc, namespaceList) => { + for (const namespace of namespaceList) { + acc.push( + buildIndexNameWithNamespace(DEFAULT_DIAGNOSTIC_INDEX, namespace), + buildIndexNameWithNamespace(ENDPOINT_ACTION_RESPONSES_DS, namespace) + ); + + if (endpointServices.isServerless()) { + acc.push(buildIndexNameWithNamespace(ENDPOINT_HEARTBEAT_INDEX_PATTERN, namespace)); + } + } + + return acc; + }, []); + + const processesDatastreamIndex = async (datastreamIndexName: string): Promise => { + if (cache.get(datastreamIndexName)) { + return; + } + + const doesDataStreamAlreadyExist = await esClient.indices + .exists({ index: datastreamIndexName }) + .catch(catchAndWrapError); + + if (doesDataStreamAlreadyExist) { + cache.set(datastreamIndexName, true); + return; + } + + await esClient.indices + .createDataStream({ name: datastreamIndexName }) + .then(() => { + indexesCreated.push(datastreamIndexName); + cache.set(datastreamIndexName, true); + }) + .catch((err) => { + // It's possible that between the `.exists()` check and this `.createDataStream()` that + // the index could have been created. If that's the case, then just ignore the error. + if (err.body?.error?.type === 'resource_already_exists_exception') { + cache.set(datastreamIndexName, true); + return; + } + + createErrors.push( + `Attempt to create datastream [${datastreamIndexName}] failed:\n${stringify( + err.body?.error ?? err + )}` + ); + }); + }; + + logger.debug( + () => + `Checking if the following datastream(s) need to be created:\n ${indicesToCreate.join( + '\n ' + )}` + ); + + await pMap(indicesToCreate, processesDatastreamIndex, { concurrency: 10 }); + + if (indexesCreated.length > 0) { + logger.info( + `Datastream(s) created in support of Elastic Defend policy [${endpointPolicyIds.join( + ', ' + )}]:\n ${indexesCreated.join('\n ')}` + ); + } else if (createErrors.length === 0) { + logger.debug(() => `Nothing to do. Datastreams already exist`); + } + + if (createErrors.length > 0) { + logger.error( + `${createErrors.length} errors encountered:\n${createErrors.join('\n--------\n')}` + ); + } +}; +createPolicyDataStreamsIfNeeded.cache = cache;