diff --git a/x-pack/plugins/fleet/server/mocks/index.ts b/x-pack/plugins/fleet/server/mocks/index.ts index 43b113899072e..ac806c1448a24 100644 --- a/x-pack/plugins/fleet/server/mocks/index.ts +++ b/x-pack/plugins/fleet/server/mocks/index.ts @@ -186,7 +186,7 @@ export const createPackagePolicyServiceMock = (): jest.Mocked => { return { - get: jest.fn(), - list: jest.fn(), - getFullAgentPolicy: jest.fn(), - getByIds: jest.fn(), - turnOffAgentTamperProtections: jest.fn(), - fetchAllAgentPolicies: jest.fn(), - fetchAllAgentPolicyIds: jest.fn(), + get: jest.fn().mockReturnValue(Promise.resolve()), + list: jest.fn().mockReturnValue(Promise.resolve()), + getFullAgentPolicy: jest.fn().mockReturnValue(Promise.resolve()), + getByIds: jest.fn().mockReturnValue(Promise.resolve()), + turnOffAgentTamperProtections: jest.fn().mockReturnValue(Promise.resolve()), + fetchAllAgentPolicies: jest.fn().mockReturnValue(Promise.resolve()), + fetchAllAgentPolicyIds: jest.fn().mockReturnValue(Promise.resolve()), }; }; diff --git a/x-pack/plugins/fleet/server/services/agent_policy.test.ts b/x-pack/plugins/fleet/server/services/agent_policy.test.ts index 00bc01aa1f2cb..608b6f739fc29 100644 --- a/x-pack/plugins/fleet/server/services/agent_policy.test.ts +++ b/x-pack/plugins/fleet/server/services/agent_policy.test.ts @@ -1285,6 +1285,38 @@ describe('Agent policy', () => { }) ).resolves.not.toThrow(); }); + + it('should run external "agentPolicyPostUpdate" callbacks when update is successful', async () => { + const soClient = getAgentPolicyCreateMock(); + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + + const postUpdateCallback = jest.fn(async (policy) => policy); + mockedAppContextService.getExternalCallbacks.mockImplementation((type) => { + if (type === 'agentPolicyPostUpdate') { + return new Set([postUpdateCallback]); + } + }); + + soClient.get.mockResolvedValue({ + attributes: {}, + id: 'test-id', + type: 'mocked', + references: [], + }); + + await expect( + agentPolicyService.update(soClient, esClient, 'test-id', { + name: 'test', + namespace: 'default', + }) + ).resolves.not.toThrow(); + + expect(mockedAppContextService.getExternalCallbacks).toHaveBeenCalledWith( + 'agentPolicyPostUpdate' + ); + + expect(postUpdateCallback).toHaveBeenCalled(); + }); }); describe('deployPolicy', () => { diff --git a/x-pack/plugins/fleet/server/services/agent_policy.ts b/x-pack/plugins/fleet/server/services/agent_policy.ts index f377f95190df0..1493f55cf6909 100644 --- a/x-pack/plugins/fleet/server/services/agent_policy.ts +++ b/x-pack/plugins/fleet/server/services/agent_policy.ts @@ -61,6 +61,7 @@ import type { PostAgentPolicyCreateCallback, PostAgentPolicyUpdateCallback, PreconfiguredAgentPolicy, + PostAgentPolicyPostUpdateCallback, } from '../types'; import { AGENT_POLICY_INDEX, @@ -306,8 +307,8 @@ class AgentPolicyService { public async runExternalCallbacks( externalCallbackType: ExternalCallback[0], - agentPolicy: NewAgentPolicy | Partial - ): Promise> { + agentPolicy: NewAgentPolicy | Partial | AgentPolicy + ): Promise | AgentPolicy> { const logger = appContextService.getLogger(); logger.debug(`Running external callbacks for ${externalCallbackType}`); try { @@ -330,6 +331,12 @@ class AgentPolicyService { ); updatedNewAgentPolicy = result; } + if (externalCallbackType === 'agentPolicyPostUpdate') { + result = await (callback as PostAgentPolicyPostUpdateCallback)( + newAgentPolicy as AgentPolicy + ); + updatedNewAgentPolicy = result; + } } newAgentPolicy = updatedNewAgentPolicy; } @@ -738,6 +745,11 @@ class AgentPolicyService { bumpRevision: true, removeProtection: false, skipValidation: options?.skipValidation ?? false, + }).then((updatedAgentPolicy) => { + return this.runExternalCallbacks( + 'agentPolicyPostUpdate', + updatedAgentPolicy + ) as unknown as AgentPolicy; }); } diff --git a/x-pack/plugins/fleet/server/services/package_policy.test.ts b/x-pack/plugins/fleet/server/services/package_policy.test.ts index 8c322d997a612..30523448e721d 100644 --- a/x-pack/plugins/fleet/server/services/package_policy.test.ts +++ b/x-pack/plugins/fleet/server/services/package_policy.test.ts @@ -1718,6 +1718,41 @@ describe('Package policy service', () => { }); }); + it('should run "packagePolicyPostUpdate" external callbacks', async () => { + const soClient = savedObjectsClientMock.create(); + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + const mockPackagePolicy = createPackagePolicyMock(); + const attributes = { + ...mockPackagePolicy, + inputs: [], + }; + + jest.spyOn(appContextService, 'getExternalCallbacks'); + + soClient.get.mockResolvedValue({ + id: 'test-package-policy', + type: LEGACY_PACKAGE_POLICY_SAVED_OBJECT_TYPE, + references: [], + attributes, + }); + + soClient.update.mockResolvedValue({ + id: 'test-package-policy', + type: LEGACY_PACKAGE_POLICY_SAVED_OBJECT_TYPE, + references: [], + attributes, + }); + + await packagePolicyService.update(soClient, esClient, 'test-package-policy', { + ...mockPackagePolicy, + inputs: [], + }); + + expect(appContextService.getExternalCallbacks).toHaveBeenCalledWith( + 'packagePolicyPostUpdate' + ); + }); + describe('remove protections', () => { beforeEach(() => { mockAgentPolicyService.bumpRevision.mockReset(); diff --git a/x-pack/plugins/fleet/server/services/package_policy.ts b/x-pack/plugins/fleet/server/services/package_policy.ts index ff1c4abcef4e9..981799329ddbe 100644 --- a/x-pack/plugins/fleet/server/services/package_policy.ts +++ b/x-pack/plugins/fleet/server/services/package_policy.ts @@ -94,6 +94,7 @@ import type { DryRunPackagePolicy, PostPackagePolicyCreateCallback, PostPackagePolicyPostCreateCallback, + PutPackagePolicyPostUpdateCallback, } from '../types'; import type { ExternalCallback } from '..'; @@ -127,6 +128,8 @@ import type { PackagePolicyClient, PackagePolicyClientFetchAllItemsOptions, PackagePolicyService, + RunExternalCallbacksPackagePolicyArgument, + RunExternalCallbacksPackagePolicyResponse, } from './package_policy_service'; import { installAssetsForInputPackagePolicy } from './epm/packages/install'; import { auditLoggingService } from './audit_logging'; @@ -1091,9 +1094,16 @@ class PackagePolicyClientImpl implements PackagePolicyClient { await Promise.all([...bumpPromises, assetRemovePromise, deleteSecretsPromise]); sendUpdatePackagePolicyTelemetryEvent(soClient, [packagePolicyUpdate], [oldPackagePolicy]); + logger.debug(`Package policy ${id} update completed`); - return newPolicy; + // Run external post-update callbacks and return + return packagePolicyService.runExternalCallbacks( + 'packagePolicyPostUpdate', + newPolicy, + soClient, + esClient + ); } public async bulkUpdate( @@ -1930,48 +1940,21 @@ class PackagePolicyClientImpl implements PackagePolicyClient { public async runExternalCallbacks( externalCallbackType: A, - packagePolicy: A extends 'packagePolicyDelete' - ? DeletePackagePoliciesResponse - : A extends 'packagePolicyPostDelete' - ? PostDeletePackagePoliciesResponse - : A extends 'packagePolicyPostCreate' - ? PackagePolicy - : A extends 'packagePolicyCreate' - ? NewPackagePolicy - : never, + packagePolicy: RunExternalCallbacksPackagePolicyArgument, soClient: SavedObjectsClientContract, esClient: ElasticsearchClient, context?: RequestHandlerContext, request?: KibanaRequest - ): Promise< - A extends 'packagePolicyDelete' - ? void - : A extends 'packagePolicyPostDelete' - ? void - : A extends 'packagePolicyPostCreate' - ? PackagePolicy - : A extends 'packagePolicyCreate' - ? NewPackagePolicy - : never - >; - public async runExternalCallbacks( - externalCallbackType: ExternalCallback[0], - packagePolicy: - | PackagePolicy - | NewPackagePolicy - | PostDeletePackagePoliciesResponse - | DeletePackagePoliciesResponse, - soClient: SavedObjectsClientContract, - esClient: ElasticsearchClient, - context?: RequestHandlerContext, - request?: KibanaRequest - ): Promise { + ): Promise> { const logger = appContextService.getLogger(); const numberOfCallbacks = appContextService.getExternalCallbacks(externalCallbackType)?.size; + let runResult: any; + logger.debug(`Running ${numberOfCallbacks} external callbacks for ${externalCallbackType}`); + try { if (externalCallbackType === 'packagePolicyPostDelete') { - return await this.runPostDeleteExternalCallbacks( + runResult = await this.runPostDeleteExternalCallbacks( packagePolicy as PostDeletePackagePoliciesResponse, soClient, esClient, @@ -1979,7 +1962,7 @@ class PackagePolicyClientImpl implements PackagePolicyClient { request ); } else if (externalCallbackType === 'packagePolicyDelete') { - return await this.runDeleteExternalCallbacks( + runResult = await this.runDeleteExternalCallbacks( packagePolicy as DeletePackagePoliciesResponse, soClient, esClient @@ -1988,21 +1971,33 @@ class PackagePolicyClientImpl implements PackagePolicyClient { if (!Array.isArray(packagePolicy)) { let newData = packagePolicy; const externalCallbacks = appContextService.getExternalCallbacks(externalCallbackType); + if (externalCallbacks && externalCallbacks.size > 0) { - let updatedNewData = newData; + let updatedNewData: any = newData; + for (const callback of externalCallbacks) { - let result; + let thisCallbackResponse; + if (externalCallbackType === 'packagePolicyPostCreate') { - result = await (callback as PostPackagePolicyPostCreateCallback)( + thisCallbackResponse = await (callback as PostPackagePolicyPostCreateCallback)( updatedNewData as PackagePolicy, soClient, esClient, context, request ); - updatedNewData = PackagePolicySchema.validate(result) as NewPackagePolicy; + updatedNewData = PackagePolicySchema.validate(thisCallbackResponse); + } else if (externalCallbackType === 'packagePolicyPostUpdate') { + thisCallbackResponse = await (callback as PutPackagePolicyPostUpdateCallback)( + updatedNewData as PackagePolicy, + soClient, + esClient, + context, + request + ); + updatedNewData = PackagePolicySchema.validate(thisCallbackResponse); } else { - result = await (callback as PostPackagePolicyCreateCallback)( + thisCallbackResponse = await (callback as PostPackagePolicyCreateCallback)( updatedNewData as NewPackagePolicy, soClient, esClient, @@ -2012,10 +2007,10 @@ class PackagePolicyClientImpl implements PackagePolicyClient { } if (externalCallbackType === 'packagePolicyCreate') { - updatedNewData = NewPackagePolicySchema.validate(result) as NewPackagePolicy; + updatedNewData = NewPackagePolicySchema.validate(thisCallbackResponse); } else if (externalCallbackType === 'packagePolicyUpdate') { const omitted = { - ...omit(result, [ + ...omit(thisCallbackResponse, [ 'id', 'spaceIds', 'version', @@ -2026,16 +2021,19 @@ class PackagePolicyClientImpl implements PackagePolicyClient { 'created_by', 'elasticsearch', ]), - inputs: result.inputs.map((input) => omit(input, ['compiled_input'])), + inputs: thisCallbackResponse.inputs.map((input) => + omit(input, ['compiled_input']) + ), }; - updatedNewData = UpdatePackagePolicySchema.validate(omitted) as PackagePolicy; + updatedNewData = UpdatePackagePolicySchema.validate(omitted); } } newData = updatedNewData; } - return newData; + + runResult = newData; } } } catch (error) { @@ -2043,6 +2041,8 @@ class PackagePolicyClientImpl implements PackagePolicyClient { logger.error(error); throw error; } + + return runResult as unknown as RunExternalCallbacksPackagePolicyResponse; } public async runPostDeleteExternalCallbacks( diff --git a/x-pack/plugins/fleet/server/services/package_policy_service.ts b/x-pack/plugins/fleet/server/services/package_policy_service.ts index f5cb879cef7cb..967efb1055cfb 100644 --- a/x-pack/plugins/fleet/server/services/package_policy_service.ts +++ b/x-pack/plugins/fleet/server/services/package_policy_service.ts @@ -40,6 +40,36 @@ export interface PackagePolicyService { get asInternalUser(): PackagePolicyClient; } +export type RunExternalCallbacksPackagePolicyArgument = + A extends 'packagePolicyDelete' + ? DeletePackagePoliciesResponse + : A extends 'packagePolicyPostDelete' + ? PostDeletePackagePoliciesResponse + : A extends 'packagePolicyCreate' + ? NewPackagePolicy + : A extends 'packagePolicyPostCreate' + ? PackagePolicy + : A extends 'packagePolicyUpdate' + ? UpdatePackagePolicy + : A extends 'packagePolicyPostUpdate' + ? PackagePolicy + : never; + +export type RunExternalCallbacksPackagePolicyResponse = + A extends 'packagePolicyDelete' + ? void + : A extends 'packagePolicyPostDelete' + ? void + : A extends 'packagePolicyCreate' + ? NewPackagePolicy + : A extends 'packagePolicyPostCreate' + ? PackagePolicy + : A extends 'packagePolicyUpdate' + ? UpdatePackagePolicy + : A extends 'packagePolicyPostUpdate' + ? PackagePolicy + : undefined; + export interface PackagePolicyClient { create( soClient: SavedObjectsClientContract, @@ -169,30 +199,12 @@ export interface PackagePolicyClient { runExternalCallbacks( externalCallbackType: A, - packagePolicy: A extends 'packagePolicyDelete' - ? DeletePackagePoliciesResponse - : A extends 'packagePolicyPostDelete' - ? PostDeletePackagePoliciesResponse - : A extends 'packagePolicyPostCreate' - ? PackagePolicy - : A extends 'packagePolicyUpdate' - ? UpdatePackagePolicy - : NewPackagePolicy, + packagePolicy: RunExternalCallbacksPackagePolicyArgument, soClient: SavedObjectsClientContract, esClient: ElasticsearchClient, context?: RequestHandlerContext, request?: KibanaRequest - ): Promise< - A extends 'packagePolicyDelete' - ? void - : A extends 'packagePolicyPostDelete' - ? void - : A extends 'packagePolicyPostCreate' - ? PackagePolicy - : A extends 'packagePolicyUpdate' - ? UpdatePackagePolicy - : NewPackagePolicy - >; + ): Promise>; runDeleteExternalCallbacks( deletedPackagePolicies: DeletePackagePoliciesResponse, diff --git a/x-pack/plugins/fleet/server/types/extensions.ts b/x-pack/plugins/fleet/server/types/extensions.ts index 594e16f619556..2293747b253e3 100644 --- a/x-pack/plugins/fleet/server/types/extensions.ts +++ b/x-pack/plugins/fleet/server/types/extensions.ts @@ -60,6 +60,14 @@ export type PutPackagePolicyUpdateCallback = ( request?: KibanaRequest ) => Promise; +export type PutPackagePolicyPostUpdateCallback = ( + packagePolicy: PackagePolicy, + soClient: SavedObjectsClientContract, + esClient: ElasticsearchClient, + context?: RequestHandlerContext, + request?: KibanaRequest +) => Promise; + export type PostAgentPolicyCreateCallback = ( agentPolicy: NewAgentPolicy ) => Promise; @@ -68,6 +76,8 @@ export type PostAgentPolicyUpdateCallback = ( agentPolicy: Partial ) => Promise>; +export type PostAgentPolicyPostUpdateCallback = (agentPolicy: AgentPolicy) => Promise; + export type ExternalCallbackCreate = ['packagePolicyCreate', PostPackagePolicyCreateCallback]; export type ExternalCallbackPostCreate = [ 'packagePolicyPostCreate', @@ -79,7 +89,12 @@ export type ExternalCallbackPostDelete = [ 'packagePolicyPostDelete', PostPackagePolicyPostDeleteCallback ]; + export type ExternalCallbackUpdate = ['packagePolicyUpdate', PutPackagePolicyUpdateCallback]; +export type ExternalCallbackPostUpdate = [ + 'packagePolicyPostUpdate', + PutPackagePolicyPostUpdateCallback +]; export type ExternalCallbackAgentPolicyCreate = [ 'agentPolicyCreate', @@ -89,6 +104,10 @@ export type ExternalCallbackAgentPolicyUpdate = [ 'agentPolicyUpdate', PostAgentPolicyUpdateCallback ]; +export type ExternalCallbackAgentPolicyPostUpdate = [ + 'agentPolicyPostUpdate', + PostAgentPolicyPostUpdateCallback +]; /** * Callbacks supported by the Fleet plugin @@ -99,7 +118,9 @@ export type ExternalCallback = | ExternalCallbackDelete | ExternalCallbackPostDelete | ExternalCallbackUpdate + | ExternalCallbackPostUpdate | ExternalCallbackAgentPolicyCreate - | ExternalCallbackAgentPolicyUpdate; + | ExternalCallbackAgentPolicyUpdate + | ExternalCallbackAgentPolicyPostUpdate; export type ExternalCallbacksStorage = Map>; diff --git a/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.ts b/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.ts index 192fb6059325a..1afa24ebbd529 100644 --- a/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.ts +++ b/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.ts @@ -31,10 +31,12 @@ import type { ResponseActionsClient } from './services'; import { getResponseActionsClient, NormalizedExternalConnectorClient } from './services'; import { getAgentPolicyCreateCallback, + getAgentPolicyPostUpdateCallback, getAgentPolicyUpdateCallback, getPackagePolicyCreateCallback, getPackagePolicyDeleteCallback, getPackagePolicyPostCreateCallback, + getPackagePolicyPostUpdateCallback, getPackagePolicyUpdateCallback, } from '../fleet_integration/fleet_integration'; import type { ManifestManager } from './services/artifacts'; @@ -117,7 +119,8 @@ export class EndpointAppContextService { this.savedObjectsFactoryService = savedObjectsFactory; this.fleetServicesFactory = new EndpointFleetServicesFactory( dependencies.fleetStartServices, - savedObjectsFactory + savedObjectsFactory, + this.createLogger('endpointFleetServices') ); this.registerFleetExtensions(); @@ -169,6 +172,8 @@ export class EndpointAppContextService { getAgentPolicyUpdateCallback(logger, productFeaturesService) ); + registerFleetCallback('agentPolicyPostUpdate', getAgentPolicyPostUpdateCallback(this)); + registerFleetCallback( 'packagePolicyCreate', getPackagePolicyCreateCallback( @@ -183,10 +188,7 @@ export class EndpointAppContextService { ) ); - registerFleetCallback( - 'packagePolicyPostCreate', - getPackagePolicyPostCreateCallback(logger, exceptionListsClient) - ); + registerFleetCallback('packagePolicyPostCreate', getPackagePolicyPostCreateCallback(this)); registerFleetCallback( 'packagePolicyUpdate', @@ -201,6 +203,8 @@ export class EndpointAppContextService { ) ); + registerFleetCallback('packagePolicyPostUpdate', getPackagePolicyPostUpdateCallback(this)); + registerFleetCallback( 'packagePolicyPostDelete', getPackagePolicyDeleteCallback(exceptionListsClient, soClient) @@ -218,6 +222,27 @@ export class EndpointAppContextService { return this.savedObjectsFactoryService; } + /** + * Is kibana running in serverless mode + */ + public isServerless(): boolean { + if (!this.setupDependencies) { + throw new EndpointAppContentServicesNotSetUpError(); + } + + // TODO:PT check what this returns when running locally with kibana in serverless emulation + + return Boolean(this.setupDependencies.cloud.isServerlessEnabled); + } + + public getInternalEsClient(): ElasticsearchClient { + if (!this.startDependencies?.esClient) { + throw new EndpointAppContentServicesNotStartedError(); + } + + return this.startDependencies.esClient; + } + private getFleetAuthzService(): FleetStartContract['authz'] { if (!this.startDependencies?.fleetStartServices) { throw new EndpointAppContentServicesNotStartedError(); diff --git a/x-pack/plugins/security_solution/server/endpoint/services/actions/clients/lib/simple_mem_cache.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/simple_mem_cache.test.ts similarity index 92% rename from x-pack/plugins/security_solution/server/endpoint/services/actions/clients/lib/simple_mem_cache.test.ts rename to x-pack/plugins/security_solution/server/endpoint/lib/simple_mem_cache.test.ts index f351e2e40d5be..27ba3bdf23945 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/actions/clients/lib/simple_mem_cache.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/simple_mem_cache.test.ts @@ -51,6 +51,13 @@ describe('SimpleMemCache class', () => { expect(cache.get(key)).toEqual(undefined); }); + it('should delete all entries from cache', () => { + cache.set(key, value); + cache.deleteAll(); + + expect(cache.get(key)).toEqual(undefined); + }); + it('should cleanup expired cache entries', () => { const key2 = 'myKey'; cache.set(key, value); // Default ttl of 10s diff --git a/x-pack/plugins/security_solution/server/endpoint/services/actions/clients/lib/simple_mem_cache.ts b/x-pack/plugins/security_solution/server/endpoint/lib/simple_mem_cache.ts similarity index 95% rename from x-pack/plugins/security_solution/server/endpoint/services/actions/clients/lib/simple_mem_cache.ts rename to x-pack/plugins/security_solution/server/endpoint/lib/simple_mem_cache.ts index fc355bf6c3797..a65a1ee6be71a 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/actions/clients/lib/simple_mem_cache.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/simple_mem_cache.ts @@ -19,6 +19,8 @@ export interface SimpleMemCacheInterface { get(key: any): TValue | undefined; /** Delete a piece of data from cache */ delete(key: any): void; + /** Delete all cached entries */ + deleteAll(): void; /** Clean up the cache by removing all expired entries */ cleanup(): void; } @@ -79,6 +81,10 @@ export class SimpleMemCache implements SimpleMemCacheInterface { this.cache.delete(key); } + public deleteAll(): void { + this.cache.clear(); + } + public cleanup(): void { for (const [cacheKey, cacheData] of this.cache.entries()) { if (this.isExpired(cacheData)) { diff --git a/x-pack/plugins/security_solution/server/endpoint/mocks/mocks.ts b/x-pack/plugins/security_solution/server/endpoint/mocks/mocks.ts index 5ab221b7bfc07..91a2bc40454b9 100644 --- a/x-pack/plugins/security_solution/server/endpoint/mocks/mocks.ts +++ b/x-pack/plugins/security_solution/server/endpoint/mocks/mocks.ts @@ -144,6 +144,8 @@ export const createMockEndpointAppContextService = ( return responseActionsClientMock.create(); }), savedObjects: createSavedObjectsClientFactoryMock({ savedObjectsServiceStart }).service, + isServerless: jest.fn().mockReturnValue(false), + getInternalEsClient: jest.fn().mockReturnValue(esClient), } as unknown as jest.Mocked; }; diff --git a/x-pack/plugins/security_solution/server/endpoint/services/actions/clients/lib/base_response_actions_client.ts b/x-pack/plugins/security_solution/server/endpoint/services/actions/clients/lib/base_response_actions_client.ts index 0411e4a9c8f65..7a8f14b6e9a8e 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/actions/clients/lib/base_response_actions_client.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/actions/clients/lib/base_response_actions_client.ts @@ -20,7 +20,7 @@ import { } from '../../../../../lib/telemetry/event_based/events'; import { NotFoundError } from '../../../../errors'; import { fetchActionRequestById } from '../../utils/fetch_action_request_by_id'; -import { SimpleMemCache } from './simple_mem_cache'; +import { SimpleMemCache } from '../../../../lib/simple_mem_cache'; import { fetchActionResponses, fetchEndpointActionResponses, @@ -581,6 +581,10 @@ export abstract class ResponseActionsClientImpl implements ResponseActionsClient >( options: ResponseActionsClientWriteActionResponseToEndpointIndexOptions ): Promise> { + // FIXME:PT need to ensure we use a index below that has the proper `namespace` when agent type is Endpoint + // Background: Endpoint responses require that the document be written to an index that has the + // correct `namespace` as defined by the Integration/Agent policy and that logic is not currently implemented. + const doc = this.buildActionResponseEsDoc(options); this.log.debug(() => `Writing response action response:\n${stringify(doc)}`); @@ -594,7 +598,7 @@ export abstract class ResponseActionsClientImpl implements ResponseActionsClient .catch((err) => { throw new ResponseActionsClientError( `Failed to create action response document: ${err.message}`, - err.statusCode ?? 500, + 500, err ); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/services/fleet/endpoint_fleet_services_factory.mocks.ts b/x-pack/plugins/security_solution/server/endpoint/services/fleet/endpoint_fleet_services_factory.mocks.ts index 91119ea3df5fb..302528b024f76 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/fleet/endpoint_fleet_services_factory.mocks.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/fleet/endpoint_fleet_services_factory.mocks.ts @@ -8,6 +8,8 @@ import type { DeeplyMockedKeys } from '@kbn/utility-types-jest'; import type { FleetStartContract } from '@kbn/fleet-plugin/server'; import { createFleetStartContractMock } from '@kbn/fleet-plugin/server/mocks'; +import type { MockedLogger } from '@kbn/logging-mocks'; +import { loggingSystemMock } from '@kbn/core-logging-server-mocks'; import type { SavedObjectsClientFactory } from '../saved_objects'; import type { EndpointFleetServicesFactoryInterface, @@ -24,35 +26,41 @@ export interface EndpointFleetServicesFactoryInterfaceMocked asInternalUser: () => EndpointInternalFleetServicesInterfaceMocked; } -interface CreateEndpointFleetServicesFactoryMockOptions { +export interface CreateEndpointFleetServicesFactoryMockOptions { fleetDependencies: DeeplyMockedKeys; savedObjects: SavedObjectsClientFactory; + logger: MockedLogger; } -export const createEndpointFleetServicesFactoryMock = ( - dependencies: Partial = {} -): { +export interface CreateEndpointFleetServicesFactoryResponse { service: EndpointFleetServicesFactoryInterfaceMocked; dependencies: CreateEndpointFleetServicesFactoryMockOptions; -} => { +} + +export const createEndpointFleetServicesFactoryMock = ( + dependencies: Partial = {} +): CreateEndpointFleetServicesFactoryResponse => { const { fleetDependencies = createFleetStartContractMock(), savedObjects = createSavedObjectsClientFactoryMock().service, + logger = loggingSystemMock.createLogger(), } = dependencies; const serviceFactoryMock = new EndpointFleetServicesFactory( fleetDependencies, - savedObjects + savedObjects, + logger ) as unknown as EndpointFleetServicesFactoryInterfaceMocked; const fleetInternalServicesMocked = serviceFactoryMock.asInternalUser(); jest.spyOn(fleetInternalServicesMocked, 'ensureInCurrentSpace'); + jest.spyOn(fleetInternalServicesMocked, 'getPolicyNamespace'); const asInternalUserSpy = jest.spyOn(serviceFactoryMock, 'asInternalUser'); asInternalUserSpy.mockReturnValue(fleetInternalServicesMocked); return { service: serviceFactoryMock, - dependencies: { fleetDependencies, savedObjects }, + dependencies: { fleetDependencies, savedObjects, logger }, }; }; diff --git a/x-pack/plugins/security_solution/server/endpoint/services/fleet/endpoint_fleet_services_factory.test.ts b/x-pack/plugins/security_solution/server/endpoint/services/fleet/endpoint_fleet_services_factory.test.ts new file mode 100644 index 0000000000000..c1f7ca004e03e --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/services/fleet/endpoint_fleet_services_factory.test.ts @@ -0,0 +1,181 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { + CreateEndpointFleetServicesFactoryResponse, + EndpointInternalFleetServicesInterfaceMocked, +} from './endpoint_fleet_services_factory.mocks'; +import { createEndpointFleetServicesFactoryMock } from './endpoint_fleet_services_factory.mocks'; +import { AgentNotFoundError } from '@kbn/fleet-plugin/server'; +import { NotFoundError } from '../../errors'; +import type { AgentPolicy, PackagePolicy } from '@kbn/fleet-plugin/common'; +import { FleetAgentPolicyGenerator } from '../../../../common/endpoint/data_generators/fleet_agent_policy_generator'; +import { FleetPackagePolicyGenerator } from '../../../../common/endpoint/data_generators/fleet_package_policy_generator'; + +describe('EndpointServiceFactory', () => { + let fleetServicesMock: EndpointInternalFleetServicesInterfaceMocked; + let fleetServicesFactoryMock: CreateEndpointFleetServicesFactoryResponse; + + beforeEach(() => { + fleetServicesFactoryMock = createEndpointFleetServicesFactoryMock(); + fleetServicesMock = fleetServicesFactoryMock.service.asInternalUser(); + }); + + it('should return fleet services when `asInternalUser()` is invoked', () => { + expect(Object.keys(fleetServicesMock)).toEqual([ + 'agent', + 'agentPolicy', + 'packages', + 'packagePolicy', + 'savedObjects', + 'endpointPolicyKuery', + 'ensureInCurrentSpace', + 'getPolicyNamespace', + ]); + }); + + describe('#ensureInCurentSpace()', () => { + it('should check agent ids', async () => { + await expect( + fleetServicesMock.ensureInCurrentSpace({ agentIds: ['123'] }) + ).resolves.toBeUndefined(); + expect( + fleetServicesFactoryMock.dependencies.fleetDependencies.agentService.asInternalUser.getByIds + ).toHaveBeenCalledWith(['123']); + expect( + fleetServicesFactoryMock.dependencies.fleetDependencies.agentPolicyService.getByIds + ).not.toHaveBeenCalled(); + expect( + fleetServicesFactoryMock.dependencies.fleetDependencies.packagePolicyService.getByIDs + ).not.toHaveBeenCalled(); + }); + + it('should check integration policy ids', async () => { + await expect( + fleetServicesMock.ensureInCurrentSpace({ integrationPolicyIds: ['123'] }) + ).resolves.toBeUndefined(); + expect( + fleetServicesFactoryMock.dependencies.fleetDependencies.agentService.asInternalUser.getByIds + ).not.toHaveBeenCalled(); + expect( + fleetServicesFactoryMock.dependencies.fleetDependencies.agentPolicyService.getByIds + ).not.toHaveBeenCalled(); + expect( + fleetServicesFactoryMock.dependencies.fleetDependencies.packagePolicyService.getByIDs + ).toHaveBeenCalledWith(expect.anything(), ['123']); + }); + + it('should check agent policy ids', async () => { + await expect( + fleetServicesMock.ensureInCurrentSpace({ agentPolicyIds: ['123'] }) + ).resolves.toBeUndefined(); + expect( + fleetServicesFactoryMock.dependencies.fleetDependencies.agentService.asInternalUser.getByIds + ).not.toHaveBeenCalled(); + expect( + fleetServicesFactoryMock.dependencies.fleetDependencies.agentPolicyService.getByIds + ).toHaveBeenCalledWith(expect.anything(), ['123']); + expect( + fleetServicesFactoryMock.dependencies.fleetDependencies.packagePolicyService.getByIDs + ).not.toHaveBeenCalled(); + }); + + it('should check agent Ids, integration policy id and agent policy ids', async () => { + await expect( + fleetServicesMock.ensureInCurrentSpace({ + integrationPolicyIds: ['123'], + agentIds: ['123'], + agentPolicyIds: ['123'], + }) + ).resolves.toBeUndefined(); + expect( + fleetServicesFactoryMock.dependencies.fleetDependencies.agentService.asInternalUser.getByIds + ).toHaveBeenCalledWith(['123']); + expect( + fleetServicesFactoryMock.dependencies.fleetDependencies.agentPolicyService.getByIds + ).toHaveBeenCalledWith(expect.anything(), ['123']); + expect( + fleetServicesFactoryMock.dependencies.fleetDependencies.packagePolicyService.getByIDs + ).toHaveBeenCalledWith(expect.anything(), ['123']); + }); + + it('should throw error any of the data is not visible in current space', async () => { + fleetServicesFactoryMock.dependencies.fleetDependencies.agentService.asInternalUser.getByIds.mockImplementation( + async () => { + throw new AgentNotFoundError('not found mock'); + } + ); + await expect( + fleetServicesMock.ensureInCurrentSpace({ + integrationPolicyIds: ['123'], + agentIds: ['123'], + agentPolicyIds: ['123'], + }) + ).rejects.toThrowError(NotFoundError); + }); + }); + + describe('#getPolicyNamespace()', () => { + let integrationPolicy: PackagePolicy; + let agentPolicy1: AgentPolicy; + let agentPolicy2: AgentPolicy; + + beforeEach(() => { + const agentPolicyGenerator = new FleetAgentPolicyGenerator('seed'); + const integrationPolicyGenerator = new FleetPackagePolicyGenerator('seed'); + + agentPolicy1 = agentPolicyGenerator.generate({ namespace: 'foo1' }); + agentPolicy2 = agentPolicyGenerator.generate({ namespace: 'foo2' }); + integrationPolicy = integrationPolicyGenerator.generate({ + namespace: undefined, + policy_ids: [agentPolicy1.id, agentPolicy2.id], + }); + + fleetServicesFactoryMock.dependencies.fleetDependencies.packagePolicyService.getByIDs.mockResolvedValue( + [integrationPolicy] + ); + fleetServicesFactoryMock.dependencies.fleetDependencies.agentPolicyService.getByIds.mockResolvedValue( + [agentPolicy1, agentPolicy2] + ); + }); + + it('should return namespace from agent policies if integration policy does not have one defined', async () => { + await expect( + fleetServicesMock.getPolicyNamespace({ + integrationPolicies: [integrationPolicy.id], + }) + ).resolves.toEqual({ + integrationPolicy: { + [integrationPolicy.id]: ['foo1', 'foo2'], + }, + }); + expect( + fleetServicesFactoryMock.dependencies.fleetDependencies.agentPolicyService.getByIds + ).toHaveBeenCalledWith(expect.anything(), [agentPolicy1.id, agentPolicy2.id]); + }); + + it('should return namespace from integration policy if defined', async () => { + integrationPolicy.namespace = 'bar'; + + await expect( + fleetServicesMock.getPolicyNamespace({ + integrationPolicies: [integrationPolicy.id], + }) + ).resolves.toEqual({ + integrationPolicy: { + [integrationPolicy.id]: ['bar'], + }, + }); + + // The agentPolicy sevice should not have been called because the package policy has + // a namespace id, so no need. + expect( + fleetServicesFactoryMock.dependencies.fleetDependencies.agentPolicyService.getByIds + ).not.toHaveBeenCalled(); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/endpoint/services/fleet/endpoint_fleet_services_factory.ts b/x-pack/plugins/security_solution/server/endpoint/services/fleet/endpoint_fleet_services_factory.ts index 50e2006272218..bbda061b3ceff 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/fleet/endpoint_fleet_services_factory.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/fleet/endpoint_fleet_services_factory.ts @@ -13,12 +13,14 @@ import type { PackageClient, } from '@kbn/fleet-plugin/server'; import { AgentNotFoundError } from '@kbn/fleet-plugin/server'; -import { PACKAGE_POLICY_SAVED_OBJECT_TYPE } from '@kbn/fleet-plugin/common'; -import type { SavedObjectsClientContract } from '@kbn/core/server'; +import type { AgentPolicy } from '@kbn/fleet-plugin/common'; +import { PACKAGE_POLICY_SAVED_OBJECT_TYPE, type PackagePolicy } from '@kbn/fleet-plugin/common'; +import type { Logger, SavedObjectsClientContract } from '@kbn/core/server'; import { AgentPolicyNotFoundError, PackagePolicyNotFoundError, } from '@kbn/fleet-plugin/server/errors'; +import { stringify } from '../../utils/stringify'; import { NotFoundError } from '../../errors'; import type { SavedObjectsClientFactory } from '../saved_objects'; @@ -37,14 +39,21 @@ export interface EndpointFleetServicesInterface { * Will check the data provided to ensure it is visible for the current space. Supports * several types of data (ex. integration policies, agent policies, etc) */ - ensureInCurrentSpace(options: EnsureInCurrentSpaceOptions): Promise; -} + ensureInCurrentSpace( + options: Pick< + CheckInCurrentSpaceOptions, + 'agentIds' | 'integrationPolicyIds' | 'agentPolicyIds' + > + ): Promise; -type EnsureInCurrentSpaceOptions = Partial<{ - agentIds: string[]; - agentPolicyIds: string[]; - integrationPolicyIds: string[]; -}>; + /** + * Retrieves the `namespace` assigned to Endpoint Integration Policies + * @param options + */ + getPolicyNamespace( + options: Pick + ): Promise; +} export interface EndpointInternalFleetServicesInterface extends EndpointFleetServicesInterface { savedObjects: SavedObjectsClientFactory; @@ -60,7 +69,8 @@ export interface EndpointFleetServicesFactoryInterface { export class EndpointFleetServicesFactory implements EndpointFleetServicesFactoryInterface { constructor( private readonly fleetDependencies: FleetStartContract, - private readonly savedObjects: SavedObjectsClientFactory + private readonly savedObjects: SavedObjectsClientFactory, + private readonly logger: Logger ) {} asInternalUser(spaceId?: string): EndpointInternalFleetServicesInterface { @@ -85,31 +95,31 @@ export class EndpointFleetServicesFactory implements EndpointFleetServicesFactor if (!soClient) { soClient = this.savedObjects.createInternalScopedSoClient({ spaceId }); } + return checkInCurrentSpace({ + soClient, + agentService: agent, + agentPolicyService: agentPolicy, + packagePolicyService: packagePolicy, + integrationPolicyIds, + agentPolicyIds, + agentIds, + }); + }; + + const getPolicyNamespace: EndpointFleetServicesInterface['getPolicyNamespace'] = async ( + options + ) => { + if (!soClient) { + soClient = this.savedObjects.createInternalScopedSoClient({ spaceId }); + } - const handlePromiseErrors = (err: Error): never => { - // We wrap the error with our own Error class so that the API can property return a 404 - if ( - err instanceof AgentNotFoundError || - err instanceof AgentPolicyNotFoundError || - err instanceof PackagePolicyNotFoundError - ) { - throw new NotFoundError(err.message, err); - } - - throw err; - }; - - await Promise.all([ - agentIds.length ? agent.getByIds(agentIds).catch(handlePromiseErrors) : null, - - agentPolicyIds.length - ? agentPolicy.getByIds(soClient, agentPolicyIds).catch(handlePromiseErrors) - : null, - - integrationPolicyIds.length - ? packagePolicy.getByIDs(soClient, integrationPolicyIds).catch(handlePromiseErrors) - : null, - ]); + return fetchEndpointPolicyNamespace({ + ...options, + soClient, + logger: this.logger, + packagePolicyService: packagePolicy, + agentPolicyService: agentPolicy, + }); }; return { @@ -123,6 +133,144 @@ export class EndpointFleetServicesFactory implements EndpointFleetServicesFactor endpointPolicyKuery: `${PACKAGE_POLICY_SAVED_OBJECT_TYPE}.package.name: "endpoint"`, ensureInCurrentSpace, + getPolicyNamespace, }; } } + +interface CheckInCurrentSpaceOptions { + soClient: SavedObjectsClientContract; + agentService: AgentClient; + agentPolicyService: AgentPolicyServiceInterface; + packagePolicyService: PackagePolicyClient; + agentIds?: string[]; + agentPolicyIds?: string[]; + integrationPolicyIds?: string[]; +} + +/** + * Checks if data provided (integration policies, agent policies and/or agentIds) are visible in + * current space + * + * @param soClient + * @param agentService + * @param agentPolicyService + * @param packagePolicyService + * @param integrationPolicyIds + * @param agentPolicyIds + * @param agentIds + * + * @throws NotFoundError + */ +const checkInCurrentSpace = async ({ + soClient, + agentService, + agentPolicyService, + packagePolicyService, + integrationPolicyIds = [], + agentPolicyIds = [], + agentIds = [], +}: CheckInCurrentSpaceOptions): Promise => { + const handlePromiseErrors = (err: Error): never => { + // We wrap the error with our own Error class so that the API can property return a 404 + if ( + err instanceof AgentNotFoundError || + err instanceof AgentPolicyNotFoundError || + err instanceof PackagePolicyNotFoundError + ) { + throw new NotFoundError(err.message, err); + } + + throw err; + }; + + await Promise.all([ + agentIds.length ? agentService.getByIds(agentIds).catch(handlePromiseErrors) : null, + + agentPolicyIds.length + ? agentPolicyService.getByIds(soClient, agentPolicyIds).catch(handlePromiseErrors) + : null, + + integrationPolicyIds.length + ? packagePolicyService.getByIDs(soClient, integrationPolicyIds).catch(handlePromiseErrors) + : null, + ]); +}; + +interface FetchEndpointPolicyNamespaceOptions { + logger: Logger; + soClient: SavedObjectsClientContract; + packagePolicyService: PackagePolicyClient; + agentPolicyService: AgentPolicyServiceInterface; + /** A list of integration policies IDs */ + integrationPolicies: string[]; +} + +export interface FetchEndpointPolicyNamespaceResponse { + integrationPolicy: Record; +} + +const fetchEndpointPolicyNamespace = async ({ + logger, + soClient, + packagePolicyService, + agentPolicyService, + integrationPolicies, +}: FetchEndpointPolicyNamespaceOptions): Promise => { + const response: FetchEndpointPolicyNamespaceResponse = { + integrationPolicy: {}, + }; + const agentPolicyIdsToRetrieve = new Set(); + const retrievedIntegrationPolicies: Record = {}; + const retrievedAgentPolicies: Record = {}; + + if (integrationPolicies.length > 0) { + logger.debug( + () => `Retrieving package policies from fleet for:\n${stringify(integrationPolicies)}` + ); + const packagePolicies = + (await packagePolicyService.getByIDs(soClient, integrationPolicies)) ?? []; + + logger.trace(() => `Fleet package policies retrieved:\n${stringify(packagePolicies)}`); + + for (const packagePolicy of packagePolicies) { + retrievedIntegrationPolicies[packagePolicy.id] = packagePolicy; + + // Integration policy does not have an explicit namespace, which means it + // inherits it from the associated agent policies, so lets retrieve those + if (!packagePolicy.namespace) { + packagePolicy.policy_ids.forEach((agentPolicyId) => { + agentPolicyIdsToRetrieve.add(agentPolicyId); + }); + } + } + } + + if (agentPolicyIdsToRetrieve.size > 0) { + const ids = Array.from(agentPolicyIdsToRetrieve); + + logger.debug(() => `Retrieving agent policies from fleet for:\n${stringify(ids)}`); + + const agentPolicies = await agentPolicyService.getByIds(soClient, ids); + + logger.trace(() => `Fleet agent policies retrieved:\n${stringify(agentPolicies)}`); + + for (const agentPolicy of agentPolicies) { + retrievedAgentPolicies[agentPolicy.id] = agentPolicy; + } + } + + for (const integrationPolicyId of integrationPolicies) { + const integrationPolicyNamespace = retrievedIntegrationPolicies[integrationPolicyId].namespace; + + response.integrationPolicy[integrationPolicyId] = integrationPolicyNamespace + ? [integrationPolicyNamespace] + : retrievedIntegrationPolicies[integrationPolicyId].policy_ids.map((agentPolicyId) => { + return retrievedAgentPolicies[agentPolicyId].namespace; + }); + } + + logger.debug(() => `Policy namespaces:\n${stringify(response)}`); + + return response; +}; diff --git a/x-pack/plugins/security_solution/server/fleet_integration/fleet_integration.test.ts b/x-pack/plugins/security_solution/server/fleet_integration/fleet_integration.test.ts index 6accb29354ee4..80337d1a927b8 100644 --- a/x-pack/plugins/security_solution/server/fleet_integration/fleet_integration.test.ts +++ b/x-pack/plugins/security_solution/server/fleet_integration/fleet_integration.test.ts @@ -7,6 +7,7 @@ import type { ExceptionListSchema } from '@kbn/securitysolution-io-ts-list-types'; +import type { ElasticsearchClientMock } from '@kbn/core/server/mocks'; import { elasticsearchServiceMock, httpServerMock, @@ -38,7 +39,10 @@ import { import { requestContextMock } from '../lib/detection_engine/routes/__mocks__'; import { requestContextFactoryMock } from '../request_context_factory.mock'; import type { EndpointAppContextServiceStartContract } from '../endpoint/endpoint_app_context_services'; -import { createMockEndpointAppContextServiceStartContract } from '../endpoint/mocks'; +import { + createMockEndpointAppContextService, + createMockEndpointAppContextServiceStartContract, +} from '../endpoint/mocks'; import { licenseMock } from '@kbn/licensing-plugin/common/licensing.mock'; import { LicenseService } from '../../common/license'; import { Subject } from 'rxjs'; @@ -73,16 +77,30 @@ import { createProductFeaturesServiceMock } from '../lib/product_features_servic import * as moment from 'moment'; import type { PostAgentPolicyCreateCallback, + PostPackagePolicyPostCreateCallback, PutPackagePolicyUpdateCallback, } from '@kbn/fleet-plugin/server/types'; import type { EndpointMetadataService } from '../endpoint/services/metadata'; import { createEndpointMetadataServiceTestContextMock } from '../endpoint/services/metadata/mocks'; +import { createPolicyDataStreamsIfNeeded as _createPolicyDataStreamsIfNeeded } from './handlers/create_policy_datastreams'; jest.mock('uuid', () => ({ v4: (): string => 'NEW_UUID', })); -describe('ingest_integration tests ', () => { +jest.mock('./handlers/create_policy_datastreams', () => { + const actualModule = jest.requireActual('./handlers/create_policy_datastreams'); + + return { + ...actualModule, + createPolicyDataStreamsIfNeeded: jest.fn(async () => {}), + }; +}); + +const createPolicyDataStreamsIfNeededMock = + _createPolicyDataStreamsIfNeeded as unknown as jest.Mock; + +describe('Fleet integrations', () => { let endpointAppContextStartContract: EndpointAppContextServiceStartContract; let req: KibanaRequest; let ctx: ReturnType; @@ -350,10 +368,20 @@ describe('ingest_integration tests ', () => { }); describe('package policy post create callback', () => { - const soClient = savedObjectsClientMock.create(); - const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; - const callback = getPackagePolicyPostCreateCallback(logger, exceptionListClient); - const policyConfig = generator.generatePolicyPackagePolicy() as PackagePolicy; + let soClient: ReturnType; + let esClient: ElasticsearchClientMock; + let callback: PostPackagePolicyPostCreateCallback; + let policyConfig: PackagePolicy; + let endpointAppContextServiceMock: ReturnType; + + beforeEach(() => { + soClient = savedObjectsClientMock.create(); + esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + endpointAppContextServiceMock = createMockEndpointAppContextService(); + endpointAppContextServiceMock.getExceptionListsClient.mockReturnValue(exceptionListClient); + callback = getPackagePolicyPostCreateCallback(endpointAppContextServiceMock); + policyConfig = generator.generatePolicyPackagePolicy() as PackagePolicy; + }); it('should create the Endpoint Event Filters List and add the correct Event Filters List Item attached to the policy given nonInteractiveSession parameter on integration config eventFilters', async () => { const integrationConfig = { @@ -374,11 +402,11 @@ describe('ingest_integration tests ', () => { req ); - expect(await exceptionListClient.createExceptionList).toHaveBeenCalledWith( + expect(exceptionListClient.createExceptionList).toHaveBeenCalledWith( expect.objectContaining({ listId: ENDPOINT_EVENT_FILTERS_LIST_ID }) ); - expect(await exceptionListClient.createExceptionListItem).toHaveBeenCalledWith( + expect(exceptionListClient.createExceptionListItem).toHaveBeenCalledWith( expect.objectContaining({ listId: ENDPOINT_EVENT_FILTERS_LIST_ID, tags: [`policy:${postCreatedPolicyConfig.id}`], @@ -413,14 +441,20 @@ describe('ingest_integration tests ', () => { req ); - expect(await exceptionListClient.createExceptionList).not.toHaveBeenCalled(); + expect(exceptionListClient.createExceptionList).not.toHaveBeenCalled(); - expect(await exceptionListClient.createExceptionListItem).not.toHaveBeenCalled(); + expect(exceptionListClient.createExceptionListItem).not.toHaveBeenCalled(); expect(postCreatedPolicyConfig.inputs[0]!.config!.integration_config.value).toEqual( integrationConfig ); }); + + it('should call `createPolicyDatastreamsIfNeeded`', async () => { + await callback(policyConfig, soClient, esClient, requestContextMock.convertContext(ctx), req); + + expect(createPolicyDataStreamsIfNeededMock).toHaveBeenCalled(); + }); }); describe('agent policy update callback', () => { diff --git a/x-pack/plugins/security_solution/server/fleet_integration/fleet_integration.ts b/x-pack/plugins/security_solution/server/fleet_integration/fleet_integration.ts index 71c935e720511..54f1ce8cc7e01 100644 --- a/x-pack/plugins/security_solution/server/fleet_integration/fleet_integration.ts +++ b/x-pack/plugins/security_solution/server/fleet_integration/fleet_integration.ts @@ -27,8 +27,13 @@ import type { InfoResponse } from '@elastic/elasticsearch/lib/api/types'; import { ProductFeatureSecurityKey } from '@kbn/security-solution-features/keys'; import type { PostAgentPolicyCreateCallback, + PostAgentPolicyPostUpdateCallback, PostAgentPolicyUpdateCallback, + PutPackagePolicyPostUpdateCallback, } from '@kbn/fleet-plugin/server/types'; +import type { EndpointInternalFleetServicesInterface } from '../endpoint/services/fleet'; +import type { EndpointAppContextService } from '../endpoint/endpoint_app_context_services'; +import { createPolicyDataStreamsIfNeeded } from './handlers/create_policy_datastreams'; import { updateAntivirusRegistrationEnabled } from '../../common/endpoint/utils/update_antivirus_registration_enabled'; import { validatePolicyAgainstProductFeatures } from './handlers/validate_policy_against_product_features'; import { validateEndpointPackagePolicy } from './handlers/validate_endpoint_package_policy'; @@ -62,6 +67,32 @@ const isEndpointPackagePolicy = ( return packagePolicy.package?.name === 'endpoint'; }; +const getEndpointPolicyForAgentPolicy = async ( + fleetServices: EndpointInternalFleetServicesInterface, + agentPolicy: AgentPolicy +): Promise => { + let agentPolicyIntegrations: PackagePolicy[] | undefined = agentPolicy.package_policies; + + if (!agentPolicyIntegrations) { + const fullAgentPolicy = await fleetServices.agentPolicy.get( + fleetServices.savedObjects.createInternalScopedSoClient(), + agentPolicy.id, + true + ); + agentPolicyIntegrations = fullAgentPolicy?.package_policies ?? []; + } + + if (Array.isArray(agentPolicyIntegrations)) { + for (const integrationPolicy of agentPolicyIntegrations) { + if (isEndpointPackagePolicy(integrationPolicy)) { + return integrationPolicy; + } + } + } + + return undefined; +}; + const shouldUpdateMetaValues = ( endpointPackagePolicy: PolicyConfig, currentLicenseType: string, @@ -279,16 +310,47 @@ export const getPackagePolicyUpdateCallback = ( }; }; +export const getPackagePolicyPostUpdateCallback = ( + endpointServices: EndpointAppContextService +): PutPackagePolicyPostUpdateCallback => { + const logger = endpointServices.createLogger('endpointPackagePolicyPostUpdate'); + + return async (packagePolicy) => { + if (!isEndpointPackagePolicy(packagePolicy)) { + return packagePolicy; + } + + logger.debug(`Processing endpoint integration policy (post update): ${packagePolicy.id}`); + + // The check below will run in the background - we don't need to wait for it + createPolicyDataStreamsIfNeeded({ + endpointServices, + endpointPolicyIds: [packagePolicy.id], + }).catch(() => {}); // to silence @typescript-eslint/no-floating-promises + + return packagePolicy; + }; +}; + export const getPackagePolicyPostCreateCallback = ( - logger: Logger, - exceptionsClient: ExceptionListClient | undefined + endpointServices: EndpointAppContextService ): PostPackagePolicyPostCreateCallback => { + const logger = endpointServices.createLogger('endpointPolicyPostCreate'); + const exceptionsClient = endpointServices.getExceptionListsClient(); + return async (packagePolicy: PackagePolicy): Promise => { // We only care about Endpoint package policies if (!exceptionsClient || !isEndpointPackagePolicy(packagePolicy)) { return packagePolicy; } + // Check and create internal datastreams for this policy if needed. + // NOTE: we don't need for it to complete here, thus no `await`. + createPolicyDataStreamsIfNeeded({ + endpointServices, + endpointPolicyIds: [packagePolicy.id], + }).catch(() => {}); // to silence @typescript-eslint/no-floating-promises + const integrationConfig = packagePolicy?.inputs[0]?.config?.integration_config; if (integrationConfig && integrationConfig?.value?.eventFilters !== undefined) { @@ -353,6 +415,31 @@ export const getAgentPolicyUpdateCallback = ( }; }; +export const getAgentPolicyPostUpdateCallback = ( + endpointServices: EndpointAppContextService +): PostAgentPolicyPostUpdateCallback => { + const logger = endpointServices.createLogger('endpointPolicyPostUpdate'); + + return async (agentPolicy) => { + const fleetServices = endpointServices.getInternalFleetServices(); + const endpointPolicy = await getEndpointPolicyForAgentPolicy(fleetServices, agentPolicy); + + if (!endpointPolicy) { + return agentPolicy; + } + + logger.debug(`Processing post-update to Fleet agent policy: [${agentPolicy.id}]`); + + // We don't need to `await` for this function to execute. It can be done in the background + createPolicyDataStreamsIfNeeded({ + endpointServices, + endpointPolicyIds: [endpointPolicy.id], + }).catch(() => {}); // to silence @typescript-eslint/no-floating-promises + + return agentPolicy; + }; +}; + export const getPackagePolicyDeleteCallback = ( exceptionsClient: ExceptionListClient | undefined, savedObjectsClient: SavedObjectsClientContract | undefined diff --git a/x-pack/plugins/security_solution/server/fleet_integration/handlers/create_policy_datastreams.test.ts b/x-pack/plugins/security_solution/server/fleet_integration/handlers/create_policy_datastreams.test.ts new file mode 100644 index 0000000000000..0efaa5516a6f9 --- /dev/null +++ b/x-pack/plugins/security_solution/server/fleet_integration/handlers/create_policy_datastreams.test.ts @@ -0,0 +1,119 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { createMockEndpointAppContextService } from '../../endpoint/mocks'; +import type { ElasticsearchClientMock } from '@kbn/core-elasticsearch-client-server-mocks'; +import type { FetchEndpointPolicyNamespaceResponse } from '../../endpoint/services/fleet'; +import { createPolicyDataStreamsIfNeeded } from './create_policy_datastreams'; + +describe('createPolicyDataStreamsIfNeeded()', () => { + let endpointServicesMock: ReturnType; + let esClientMock: ElasticsearchClientMock; + let policyNamespacesMock: FetchEndpointPolicyNamespaceResponse; + + beforeEach(() => { + endpointServicesMock = createMockEndpointAppContextService(); + + esClientMock = endpointServicesMock.getInternalEsClient() as ElasticsearchClientMock; + esClientMock.indices.exists.mockResolvedValue(false); + + policyNamespacesMock = { integrationPolicy: { '123': ['foo1', 'foo2'] } }; + ( + endpointServicesMock.getInternalFleetServices().getPolicyNamespace as jest.Mock + ).mockResolvedValue(policyNamespacesMock); + }); + + afterEach(() => { + createPolicyDataStreamsIfNeeded.cache.deleteAll(); + }); + + it('should create datastreams if they do not exist', async () => { + await createPolicyDataStreamsIfNeeded({ + endpointServices: endpointServicesMock, + endpointPolicyIds: ['123'], + }); + + expect(esClientMock.indices.createDataStream).toHaveBeenCalledTimes(4); + [ + '.logs-endpoint.diagnostic.collection-foo1', + '.logs-endpoint.diagnostic.collection-foo2', + '.logs-endpoint.action.responses-foo1', + '.logs-endpoint.action.responses-foo2', + ].forEach((indexName) => { + expect(esClientMock.indices.createDataStream).toHaveBeenCalledWith({ + name: indexName, + }); + }); + }); + + it('should not create datastream if they already exist', async () => { + esClientMock.indices.exists.mockImplementation(async (options) => { + return ( + options.index === '.logs-endpoint.action.responses-foo1' || + options.index === '.logs-endpoint.diagnostic.collection-foo1' + ); + }); + + await createPolicyDataStreamsIfNeeded({ + endpointServices: endpointServicesMock, + endpointPolicyIds: ['123'], + }); + + expect(esClientMock.indices.createDataStream).toHaveBeenCalledTimes(2); + ['.logs-endpoint.diagnostic.collection-foo2', '.logs-endpoint.action.responses-foo2'].forEach( + (indexName) => { + expect(esClientMock.indices.createDataStream).toHaveBeenCalledWith({ + name: indexName, + }); + } + ); + }); + + it('should create heartbeat index when running in serverless', async () => { + (endpointServicesMock.isServerless as jest.Mock).mockReturnValue(true); + await createPolicyDataStreamsIfNeeded({ + endpointServices: endpointServicesMock, + endpointPolicyIds: ['123'], + }); + + expect(esClientMock.indices.createDataStream).toHaveBeenCalledTimes(6); + [ + '.logs-endpoint.diagnostic.collection-foo1', + '.logs-endpoint.diagnostic.collection-foo2', + '.logs-endpoint.action.responses-foo1', + '.logs-endpoint.action.responses-foo2', + '.logs-endpoint.heartbeat-foo1', + '.logs-endpoint.heartbeat-foo2', + ].forEach((indexName) => { + expect(esClientMock.indices.createDataStream).toHaveBeenCalledWith({ + name: indexName, + }); + }); + }); + + it('should not call ES if index existence was already checked', async () => { + createPolicyDataStreamsIfNeeded.cache.set('.logs-endpoint.action.responses-foo1', true); + await createPolicyDataStreamsIfNeeded({ + endpointServices: endpointServicesMock, + endpointPolicyIds: ['123'], + }); + + expect(esClientMock.indices.exists).not.toHaveBeenCalledWith({ + index: '.logs-endpoint.action.responses-foo1', + }); + expect(esClientMock.indices.createDataStream).toHaveBeenCalledTimes(3); + [ + '.logs-endpoint.diagnostic.collection-foo1', + '.logs-endpoint.diagnostic.collection-foo2', + '.logs-endpoint.action.responses-foo2', + ].forEach((indexName) => { + expect(esClientMock.indices.createDataStream).toHaveBeenCalledWith({ + name: indexName, + }); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/fleet_integration/handlers/create_policy_datastreams.ts b/x-pack/plugins/security_solution/server/fleet_integration/handlers/create_policy_datastreams.ts new file mode 100644 index 0000000000000..93fec3526a7b3 --- /dev/null +++ b/x-pack/plugins/security_solution/server/fleet_integration/handlers/create_policy_datastreams.ts @@ -0,0 +1,159 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import pMap from 'p-map'; +import type { EndpointAppContextService } from '../../endpoint/endpoint_app_context_services'; +import { catchAndWrapError } from '../../endpoint/utils'; +import type { SimpleMemCacheInterface } from '../../endpoint/lib/simple_mem_cache'; +import { SimpleMemCache } from '../../endpoint/lib/simple_mem_cache'; +import { + ENDPOINT_ACTION_RESPONSES_DS, + ENDPOINT_HEARTBEAT_INDEX_PATTERN, +} from '../../../common/endpoint/constants'; +import { DEFAULT_DIAGNOSTIC_INDEX } from '../../lib/telemetry/constants'; +import { stringify } from '../../endpoint/utils/stringify'; + +const buildIndexNameWithNamespace = ( + indexNamePrefixOrPattern: string, + namespace: string +): string => { + if (indexNamePrefixOrPattern.endsWith('*')) { + const hasDash = indexNamePrefixOrPattern.endsWith('-*'); + return `${indexNamePrefixOrPattern.substring(0, indexNamePrefixOrPattern.length - 1)}${ + hasDash ? '' : '-' + }${namespace}`; + } + + return `${indexNamePrefixOrPattern}${ + indexNamePrefixOrPattern.endsWith('-') ? '' : '-' + }${namespace}`; +}; + +const cache = new SimpleMemCache({ + // Cache of created Datastreams last for 12h, at which point it is checked again. + // This is just a safeguard case (for whatever reason) the index is deleted + // 1.8e+7 === hours + ttl: 1.8e7, +}); + +interface PolicyDataStreamsCreator { + (options: CreatePolicyDataStreamsOptions): Promise; + cache: SimpleMemCacheInterface; +} + +export interface CreatePolicyDataStreamsOptions { + endpointServices: EndpointAppContextService; + endpointPolicyIds: string[]; +} + +/** + * Ensures that the DOT index Datastreams necessary to support Elastic Defend are crated (prior to + * endpoint writing data to them) + */ +export const createPolicyDataStreamsIfNeeded: PolicyDataStreamsCreator = async ({ + endpointServices, + endpointPolicyIds, +}: CreatePolicyDataStreamsOptions): Promise => { + const logger = endpointServices.createLogger('endpointPolicyDatastreamCreator'); + const esClient = endpointServices.getInternalEsClient(); + + logger.debug( + () => + `Checking if datastreams need to be created for Endpoint integration policy [${endpointPolicyIds.join( + ', ' + )}]` + ); + + // FIXME:PT Need to ensure that the datastreams are created in all associated space ids that the policy is shared with + // This can be deferred to activity around support of Spaces - team issue: 8199 (epic) + // We might need to do much here other than to ensure we can access all policies across all spaces in order to get the namespace value + + const fleetServices = endpointServices.getInternalFleetServices(); + const policyNamespaces = await fleetServices.getPolicyNamespace({ + integrationPolicies: endpointPolicyIds, + }); + const indexesCreated: string[] = []; + const createErrors: string[] = []; + const indicesToCreate: string[] = Object.values(policyNamespaces.integrationPolicy).reduce< + string[] + >((acc, namespaceList) => { + for (const namespace of namespaceList) { + acc.push( + buildIndexNameWithNamespace(DEFAULT_DIAGNOSTIC_INDEX, namespace), + buildIndexNameWithNamespace(ENDPOINT_ACTION_RESPONSES_DS, namespace) + ); + + if (endpointServices.isServerless()) { + acc.push(buildIndexNameWithNamespace(ENDPOINT_HEARTBEAT_INDEX_PATTERN, namespace)); + } + } + + return acc; + }, []); + + const processesDatastreamIndex = async (datastreamIndexName: string): Promise => { + if (cache.get(datastreamIndexName)) { + return; + } + + const doesDataStreamAlreadyExist = await esClient.indices + .exists({ index: datastreamIndexName }) + .catch(catchAndWrapError); + + if (doesDataStreamAlreadyExist) { + cache.set(datastreamIndexName, true); + return; + } + + await esClient.indices + .createDataStream({ name: datastreamIndexName }) + .then(() => { + indexesCreated.push(datastreamIndexName); + cache.set(datastreamIndexName, true); + }) + .catch((err) => { + // It's possible that between the `.exists()` check and this `.createDataStream()` that + // the index could have been created. If that's the case, then just ignore the error. + if (err.body?.error?.type === 'resource_already_exists_exception') { + cache.set(datastreamIndexName, true); + return; + } + + createErrors.push( + `Attempt to create datastream [${datastreamIndexName}] failed:\n${stringify( + err.body?.error ?? err + )}` + ); + }); + }; + + logger.debug( + () => + `Checking if the following datastream(s) need to be created:\n ${indicesToCreate.join( + '\n ' + )}` + ); + + await pMap(indicesToCreate, processesDatastreamIndex, { concurrency: 10 }); + + if (indexesCreated.length > 0) { + logger.info( + `Datastream(s) created in support of Elastic Defend policy [${endpointPolicyIds.join( + ', ' + )}]:\n ${indexesCreated.join('\n ')}` + ); + } else if (createErrors.length === 0) { + logger.debug(() => `Nothing to do. Datastreams already exist`); + } + + if (createErrors.length > 0) { + logger.error( + `${createErrors.length} errors encountered:\n${createErrors.join('\n--------\n')}` + ); + } +}; +createPolicyDataStreamsIfNeeded.cache = cache;