From 13ec56db8bf64621298c30693000dd68eb9a272c Mon Sep 17 00:00:00 2001 From: Alex Kahan Date: Tue, 21 Jul 2020 15:18:01 -0400 Subject: [PATCH] Limit concurrent access to download API + Replace with LRU cache (#72503) * Limit concurrent access to download API * Replacing cache with LRU Cache * Configure the LRU cache Co-authored-by: Elastic Machine --- .../common/endpoint/constants.ts | 2 + .../endpoint/lib/artifacts/cache.test.ts | 48 ------------- .../server/endpoint/lib/artifacts/cache.ts | 37 ---------- .../server/endpoint/lib/artifacts/index.ts | 1 - .../artifacts/download_exception_list.test.ts | 8 +-- .../artifacts/download_exception_list.ts | 7 +- .../endpoint/routes/limited_concurrency.ts | 72 +++++++++++++++++++ .../manifest_manager/manifest_manager.mock.ts | 6 +- .../manifest_manager/manifest_manager.test.ts | 11 ++- .../manifest_manager/manifest_manager.ts | 6 +- .../security_solution/server/plugin.ts | 10 ++- 11 files changed, 100 insertions(+), 108 deletions(-) delete mode 100644 x-pack/plugins/security_solution/server/endpoint/lib/artifacts/cache.test.ts delete mode 100644 x-pack/plugins/security_solution/server/endpoint/lib/artifacts/cache.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/routes/limited_concurrency.ts diff --git a/x-pack/plugins/security_solution/common/endpoint/constants.ts b/x-pack/plugins/security_solution/common/endpoint/constants.ts index e311e358e6146..6ea0c36328eed 100644 --- a/x-pack/plugins/security_solution/common/endpoint/constants.ts +++ b/x-pack/plugins/security_solution/common/endpoint/constants.ts @@ -9,3 +9,5 @@ export const alertsIndexPattern = 'logs-endpoint.alerts-*'; export const metadataIndexPattern = 'metrics-endpoint.metadata-*'; export const policyIndexPattern = 'metrics-endpoint.policy-*'; export const telemetryIndexPattern = 'metrics-endpoint.telemetry-*'; +export const LIMITED_CONCURRENCY_ENDPOINT_ROUTE_TAG = 'endpoint:limited-concurrency'; +export const LIMITED_CONCURRENCY_ENDPOINT_COUNT = 100; diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/cache.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/cache.test.ts deleted file mode 100644 index 00c764d0b912e..0000000000000 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/cache.test.ts +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { ExceptionsCache } from './cache'; - -describe('ExceptionsCache tests', () => { - let cache: ExceptionsCache; - const body = Buffer.from('body'); - - beforeEach(() => { - jest.clearAllMocks(); - cache = new ExceptionsCache(3); - }); - - test('it should cache', async () => { - cache.set('test', body); - const cacheResp = cache.get('test'); - expect(cacheResp).toEqual(body); - }); - - test('it should handle cache miss', async () => { - cache.set('test', body); - const cacheResp = cache.get('not test'); - expect(cacheResp).toEqual(undefined); - }); - - test('it should handle cache eviction', async () => { - const a = Buffer.from('a'); - const b = Buffer.from('b'); - const c = Buffer.from('c'); - const d = Buffer.from('d'); - cache.set('1', a); - cache.set('2', b); - cache.set('3', c); - const cacheResp = cache.get('1'); - expect(cacheResp).toEqual(a); - - cache.set('4', d); - const secondResp = cache.get('1'); - expect(secondResp).toEqual(undefined); - expect(cache.get('2')).toEqual(b); - expect(cache.get('3')).toEqual(c); - expect(cache.get('4')).toEqual(d); - }); -}); diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/cache.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/cache.ts deleted file mode 100644 index b9d3bae4e6ef9..0000000000000 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/cache.ts +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -const DEFAULT_MAX_SIZE = 10; - -/** - * FIFO cache implementation for artifact downloads. - */ -export class ExceptionsCache { - private cache: Map; - private queue: string[]; - private maxSize: number; - - constructor(maxSize: number) { - this.cache = new Map(); - this.queue = []; - this.maxSize = maxSize || DEFAULT_MAX_SIZE; - } - - set(id: string, body: Buffer) { - if (this.queue.length + 1 > this.maxSize) { - const entry = this.queue.shift(); - if (entry !== undefined) { - this.cache.delete(entry); - } - } - this.queue.push(id); - this.cache.set(id, body); - } - - get(id: string): Buffer | undefined { - return this.cache.get(id); - } -} diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/index.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/index.ts index ee7d44459aa38..21a9047a04299 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/index.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/index.ts @@ -4,7 +4,6 @@ * you may not use this file except in compliance with the Elastic License. */ -export * from './cache'; export * from './common'; export * from './lists'; export * from './manifest'; diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts index 8c6faee7f7a5d..4454da855569b 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts @@ -4,6 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ import { deflateSync, inflateSync } from 'zlib'; +import LRU from 'lru-cache'; import { ILegacyClusterClient, IRouter, @@ -22,7 +23,6 @@ import { httpServerMock, loggingSystemMock, } from 'src/core/server/mocks'; -import { ExceptionsCache } from '../../lib/artifacts/cache'; import { ArtifactConstants } from '../../lib/artifacts'; import { registerDownloadExceptionListRoute } from './download_exception_list'; import { EndpointAppContextService } from '../../endpoint_app_context_services'; @@ -97,7 +97,7 @@ describe('test alerts route', () => { let routeConfig: RouteConfig; let routeHandler: RequestHandler; let endpointAppContextService: EndpointAppContextService; - let cache: ExceptionsCache; + let cache: LRU; let ingestSavedObjectClient: jest.Mocked; beforeEach(() => { @@ -108,7 +108,7 @@ describe('test alerts route', () => { mockClusterClient.asScoped.mockReturnValue(mockScopedClient); routerMock = httpServiceMock.createRouter(); endpointAppContextService = new EndpointAppContextService(); - cache = new ExceptionsCache(5); + cache = new LRU({ max: 10, maxAge: 1000 * 60 * 60 }); const startContract = createMockEndpointAppContextServiceStartContract(); // The authentication with the Fleet Plugin needs a separate scoped SO Client @@ -164,7 +164,7 @@ describe('test alerts route', () => { path.startsWith('/api/endpoint/artifacts/download') )!; - expect(routeConfig.options).toEqual(undefined); + expect(routeConfig.options).toEqual({ tags: ['endpoint:limited-concurrency'] }); await routeHandler( ({ diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts index 218f7c059da48..38e900c4d5015 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts @@ -11,11 +11,13 @@ import { IKibanaResponse, SavedObject, } from 'src/core/server'; +import LRU from 'lru-cache'; // eslint-disable-next-line @kbn/eslint/no-restricted-paths import { authenticateAgentWithAccessToken } from '../../../../../ingest_manager/server/services/agents/authenticate'; import { validate } from '../../../../common/validate'; +import { LIMITED_CONCURRENCY_ENDPOINT_ROUTE_TAG } from '../../../../common/endpoint/constants'; import { buildRouteValidation } from '../../../utils/build_validation/route_validation'; -import { ArtifactConstants, ExceptionsCache } from '../../lib/artifacts'; +import { ArtifactConstants } from '../../lib/artifacts'; import { DownloadArtifactRequestParamsSchema, downloadArtifactRequestParamsSchema, @@ -32,7 +34,7 @@ const allowlistBaseRoute: string = '/api/endpoint/artifacts'; export function registerDownloadExceptionListRoute( router: IRouter, endpointContext: EndpointAppContext, - cache: ExceptionsCache + cache: LRU ) { router.get( { @@ -43,6 +45,7 @@ export function registerDownloadExceptionListRoute( DownloadArtifactRequestParamsSchema >(downloadArtifactRequestParamsSchema), }, + options: { tags: [LIMITED_CONCURRENCY_ENDPOINT_ROUTE_TAG] }, }, async (context, req, res) => { let scopedSOClient: SavedObjectsClientContract; diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/limited_concurrency.ts b/x-pack/plugins/security_solution/server/endpoint/routes/limited_concurrency.ts new file mode 100644 index 0000000000000..767ba31311a77 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/routes/limited_concurrency.ts @@ -0,0 +1,72 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { + CoreSetup, + KibanaRequest, + LifecycleResponseFactory, + OnPreAuthToolkit, +} from 'kibana/server'; +import { + LIMITED_CONCURRENCY_ENDPOINT_ROUTE_TAG, + LIMITED_CONCURRENCY_ENDPOINT_COUNT, +} from '../../../common/endpoint/constants'; + +class MaxCounter { + constructor(private readonly max: number = 1) {} + private counter = 0; + valueOf() { + return this.counter; + } + increase() { + if (this.counter < this.max) { + this.counter += 1; + } + } + decrease() { + if (this.counter > 0) { + this.counter -= 1; + } + } + lessThanMax() { + return this.counter < this.max; + } +} + +function shouldHandleRequest(request: KibanaRequest) { + const tags = request.route.options.tags; + return tags.includes(LIMITED_CONCURRENCY_ENDPOINT_ROUTE_TAG); +} + +export function registerLimitedConcurrencyRoutes(core: CoreSetup) { + const counter = new MaxCounter(LIMITED_CONCURRENCY_ENDPOINT_COUNT); + core.http.registerOnPreAuth(function preAuthHandler( + request: KibanaRequest, + response: LifecycleResponseFactory, + toolkit: OnPreAuthToolkit + ) { + if (!shouldHandleRequest(request)) { + return toolkit.next(); + } + + if (!counter.lessThanMax()) { + return response.customError({ + body: 'Too Many Requests', + statusCode: 429, + }); + } + + counter.increase(); + + // requests.events.aborted$ has a bug (but has test which explicitly verifies) where it's fired even when the request completes + // https://github.com/elastic/kibana/pull/70495#issuecomment-656288766 + request.events.aborted$.toPromise().then(() => { + counter.decrease(); + }); + + return toolkit.next(); + }); +} diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts index 2ebffa6fb3ad8..592ffb0eae62a 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts @@ -9,7 +9,7 @@ import { Logger } from 'src/core/server'; import { PackageConfigServiceInterface } from '../../../../../../ingest_manager/server'; import { createPackageConfigServiceMock } from '../../../../../../ingest_manager/server/mocks'; import { listMock } from '../../../../../../lists/server/mocks'; -import { ExceptionsCache } from '../../../lib/artifacts'; +import LRU from 'lru-cache'; import { getArtifactClientMock } from '../artifact_client.mock'; import { getManifestClientMock } from '../manifest_client.mock'; import { ManifestManager } from './manifest_manager'; @@ -28,11 +28,11 @@ export enum ManifestManagerMockType { export const getManifestManagerMock = (opts?: { mockType?: ManifestManagerMockType; - cache?: ExceptionsCache; + cache?: LRU; packageConfigService?: jest.Mocked; savedObjectsClient?: ReturnType; }): ManifestManager => { - let cache = new ExceptionsCache(5); + let cache = new LRU({ max: 10, maxAge: 1000 * 60 * 60 }); if (opts?.cache !== undefined) { cache = opts.cache; } diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts index ff331f7d017f4..c838f772fb66b 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts @@ -7,13 +7,10 @@ import { inflateSync } from 'zlib'; import { savedObjectsClientMock } from 'src/core/server/mocks'; import { createPackageConfigServiceMock } from '../../../../../../ingest_manager/server/mocks'; -import { - ArtifactConstants, - ManifestConstants, - ExceptionsCache, - isCompleteArtifact, -} from '../../../lib/artifacts'; +import { ArtifactConstants, ManifestConstants, isCompleteArtifact } from '../../../lib/artifacts'; + import { getManifestManagerMock } from './manifest_manager.mock'; +import LRU from 'lru-cache'; describe('manifest_manager', () => { describe('ManifestManager sanity checks', () => { @@ -41,7 +38,7 @@ describe('manifest_manager', () => { }); test('ManifestManager populates cache properly', async () => { - const cache = new ExceptionsCache(5); + const cache = new LRU({ max: 10, maxAge: 1000 * 60 * 60 }); const manifestManager = getManifestManagerMock({ cache }); const oldManifest = await manifestManager.getLastComputedManifest( ManifestConstants.SCHEMA_VERSION diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts index 2501f07cb26e0..13ca51e1f2b39 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts @@ -5,6 +5,7 @@ */ import { Logger, SavedObjectsClientContract } from 'src/core/server'; +import LRU from 'lru-cache'; import { PackageConfigServiceInterface } from '../../../../../../ingest_manager/server'; import { ExceptionListClient } from '../../../../../../lists/server'; import { ManifestSchemaVersion } from '../../../../../common/endpoint/schema/common'; @@ -16,7 +17,6 @@ import { Manifest, buildArtifact, getFullEndpointExceptionList, - ExceptionsCache, ManifestDiff, getArtifactId, } from '../../../lib/artifacts'; @@ -33,7 +33,7 @@ export interface ManifestManagerContext { exceptionListClient: ExceptionListClient; packageConfigService: PackageConfigServiceInterface; logger: Logger; - cache: ExceptionsCache; + cache: LRU; } export interface ManifestSnapshotOpts { @@ -51,7 +51,7 @@ export class ManifestManager { protected packageConfigService: PackageConfigServiceInterface; protected savedObjectsClient: SavedObjectsClientContract; protected logger: Logger; - protected cache: ExceptionsCache; + protected cache: LRU; constructor(context: ManifestManagerContext) { this.artifactClient = context.artifactClient; diff --git a/x-pack/plugins/security_solution/server/plugin.ts b/x-pack/plugins/security_solution/server/plugin.ts index 22b55c64a1657..611f35c6d402a 100644 --- a/x-pack/plugins/security_solution/server/plugin.ts +++ b/x-pack/plugins/security_solution/server/plugin.ts @@ -7,6 +7,7 @@ import { Observable } from 'rxjs'; import { first } from 'rxjs/operators'; import { i18n } from '@kbn/i18n'; +import LRU from 'lru-cache'; import { CoreSetup, @@ -34,13 +35,14 @@ import { isAlertExecutor } from './lib/detection_engine/signals/types'; import { signalRulesAlertType } from './lib/detection_engine/signals/signal_rule_alert_type'; import { rulesNotificationAlertType } from './lib/detection_engine/notifications/rules_notification_alert_type'; import { isNotificationAlertExecutor } from './lib/detection_engine/notifications/types'; -import { ManifestTask, ExceptionsCache } from './endpoint/lib/artifacts'; +import { ManifestTask } from './endpoint/lib/artifacts'; import { initSavedObjects, savedObjectTypes } from './saved_objects'; import { AppClientFactory } from './client'; import { createConfig$, ConfigType } from './config'; import { initUiSettings } from './ui_settings'; import { APP_ID, APP_ICON, SERVER_APP_ID, SecurityPageName } from '../common/constants'; import { registerEndpointRoutes } from './endpoint/routes/metadata'; +import { registerLimitedConcurrencyRoutes } from './endpoint/routes/limited_concurrency'; import { registerResolverRoutes } from './endpoint/routes/resolver'; import { registerPolicyRoutes } from './endpoint/routes/policy'; import { ArtifactClient, ManifestManager } from './endpoint/services'; @@ -94,14 +96,15 @@ export class Plugin implements IPlugin; constructor(context: PluginInitializerContext) { this.context = context; this.logger = context.logger.get('plugins', APP_ID); this.config$ = createConfig$(context); this.appClientFactory = new AppClientFactory(); - this.exceptionsCache = new ExceptionsCache(5); // TODO + // Cache up to three artifacts with a max retention of 5 mins each + this.exceptionsCache = new LRU({ max: 3, maxAge: 1000 * 60 * 5 }); this.logger.debug('plugin initialized'); } @@ -149,6 +152,7 @@ export class Plugin implements IPlugin