diff --git a/x-pack/plugins/security/server/session_management/session_index.test.ts b/x-pack/plugins/security/server/session_management/session_index.test.ts index 9ad94020a25f6..e7cb34be0278f 100644 --- a/x-pack/plugins/security/server/session_management/session_index.test.ts +++ b/x-pack/plugins/security/server/session_management/session_index.test.ts @@ -6,6 +6,7 @@ */ import { errors } from '@elastic/elasticsearch'; +import type { BulkResponse, SearchResponse } from '@elastic/elasticsearch/lib/api/types'; import type { DeeplyMockedKeys } from '@kbn/utility-types/jest'; import type { ElasticsearchClient } from 'src/core/server'; @@ -229,12 +230,12 @@ describe('Session index', () => { beforeEach(() => { mockElasticsearchClient.search.mockResolvedValue( securityMock.createApiResponse({ - body: { hits: { hits: [sessionValue] } } as any, + body: { hits: { total: 1, hits: [sessionValue] } } as SearchResponse, }) ); mockElasticsearchClient.bulk.mockResolvedValue( securityMock.createApiResponse({ - body: { items: [{}] } as any, + body: { items: [{}] } as BulkResponse, }) ); jest.spyOn(Date, 'now').mockImplementation(() => now); @@ -262,60 +263,63 @@ describe('Session index', () => { await sessionIndex.cleanUp(); expect(mockElasticsearchClient.search).toHaveBeenCalledTimes(1); - expect(mockElasticsearchClient.search).toHaveBeenCalledWith( - { - index: indexName, - _source_includes: 'usernameHash,provider', - body: { - query: { - bool: { - should: [ - // All expired sessions based on the lifespan, no matter which provider they belong to. - { range: { lifespanExpiration: { lte: now } } }, - // All sessions that belong to the providers that aren't configured. - { - bool: { - must_not: { - bool: { - should: [ - { - bool: { - must: [ - { term: { 'provider.type': 'basic' } }, - { term: { 'provider.name': 'basic' } }, - ], - }, + expect(mockElasticsearchClient.search).toHaveBeenCalledWith({ + index: indexName, + _source_includes: 'usernameHash,provider', + size: 1_000, + body: { + query: { + bool: { + should: [ + // All expired sessions based on the lifespan, no matter which provider they belong to. + { range: { lifespanExpiration: { lte: now } } }, + // All sessions that belong to the providers that aren't configured. + { + bool: { + must_not: { + bool: { + should: [ + { + bool: { + must: [ + { term: { 'provider.type': 'basic' } }, + { term: { 'provider.name': 'basic' } }, + ], }, - ], - minimum_should_match: 1, - }, + }, + ], + minimum_should_match: 1, }, }, }, - // The sessions that belong to a particular provider that are expired based on the idle timeout. - { - bool: { - must: [ - { term: { 'provider.type': 'basic' } }, - { term: { 'provider.name': 'basic' } }, - ], - should: [{ range: { idleTimeoutExpiration: { lte: now } } }], - minimum_should_match: 1, - }, + }, + // The sessions that belong to a particular provider that are expired based on the idle timeout. + { + bool: { + must: [ + { term: { 'provider.type': 'basic' } }, + { term: { 'provider.name': 'basic' } }, + ], + should: [{ range: { idleTimeoutExpiration: { lte: now } } }], + minimum_should_match: 1, }, - ], - }, + }, + ], }, }, }, - { ignore: [409, 404] } - ); + }); expect(mockElasticsearchClient.bulk).toHaveBeenCalledTimes(1); - expect(mockElasticsearchClient.bulk).toHaveBeenCalledWith({ - index: indexName, - operations: [{ delete: { _id: sessionValue._id } }], - }); + expect(mockElasticsearchClient.bulk).toHaveBeenCalledWith( + { + index: indexName, + operations: [{ delete: { _id: sessionValue._id } }], + }, + { + ignore: [409, 404], + } + ); }); it('when only `lifespan` is configured', async () => { @@ -334,70 +338,73 @@ describe('Session index', () => { await sessionIndex.cleanUp(); expect(mockElasticsearchClient.search).toHaveBeenCalledTimes(1); - expect(mockElasticsearchClient.search).toHaveBeenCalledWith( - { - index: indexName, - _source_includes: 'usernameHash,provider', - body: { - query: { - bool: { - should: [ - // All expired sessions based on the lifespan, no matter which provider they belong to. - { range: { lifespanExpiration: { lte: now } } }, - // All sessions that belong to the providers that aren't configured. - { - bool: { - must_not: { - bool: { - should: [ - { - bool: { - must: [ - { term: { 'provider.type': 'basic' } }, - { term: { 'provider.name': 'basic' } }, - ], - }, + expect(mockElasticsearchClient.search).toHaveBeenCalledWith({ + index: indexName, + _source_includes: 'usernameHash,provider', + size: 1_000, + body: { + query: { + bool: { + should: [ + // All expired sessions based on the lifespan, no matter which provider they belong to. + { range: { lifespanExpiration: { lte: now } } }, + // All sessions that belong to the providers that aren't configured. + { + bool: { + must_not: { + bool: { + should: [ + { + bool: { + must: [ + { term: { 'provider.type': 'basic' } }, + { term: { 'provider.name': 'basic' } }, + ], }, - ], - minimum_should_match: 1, - }, + }, + ], + minimum_should_match: 1, }, }, }, - // The sessions that belong to a particular provider but don't have a configured lifespan. - { - bool: { - must: [ - { term: { 'provider.type': 'basic' } }, - { term: { 'provider.name': 'basic' } }, - ], - must_not: { exists: { field: 'lifespanExpiration' } }, - }, + }, + // The sessions that belong to a particular provider but don't have a configured lifespan. + { + bool: { + must: [ + { term: { 'provider.type': 'basic' } }, + { term: { 'provider.name': 'basic' } }, + ], + must_not: { exists: { field: 'lifespanExpiration' } }, }, - // The sessions that belong to a particular provider that are expired based on the idle timeout. - { - bool: { - must: [ - { term: { 'provider.type': 'basic' } }, - { term: { 'provider.name': 'basic' } }, - ], - should: [{ range: { idleTimeoutExpiration: { lte: now } } }], - minimum_should_match: 1, - }, + }, + // The sessions that belong to a particular provider that are expired based on the idle timeout. + { + bool: { + must: [ + { term: { 'provider.type': 'basic' } }, + { term: { 'provider.name': 'basic' } }, + ], + should: [{ range: { idleTimeoutExpiration: { lte: now } } }], + minimum_should_match: 1, }, - ], - }, + }, + ], }, }, }, - { ignore: [409, 404] } - ); + }); expect(mockElasticsearchClient.bulk).toHaveBeenCalledTimes(1); - expect(mockElasticsearchClient.bulk).toHaveBeenCalledWith({ - index: indexName, - operations: [{ delete: { _id: sessionValue._id } }], - }); + expect(mockElasticsearchClient.bulk).toHaveBeenCalledWith( + { + index: indexName, + operations: [{ delete: { _id: sessionValue._id } }], + }, + { + ignore: [409, 404], + } + ); }); it('when only `idleTimeout` is configured', async () => { @@ -417,64 +424,67 @@ describe('Session index', () => { await sessionIndex.cleanUp(); expect(mockElasticsearchClient.search).toHaveBeenCalledTimes(1); - expect(mockElasticsearchClient.search).toHaveBeenCalledWith( - { - index: indexName, - _source_includes: 'usernameHash,provider', - body: { - query: { - bool: { - should: [ - // All expired sessions based on the lifespan, no matter which provider they belong to. - { range: { lifespanExpiration: { lte: now } } }, - // All sessions that belong to the providers that aren't configured. - { - bool: { - must_not: { - bool: { - should: [ - { - bool: { - must: [ - { term: { 'provider.type': 'basic' } }, - { term: { 'provider.name': 'basic' } }, - ], - }, + expect(mockElasticsearchClient.search).toHaveBeenCalledWith({ + index: indexName, + _source_includes: 'usernameHash,provider', + size: 1_000, + body: { + query: { + bool: { + should: [ + // All expired sessions based on the lifespan, no matter which provider they belong to. + { range: { lifespanExpiration: { lte: now } } }, + // All sessions that belong to the providers that aren't configured. + { + bool: { + must_not: { + bool: { + should: [ + { + bool: { + must: [ + { term: { 'provider.type': 'basic' } }, + { term: { 'provider.name': 'basic' } }, + ], }, - ], - minimum_should_match: 1, - }, + }, + ], + minimum_should_match: 1, }, }, }, - // The sessions that belong to a particular provider that are either expired based on the idle timeout - // or don't have it configured at all. - { - bool: { - must: [ - { term: { 'provider.type': 'basic' } }, - { term: { 'provider.name': 'basic' } }, - ], - should: [ - { range: { idleTimeoutExpiration: { lte: now - 3 * idleTimeout } } }, - { bool: { must_not: { exists: { field: 'idleTimeoutExpiration' } } } }, - ], - minimum_should_match: 1, - }, + }, + // The sessions that belong to a particular provider that are either expired based on the idle timeout + // or don't have it configured at all. + { + bool: { + must: [ + { term: { 'provider.type': 'basic' } }, + { term: { 'provider.name': 'basic' } }, + ], + should: [ + { range: { idleTimeoutExpiration: { lte: now - 3 * idleTimeout } } }, + { bool: { must_not: { exists: { field: 'idleTimeoutExpiration' } } } }, + ], + minimum_should_match: 1, }, - ], - }, + }, + ], }, }, }, - { ignore: [409, 404] } - ); + }); expect(mockElasticsearchClient.bulk).toHaveBeenCalledTimes(1); - expect(mockElasticsearchClient.bulk).toHaveBeenCalledWith({ - index: indexName, - operations: [{ delete: { _id: sessionValue._id } }], - }); + expect(mockElasticsearchClient.bulk).toHaveBeenCalledWith( + { + index: indexName, + operations: [{ delete: { _id: sessionValue._id } }], + }, + { + ignore: [409, 404], + } + ); }); it('when both `lifespan` and `idleTimeout` are configured', async () => { @@ -494,74 +504,77 @@ describe('Session index', () => { await sessionIndex.cleanUp(); expect(mockElasticsearchClient.search).toHaveBeenCalledTimes(1); - expect(mockElasticsearchClient.search).toHaveBeenCalledWith( - { - index: indexName, - _source_includes: 'usernameHash,provider', - body: { - query: { - bool: { - should: [ - // All expired sessions based on the lifespan, no matter which provider they belong to. - { range: { lifespanExpiration: { lte: now } } }, - // All sessions that belong to the providers that aren't configured. - { - bool: { - must_not: { - bool: { - should: [ - { - bool: { - must: [ - { term: { 'provider.type': 'basic' } }, - { term: { 'provider.name': 'basic' } }, - ], - }, + expect(mockElasticsearchClient.search).toHaveBeenCalledWith({ + index: indexName, + _source_includes: 'usernameHash,provider', + size: 1_000, + body: { + query: { + bool: { + should: [ + // All expired sessions based on the lifespan, no matter which provider they belong to. + { range: { lifespanExpiration: { lte: now } } }, + // All sessions that belong to the providers that aren't configured. + { + bool: { + must_not: { + bool: { + should: [ + { + bool: { + must: [ + { term: { 'provider.type': 'basic' } }, + { term: { 'provider.name': 'basic' } }, + ], }, - ], - minimum_should_match: 1, - }, + }, + ], + minimum_should_match: 1, }, }, }, - // The sessions that belong to a particular provider but don't have a configured lifespan. - { - bool: { - must: [ - { term: { 'provider.type': 'basic' } }, - { term: { 'provider.name': 'basic' } }, - ], - must_not: { exists: { field: 'lifespanExpiration' } }, - }, + }, + // The sessions that belong to a particular provider but don't have a configured lifespan. + { + bool: { + must: [ + { term: { 'provider.type': 'basic' } }, + { term: { 'provider.name': 'basic' } }, + ], + must_not: { exists: { field: 'lifespanExpiration' } }, }, - // The sessions that belong to a particular provider that are either expired based on the idle timeout - // or don't have it configured at all. - { - bool: { - must: [ - { term: { 'provider.type': 'basic' } }, - { term: { 'provider.name': 'basic' } }, - ], - should: [ - { range: { idleTimeoutExpiration: { lte: now - 3 * idleTimeout } } }, - { bool: { must_not: { exists: { field: 'idleTimeoutExpiration' } } } }, - ], - minimum_should_match: 1, - }, + }, + // The sessions that belong to a particular provider that are either expired based on the idle timeout + // or don't have it configured at all. + { + bool: { + must: [ + { term: { 'provider.type': 'basic' } }, + { term: { 'provider.name': 'basic' } }, + ], + should: [ + { range: { idleTimeoutExpiration: { lte: now - 3 * idleTimeout } } }, + { bool: { must_not: { exists: { field: 'idleTimeoutExpiration' } } } }, + ], + minimum_should_match: 1, }, - ], - }, + }, + ], }, }, }, - { ignore: [409, 404] } - ); + }); expect(mockElasticsearchClient.bulk).toHaveBeenCalledTimes(1); - expect(mockElasticsearchClient.bulk).toHaveBeenCalledWith({ - index: indexName, - operations: [{ delete: { _id: sessionValue._id } }], - }); + expect(mockElasticsearchClient.bulk).toHaveBeenCalledWith( + { + index: indexName, + operations: [{ delete: { _id: sessionValue._id } }], + }, + { + ignore: [409, 404], + } + ); }); it('when both `lifespan` and `idleTimeout` are configured and multiple providers are enabled', async () => { @@ -596,107 +609,127 @@ describe('Session index', () => { await sessionIndex.cleanUp(); expect(mockElasticsearchClient.search).toHaveBeenCalledTimes(1); - expect(mockElasticsearchClient.search).toHaveBeenCalledWith( - { - index: indexName, - _source_includes: 'usernameHash,provider', - body: { - query: { - bool: { - should: [ - // All expired sessions based on the lifespan, no matter which provider they belong to. - { range: { lifespanExpiration: { lte: now } } }, - // All sessions that belong to the providers that aren't configured. - { - bool: { - must_not: { - bool: { - should: [ - { - bool: { - must: [ - { term: { 'provider.type': 'basic' } }, - { term: { 'provider.name': 'basic1' } }, - ], - }, + expect(mockElasticsearchClient.search).toHaveBeenCalledWith({ + index: indexName, + _source_includes: 'usernameHash,provider', + size: 1_000, + body: { + query: { + bool: { + should: [ + // All expired sessions based on the lifespan, no matter which provider they belong to. + { range: { lifespanExpiration: { lte: now } } }, + // All sessions that belong to the providers that aren't configured. + { + bool: { + must_not: { + bool: { + should: [ + { + bool: { + must: [ + { term: { 'provider.type': 'basic' } }, + { term: { 'provider.name': 'basic1' } }, + ], }, - { - bool: { - must: [ - { term: { 'provider.type': 'saml' } }, - { term: { 'provider.name': 'saml1' } }, - ], - }, + }, + { + bool: { + must: [ + { term: { 'provider.type': 'saml' } }, + { term: { 'provider.name': 'saml1' } }, + ], }, - ], - minimum_should_match: 1, - }, + }, + ], + minimum_should_match: 1, }, }, }, - // The sessions that belong to a Basic provider but don't have a configured lifespan. - { - bool: { - must: [ - { term: { 'provider.type': 'basic' } }, - { term: { 'provider.name': 'basic1' } }, - ], - must_not: { exists: { field: 'lifespanExpiration' } }, - }, + }, + // The sessions that belong to a Basic provider but don't have a configured lifespan. + { + bool: { + must: [ + { term: { 'provider.type': 'basic' } }, + { term: { 'provider.name': 'basic1' } }, + ], + must_not: { exists: { field: 'lifespanExpiration' } }, }, - // The sessions that belong to a Basic provider that are either expired based on the idle timeout - // or don't have it configured at all. - { - bool: { - must: [ - { term: { 'provider.type': 'basic' } }, - { term: { 'provider.name': 'basic1' } }, - ], - should: [ - { range: { idleTimeoutExpiration: { lte: now - 3 * globalIdleTimeout } } }, - { bool: { must_not: { exists: { field: 'idleTimeoutExpiration' } } } }, - ], - minimum_should_match: 1, - }, + }, + // The sessions that belong to a Basic provider that are either expired based on the idle timeout + // or don't have it configured at all. + { + bool: { + must: [ + { term: { 'provider.type': 'basic' } }, + { term: { 'provider.name': 'basic1' } }, + ], + should: [ + { range: { idleTimeoutExpiration: { lte: now - 3 * globalIdleTimeout } } }, + { bool: { must_not: { exists: { field: 'idleTimeoutExpiration' } } } }, + ], + minimum_should_match: 1, }, - // The sessions that belong to a SAML provider but don't have a configured lifespan. - { - bool: { - must: [ - { term: { 'provider.type': 'saml' } }, - { term: { 'provider.name': 'saml1' } }, - ], - must_not: { exists: { field: 'lifespanExpiration' } }, - }, + }, + // The sessions that belong to a SAML provider but don't have a configured lifespan. + { + bool: { + must: [ + { term: { 'provider.type': 'saml' } }, + { term: { 'provider.name': 'saml1' } }, + ], + must_not: { exists: { field: 'lifespanExpiration' } }, }, - // The sessions that belong to a SAML provider that are either expired based on the idle timeout - // or don't have it configured at all. - { - bool: { - must: [ - { term: { 'provider.type': 'saml' } }, - { term: { 'provider.name': 'saml1' } }, - ], - should: [ - { range: { idleTimeoutExpiration: { lte: now - 3 * samlIdleTimeout } } }, - { bool: { must_not: { exists: { field: 'idleTimeoutExpiration' } } } }, - ], - minimum_should_match: 1, - }, + }, + // The sessions that belong to a SAML provider that are either expired based on the idle timeout + // or don't have it configured at all. + { + bool: { + must: [ + { term: { 'provider.type': 'saml' } }, + { term: { 'provider.name': 'saml1' } }, + ], + should: [ + { range: { idleTimeoutExpiration: { lte: now - 3 * samlIdleTimeout } } }, + { bool: { must_not: { exists: { field: 'idleTimeoutExpiration' } } } }, + ], + minimum_should_match: 1, }, - ], - }, + }, + ], }, }, }, - { ignore: [409, 404] } - ); + }); expect(mockElasticsearchClient.bulk).toHaveBeenCalledTimes(1); - expect(mockElasticsearchClient.bulk).toHaveBeenCalledWith({ - index: indexName, - operations: [{ delete: { _id: sessionValue._id } }], - }); + expect(mockElasticsearchClient.bulk).toHaveBeenCalledWith( + { + index: indexName, + operations: [{ delete: { _id: sessionValue._id } }], + }, + { + ignore: [409, 404], + } + ); + }); + + it('should clean up sessions in batches of 1000', async () => { + for (const value of [2500, 1500, 500]) { + mockElasticsearchClient.search.mockResolvedValueOnce( + securityMock.createApiResponse({ + body: { + hits: { total: { value, relation: 'eq' }, hits: [sessionValue] }, + } as SearchResponse, + }) + ); + } + + await sessionIndex.cleanUp(); + + expect(mockElasticsearchClient.search).toHaveBeenCalledTimes(3); + expect(mockElasticsearchClient.bulk).toHaveBeenCalledTimes(3); }); it('should log audit event', async () => { diff --git a/x-pack/plugins/security/server/session_management/session_index.ts b/x-pack/plugins/security/server/session_management/session_index.ts index ef04f5704b040..df3fa30ffb8aa 100644 --- a/x-pack/plugins/security/server/session_management/session_index.ts +++ b/x-pack/plugins/security/server/session_management/session_index.ts @@ -5,7 +5,10 @@ * 2.0. */ -import type { BulkOperationContainer } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import type { + BulkOperationContainer, + SearchTotalHits, +} from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import type { ElasticsearchClient, Logger } from 'src/core/server'; @@ -39,6 +42,11 @@ export type InvalidateSessionsFilter = */ const SESSION_INDEX_TEMPLATE_VERSION = 1; +/** + * Number of sessions to remove per batch during cleanup. Must be below 10,000 (maximum pagination size). + */ +const SESSION_INDEX_CLEANUP_BATCH_SIZE = 1_000; + /** * Returns index template that is used for the current version of the session index. */ @@ -490,16 +498,16 @@ export class SessionIndex { } const operations: Array>> = []; + let total = 0; try { const { body: searchResponse } = - await this.options.elasticsearchClient.search( - { - index: this.indexName, - body: { query: { bool: { should: deleteQueries } } }, - _source_includes: 'usernameHash,provider', - }, - { ignore: [409, 404] } - ); + await this.options.elasticsearchClient.search({ + index: this.indexName, + body: { query: { bool: { should: deleteQueries } } }, + _source_includes: 'usernameHash,provider', + size: SESSION_INDEX_CLEANUP_BATCH_SIZE, + }); + total = (searchResponse.hits.total as SearchTotalHits).value; searchResponse.hits.hits.forEach(({ _id, _source }) => { const { usernameHash, provider } = _source!; this.options.auditLogger.log( @@ -514,10 +522,14 @@ export class SessionIndex { if (operations.length > 0) { try { - const { body: deleteResponse } = await this.options.elasticsearchClient.bulk({ - index: this.indexName, - operations, - }); + const { body: deleteResponse } = await this.options.elasticsearchClient.bulk( + { + index: this.indexName, + operations, + refresh: false, + }, + { ignore: [409, 404] } + ); if (deleteResponse.errors) { const errorCount = deleteResponse.items.reduce( (count, item) => (item.delete!.error ? count + 1 : count), @@ -542,5 +554,9 @@ export class SessionIndex { throw err; } } + + if (total > SESSION_INDEX_CLEANUP_BATCH_SIZE) { + await this.cleanUp(); + } } }