diff --git a/docs/development/core/public/kibana-plugin-core-public.kibanaexecutioncontext.md b/docs/development/core/public/kibana-plugin-core-public.kibanaexecutioncontext.md index afd66fbba18d3..06154c814c4e7 100644 --- a/docs/development/core/public/kibana-plugin-core-public.kibanaexecutioncontext.md +++ b/docs/development/core/public/kibana-plugin-core-public.kibanaexecutioncontext.md @@ -18,6 +18,6 @@ export interface KibanaExecutionContext | [description](./kibana-plugin-core-public.kibanaexecutioncontext.description.md) | string | human readable description. For example, a vis title, action name | | [id](./kibana-plugin-core-public.kibanaexecutioncontext.id.md) | string | unique value to identify the source | | [name](./kibana-plugin-core-public.kibanaexecutioncontext.name.md) | string | public name of a user-facing feature | -| [type](./kibana-plugin-core-public.kibanaexecutioncontext.type.md) | string | Kibana application initated an operation. Can be narrowed to an enum later. | +| [type](./kibana-plugin-core-public.kibanaexecutioncontext.type.md) | string | Kibana application initated an operation. | | [url](./kibana-plugin-core-public.kibanaexecutioncontext.url.md) | string | in browser - url to navigate to a current page, on server - endpoint path, for task: task SO url | diff --git a/docs/development/core/public/kibana-plugin-core-public.kibanaexecutioncontext.type.md b/docs/development/core/public/kibana-plugin-core-public.kibanaexecutioncontext.type.md index ca339ddd9d646..48009ebaaeaa5 100644 --- a/docs/development/core/public/kibana-plugin-core-public.kibanaexecutioncontext.type.md +++ b/docs/development/core/public/kibana-plugin-core-public.kibanaexecutioncontext.type.md @@ -4,7 +4,7 @@ ## KibanaExecutionContext.type property -Kibana application initated an operation. Can be narrowed to an enum later. +Kibana application initated an operation. Signature: diff --git a/docs/development/core/server/kibana-plugin-core-server.md b/docs/development/core/server/kibana-plugin-core-server.md index a26f8bd7b1594..a3f925151af61 100644 --- a/docs/development/core/server/kibana-plugin-core-server.md +++ b/docs/development/core/server/kibana-plugin-core-server.md @@ -329,6 +329,7 @@ The plugin integrates with the core system via lifecycle events: `setup` | [SavedObjectsImportHook](./kibana-plugin-core-server.savedobjectsimporthook.md) | A hook associated with a specific saved object type, that will be invoked during the import process. The hook will have access to the objects of the registered type.Currently, the only supported feature for import hooks is to return warnings to be displayed in the UI when the import succeeds. The only interactions the hook can have with the import process is via the hook's response. Mutating the objects inside the hook's code will have no effect. | | [SavedObjectsImportWarning](./kibana-plugin-core-server.savedobjectsimportwarning.md) | Composite type of all the possible types of import warnings.See [SavedObjectsImportSimpleWarning](./kibana-plugin-core-server.savedobjectsimportsimplewarning.md) and [SavedObjectsImportActionRequiredWarning](./kibana-plugin-core-server.savedobjectsimportactionrequiredwarning.md) for more details. | | [SavedObjectsNamespaceType](./kibana-plugin-core-server.savedobjectsnamespacetype.md) | The namespace type dictates how a saved object can be interacted in relation to namespaces. Each type is mutually exclusive: \* single (default): This type of saved object is namespace-isolated, e.g., it exists in only one namespace. \* multiple: This type of saved object is shareable, e.g., it can exist in one or more namespaces. \* multiple-isolated: This type of saved object is namespace-isolated, e.g., it exists in only one namespace, but object IDs must be unique across all namespaces. This is intended to be an intermediate step when objects with a "single" namespace type are being converted to a "multiple" namespace type. In other words, objects with a "multiple-isolated" namespace type will be \*share-capable\*, but will not actually be shareable until the namespace type is changed to "multiple". \* agnostic: This type of saved object is global. | +| [SavedObjectTypeExcludeFromUpgradeFilterHook](./kibana-plugin-core-server.savedobjecttypeexcludefromupgradefilterhook.md) | If defined, allows a type to run a search query and return a query filter that may match any documents which may be excluded from the next migration upgrade process. Useful for cleaning up large numbers of old documents which are no longer needed and may slow the migration process.If this hook fails, the migration will proceed without these documents having been filtered out, so this should not be used as a guarantee that these documents have been deleted.Experimental and subject to change | | [SavedObjectUnsanitizedDoc](./kibana-plugin-core-server.savedobjectunsanitizeddoc.md) | Describes Saved Object documents from Kibana < 7.0.0 which don't have a references root property defined. This type should only be used in migrations. | | [ScopeableRequest](./kibana-plugin-core-server.scopeablerequest.md) | A user credentials container. It accommodates the necessary auth credentials to impersonate the current user.See [KibanaRequest](./kibana-plugin-core-server.kibanarequest.md). | | [ServiceStatusLevel](./kibana-plugin-core-server.servicestatuslevel.md) | A convenience type that represents the union of each value in [ServiceStatusLevels](./kibana-plugin-core-server.servicestatuslevels.md). | diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectstype.excludeonupgrade.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectstype.excludeonupgrade.md new file mode 100644 index 0000000000000..f5446b37dc29c --- /dev/null +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectstype.excludeonupgrade.md @@ -0,0 +1,13 @@ + + +[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [SavedObjectsType](./kibana-plugin-core-server.savedobjectstype.md) > [excludeOnUpgrade](./kibana-plugin-core-server.savedobjectstype.excludeonupgrade.md) + +## SavedObjectsType.excludeOnUpgrade property + +If defined, allows a type to exclude unneeded documents from the migration process and effectively be deleted. See [SavedObjectTypeExcludeFromUpgradeFilterHook](./kibana-plugin-core-server.savedobjecttypeexcludefromupgradefilterhook.md) for more details. + +Signature: + +```typescript +excludeOnUpgrade?: SavedObjectTypeExcludeFromUpgradeFilterHook; +``` diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectstype.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectstype.md index c3aba5261561f..bdb90be2bc8fc 100644 --- a/docs/development/core/server/kibana-plugin-core-server.savedobjectstype.md +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectstype.md @@ -52,6 +52,7 @@ Example after converting to a multi-namespace (shareable) type in 8.1: ``` Note: migration function(s) can be optionally specified for any of these versions and will not interfere with the conversion process. | +| [excludeOnUpgrade](./kibana-plugin-core-server.savedobjectstype.excludeonupgrade.md) | SavedObjectTypeExcludeFromUpgradeFilterHook | If defined, allows a type to exclude unneeded documents from the migration process and effectively be deleted. See [SavedObjectTypeExcludeFromUpgradeFilterHook](./kibana-plugin-core-server.savedobjecttypeexcludefromupgradefilterhook.md) for more details. | | [hidden](./kibana-plugin-core-server.savedobjectstype.hidden.md) | boolean | Is the type hidden by default. If true, repositories will not have access to this type unless explicitly declared as an extraType when creating the repository.See [createInternalRepository](./kibana-plugin-core-server.savedobjectsservicestart.createinternalrepository.md). | | [indexPattern](./kibana-plugin-core-server.savedobjectstype.indexpattern.md) | string | If defined, the type instances will be stored in the given index instead of the default one. | | [management](./kibana-plugin-core-server.savedobjectstype.management.md) | SavedObjectsTypeManagementDefinition<Attributes> | An optional [saved objects management section](./kibana-plugin-core-server.savedobjectstypemanagementdefinition.md) definition for the type. | diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjecttypeexcludefromupgradefilterhook.md b/docs/development/core/server/kibana-plugin-core-server.savedobjecttypeexcludefromupgradefilterhook.md new file mode 100644 index 0000000000000..bd1c43529895c --- /dev/null +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjecttypeexcludefromupgradefilterhook.md @@ -0,0 +1,19 @@ + + +[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [SavedObjectTypeExcludeFromUpgradeFilterHook](./kibana-plugin-core-server.savedobjecttypeexcludefromupgradefilterhook.md) + +## SavedObjectTypeExcludeFromUpgradeFilterHook type + +If defined, allows a type to run a search query and return a query filter that may match any documents which may be excluded from the next migration upgrade process. Useful for cleaning up large numbers of old documents which are no longer needed and may slow the migration process. + +If this hook fails, the migration will proceed without these documents having been filtered out, so this should not be used as a guarantee that these documents have been deleted. + +Experimental and subject to change + +Signature: + +```typescript +export declare type SavedObjectTypeExcludeFromUpgradeFilterHook = (toolkit: { + readonlyEsClient: Pick; +}) => estypes.QueryDslQueryContainer | Promise; +``` diff --git a/src/core/server/index.ts b/src/core/server/index.ts index c77a3a967364c..191a3bd972f15 100644 --- a/src/core/server/index.ts +++ b/src/core/server/index.ts @@ -312,6 +312,7 @@ export type { SavedObjectsCreatePointInTimeFinderDependencies, SavedObjectsCreatePointInTimeFinderOptions, SavedObjectsCreateOptions, + SavedObjectTypeExcludeFromUpgradeFilterHook, SavedObjectsExportResultDetails, SavedObjectsExportExcludedObject, SavedObjectsFindResult, diff --git a/src/core/server/saved_objects/index.ts b/src/core/server/saved_objects/index.ts index 5f853d49219dc..b689b348e9e4c 100644 --- a/src/core/server/saved_objects/index.ts +++ b/src/core/server/saved_objects/index.ts @@ -90,6 +90,7 @@ export type { SavedObjectStatusMeta, SavedObjectsType, SavedObjectsTypeManagementDefinition, + SavedObjectTypeExcludeFromUpgradeFilterHook, } from './types'; export { savedObjectsConfig, savedObjectsMigrationConfig } from './saved_objects_config'; diff --git a/src/core/server/saved_objects/migrationsv2/actions/calculate_exclude_filters.test.ts b/src/core/server/saved_objects/migrationsv2/actions/calculate_exclude_filters.test.ts new file mode 100644 index 0000000000000..0f41233fd58f2 --- /dev/null +++ b/src/core/server/saved_objects/migrationsv2/actions/calculate_exclude_filters.test.ts @@ -0,0 +1,113 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { errors as esErrors } from '@elastic/elasticsearch'; +import { elasticsearchClientMock } from '../../../elasticsearch/client/mocks'; +import { calculateExcludeFilters } from './calculate_exclude_filters'; + +describe('calculateExcludeFilters', () => { + const client = elasticsearchClientMock.createInternalClient(); + + it('calls each provided hook and returns combined filter', async () => { + const hook1 = jest.fn().mockReturnValue({ bool: { must: { term: { fieldA: '123' } } } }); + const hook2 = jest.fn().mockResolvedValue({ bool: { must: { term: { fieldB: 'abc' } } } }); + + const task = calculateExcludeFilters({ + client, + excludeFromUpgradeFilterHooks: { type1: hook1, type2: hook2 }, + }); + const result = await task(); + + expect(hook1).toHaveBeenCalledWith({ readonlyEsClient: { search: expect.any(Function) } }); + expect(hook2).toHaveBeenCalledWith({ readonlyEsClient: { search: expect.any(Function) } }); + expect(Either.isRight(result)).toBe(true); + expect((result as Either.Right).right).toEqual({ + excludeFilter: { + bool: { + must_not: [ + { bool: { must: { term: { fieldA: '123' } } } }, + { bool: { must: { term: { fieldB: 'abc' } } } }, + ], + }, + }, + errorsByType: {}, + }); + }); + + it('ignores hooks that return non-retryable errors', async () => { + const error = new Error('blah!'); + const hook1 = jest.fn().mockRejectedValue(error); + const hook2 = jest.fn().mockResolvedValue({ bool: { must: { term: { fieldB: 'abc' } } } }); + + const task = calculateExcludeFilters({ + client, + excludeFromUpgradeFilterHooks: { type1: hook1, type2: hook2 }, + }); + const result = await task(); + + expect(Either.isRight(result)).toBe(true); + expect((result as Either.Right).right).toEqual({ + excludeFilter: { + bool: { + must_not: [{ bool: { must: { term: { fieldB: 'abc' } } } }], + }, + }, + errorsByType: { type1: error }, + }); + }); + + it('returns Either.Left if a hook returns a retryable error', async () => { + const error = new esErrors.NoLivingConnectionsError( + 'reason', + elasticsearchClientMock.createApiResponse() + ); + const hook1 = jest.fn().mockRejectedValue(error); + const hook2 = jest.fn().mockResolvedValue({ bool: { must: { term: { fieldB: 'abc' } } } }); + + const task = calculateExcludeFilters({ + client, + excludeFromUpgradeFilterHooks: { type1: hook1, type2: hook2 }, + }); + const result = await task(); + + expect(Either.isLeft(result)).toBe(true); + expect((result as Either.Left).left).toEqual({ + type: 'retryable_es_client_error', + message: 'reason', + error, + }); + }); + + it('ignores and returns errors for hooks that take longer than timeout', async () => { + const hook1 = jest.fn().mockReturnValue(new Promise((r) => setTimeout(r, 40_000))); + const hook2 = jest.fn().mockResolvedValue({ bool: { must: { term: { fieldB: 'abc' } } } }); + + const task = calculateExcludeFilters({ + client, + excludeFromUpgradeFilterHooks: { type1: hook1, type2: hook2 }, + hookTimeoutMs: 100, + }); + const resultPromise = task(); + await new Promise((r) => setTimeout(r, 110)); + const result = await resultPromise; + + expect(Either.isRight(result)).toBe(true); + expect((result as Either.Right).right).toEqual({ + excludeFilter: { + bool: { + must_not: [{ bool: { must: { term: { fieldB: 'abc' } } } }], + }, + }, + errorsByType: expect.any(Object), + }); + expect((result as Either.Right).right.errorsByType.type1.toString()).toMatchInlineSnapshot( + `"Error: excludeFromUpgrade hook timed out after 0.1 seconds."` + ); + }); +}); diff --git a/src/core/server/saved_objects/migrationsv2/actions/calculate_exclude_filters.ts b/src/core/server/saved_objects/migrationsv2/actions/calculate_exclude_filters.ts new file mode 100644 index 0000000000000..9d57db079e33e --- /dev/null +++ b/src/core/server/saved_objects/migrationsv2/actions/calculate_exclude_filters.ts @@ -0,0 +1,107 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { estypes } from '@elastic/elasticsearch'; +import { withTimeout } from '@kbn/std'; +import * as Either from 'fp-ts/lib/Either'; +import * as TaskEither from 'fp-ts/lib/TaskEither'; +import { RetryableEsClientError } from '.'; +import { ElasticsearchClient } from '../../../elasticsearch'; +import { SavedObjectTypeExcludeFromUpgradeFilterHook } from '../../types'; +import { catchRetryableEsClientErrors } from './catch_retryable_es_client_errors'; + +export interface CalculateExcludeFiltersParams { + client: ElasticsearchClient; + excludeFromUpgradeFilterHooks: Record; + hookTimeoutMs?: number; +} + +export interface CalculatedExcludeFilter { + /** Composite filter of all calculated filters */ + excludeFilter: estypes.QueryDslQueryContainer; + /** Any errors that were encountered during filter calculation, keyed by the type name */ + errorsByType: Record; +} + +export const calculateExcludeFilters = ({ + client, + excludeFromUpgradeFilterHooks, + hookTimeoutMs = 30_000, // default to 30s, exposed for testing +}: CalculateExcludeFiltersParams): TaskEither.TaskEither< + RetryableEsClientError, + CalculatedExcludeFilter +> => () => { + return Promise.all< + | Either.Right + | Either.Left<{ soType: string; error: Error | RetryableEsClientError }> + >( + Object.entries(excludeFromUpgradeFilterHooks).map(([soType, hook]) => + withTimeout({ + promise: Promise.resolve( + hook({ readonlyEsClient: { search: client.search.bind(client) } }) + ), + timeoutMs: hookTimeoutMs, + }) + .then((result) => + result.timedout + ? Either.left({ + soType, + error: new Error( + `excludeFromUpgrade hook timed out after ${hookTimeoutMs / 1000} seconds.` + ), + }) + : Either.right(result.value) + ) + .catch((error) => { + const retryableError = catchRetryableEsClientErrors(error); + if (Either.isLeft(retryableError)) { + return Either.left({ soType, error: retryableError.left }); + } else { + // Really should never happen, only here to satisfy TypeScript + return Either.left({ + soType, + error: new Error( + `Unexpected return value from catchRetryableEsClientErrors: "${retryableError.toString()}"` + ), + }); + } + }) + .catch((error: Error) => Either.left({ soType, error })) + ) + ).then((results) => { + const retryableError = results.find( + (r) => + Either.isLeft(r) && + !(r.left.error instanceof Error) && + r.left.error.type === 'retryable_es_client_error' + ) as Either.Left<{ soType: string; error: RetryableEsClientError }> | undefined; + if (retryableError) { + return Either.left(retryableError.left.error); + } + + const errorsByType: Array<[string, Error]> = []; + const filters: estypes.QueryDslQueryContainer[] = []; + + // Loop through all results and collect successes and errors + results.forEach((r) => + Either.isRight(r) + ? filters.push(r.right) + : Either.isLeft(r) && errorsByType.push([r.left.soType, r.left.error as Error]) + ); + + // Composite filter from all calculated filters that successfully executed + const excludeFilter: estypes.QueryDslQueryContainer = { + bool: { must_not: filters }, + }; + + return Either.right({ + excludeFilter, + errorsByType: Object.fromEntries(errorsByType), + }); + }); +}; diff --git a/src/core/server/saved_objects/migrationsv2/actions/index.ts b/src/core/server/saved_objects/migrationsv2/actions/index.ts index 6bfcddfe1f6de..97ffcff25dd76 100644 --- a/src/core/server/saved_objects/migrationsv2/actions/index.ts +++ b/src/core/server/saved_objects/migrationsv2/actions/index.ts @@ -98,6 +98,12 @@ export { searchForOutdatedDocuments } from './search_for_outdated_documents'; export type { BulkOverwriteTransformedDocumentsParams } from './bulk_overwrite_transformed_documents'; export { bulkOverwriteTransformedDocuments } from './bulk_overwrite_transformed_documents'; +export type { + CalculateExcludeFiltersParams, + CalculatedExcludeFilter, +} from './calculate_exclude_filters'; +export { calculateExcludeFilters } from './calculate_exclude_filters'; + export { pickupUpdatedMappings, waitForTask, waitForIndexStatusYellow }; export type { AliasNotFound, RemoveIndexNotAConcreteIndex }; diff --git a/src/core/server/saved_objects/migrationsv2/initial_state.test.ts b/src/core/server/saved_objects/migrationsv2/initial_state.test.ts index 74d29c8fa15a9..4066efeb65de0 100644 --- a/src/core/server/saved_objects/migrationsv2/initial_state.test.ts +++ b/src/core/server/saved_objects/migrationsv2/initial_state.test.ts @@ -39,6 +39,7 @@ describe('createInitialState', () => { batchSize: 1000, controlState: 'INIT', currentAlias: '.kibana_task_manager', + excludeFromUpgradeFilterHooks: {}, indexPrefix: '.kibana_task_manager', kibanaVersion: '8.1.0', knownTypes: [], @@ -136,6 +137,31 @@ describe('createInitialState', () => { expect(initialState.knownTypes).toEqual(['foo', 'bar']); }); + it('returns state with the correct `excludeFromUpgradeFilterHooks`', () => { + const fooExcludeOnUpgradeHook = jest.fn(); + typeRegistry.registerType({ + name: 'foo', + namespaceType: 'single', + hidden: false, + mappings: { properties: {} }, + excludeOnUpgrade: fooExcludeOnUpgradeHook, + }); + + const initialState = createInitialState({ + kibanaVersion: '8.1.0', + targetMappings: { + dynamic: 'strict', + properties: { my_type: { properties: { title: { type: 'text' } } } }, + }, + migrationVersionPerType: {}, + indexPrefix: '.kibana_task_manager', + migrationsConfig, + typeRegistry, + }); + + expect(initialState.excludeFromUpgradeFilterHooks).toEqual({ foo: fooExcludeOnUpgradeHook }); + }); + it('returns state with a preMigration script', () => { const preMigrationScript = "ctx._id = ctx._source.type + ':' + ctx._id"; const initialState = createInitialState({ diff --git a/src/core/server/saved_objects/migrationsv2/initial_state.ts b/src/core/server/saved_objects/migrationsv2/initial_state.ts index e13a4b441fb25..dce37b384a4f7 100644 --- a/src/core/server/saved_objects/migrationsv2/initial_state.ts +++ b/src/core/server/saved_objects/migrationsv2/initial_state.ts @@ -58,6 +58,12 @@ export const createInitialState = ({ }; const knownTypes = typeRegistry.getAllTypes().map((type) => type.name); + const excludeFilterHooks = Object.fromEntries( + typeRegistry + .getAllTypes() + .filter((type) => !!type.excludeOnUpgrade) + .map((type) => [type.name, type.excludeOnUpgrade!]) + ); return { controlState: 'INIT', @@ -79,5 +85,6 @@ export const createInitialState = ({ logs: [], unusedTypesQuery: excludeUnusedTypesQuery, knownTypes, + excludeFromUpgradeFilterHooks: excludeFilterHooks, }; }; diff --git a/src/core/server/saved_objects/migrationsv2/integration_tests/archives/7.13_1.5k_failed_action_tasks.zip b/src/core/server/saved_objects/migrationsv2/integration_tests/archives/7.13_1.5k_failed_action_tasks.zip new file mode 100644 index 0000000000000..69aa04d7fc40f Binary files /dev/null and b/src/core/server/saved_objects/migrationsv2/integration_tests/archives/7.13_1.5k_failed_action_tasks.zip differ diff --git a/src/core/server/saved_objects/migrationsv2/integration_tests/migration_7_13_0_failed_action_tasks.test.ts b/src/core/server/saved_objects/migrationsv2/integration_tests/migration_7_13_0_failed_action_tasks.test.ts new file mode 100644 index 0000000000000..2756fb8fd8185 --- /dev/null +++ b/src/core/server/saved_objects/migrationsv2/integration_tests/migration_7_13_0_failed_action_tasks.test.ts @@ -0,0 +1,147 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import Path from 'path'; +import fs from 'fs/promises'; +import * as kbnTestServer from '../../../../test_helpers/kbn_server'; +import { Root } from '../../../root'; +import { ElasticsearchClient } from '../../../elasticsearch'; + +const logFilePath = Path.join(__dirname, '7_13_failed_action_tasks_test.log'); + +async function removeLogFile() { + // ignore errors if it doesn't exist + await fs.unlink(logFilePath).catch(() => void 0); +} + +describe('migration from 7.13 to 7.14+ with many failed action_tasks', () => { + let esServer: kbnTestServer.TestElasticsearchUtils; + let root: Root; + let startES: () => Promise; + + beforeAll(async () => { + await removeLogFile(); + }); + + beforeEach(() => { + ({ startES } = kbnTestServer.createTestServers({ + adjustTimeout: (t: number) => jest.setTimeout(t), + settings: { + es: { + license: 'basic', + dataArchive: Path.join(__dirname, 'archives', '7.13_1.5k_failed_action_tasks.zip'), + }, + }, + })); + }); + + afterEach(async () => { + if (root) { + await root.shutdown(); + } + if (esServer) { + await esServer.stop(); + } + + await new Promise((resolve) => setTimeout(resolve, 10000)); + }); + + const getCounts = async ( + kibanaIndexName = '.kibana', + taskManagerIndexName = '.kibana_task_manager' + ): Promise<{ tasksCount: number; actionTaskParamsCount: number }> => { + const esClient: ElasticsearchClient = esServer.es.getClient(); + + const actionTaskParamsResponse = await esClient.count({ + index: kibanaIndexName, + body: { + query: { + bool: { must: { term: { type: 'action_task_params' } } }, + }, + }, + }); + const tasksResponse = await esClient.count({ + index: taskManagerIndexName, + body: { + query: { + bool: { must: { term: { type: 'task' } } }, + }, + }, + }); + + return { + actionTaskParamsCount: actionTaskParamsResponse.body.count, + tasksCount: tasksResponse.body.count, + }; + }; + + it('filters out all outdated action_task_params and action tasks', async () => { + esServer = await startES(); + + // Verify counts in current index before migration starts + expect(await getCounts()).toEqual({ + actionTaskParamsCount: 2010, + tasksCount: 2020, + }); + + root = createRoot(); + await root.preboot(); + await root.setup(); + await root.start(); + + // Bulk of tasks should have been filtered out of current index + const { actionTaskParamsCount, tasksCount } = await getCounts(); + // Use toBeLessThan to avoid flakiness in the case that TM starts manipulating docs before the counts are taken + expect(actionTaskParamsCount).toBeLessThan(1000); + expect(tasksCount).toBeLessThan(1000); + + // Verify that docs were not deleted from old index + expect(await getCounts('.kibana_7.13.5_001', '.kibana_task_manager_7.13.5_001')).toEqual({ + actionTaskParamsCount: 2010, + tasksCount: 2020, + }); + }); +}); + +function createRoot() { + return kbnTestServer.createRootWithCorePlugins( + { + migrations: { + skip: false, + enableV2: true, + batchSize: 250, + }, + logging: { + appenders: { + file: { + type: 'file', + fileName: logFilePath, + layout: { + type: 'json', + }, + }, + }, + loggers: [ + { + name: 'root', + appenders: ['file'], + }, + ], + }, + xpack: { + reporting: { + // browser install causes issues with Jest + enabled: false, + }, + }, + }, + { + oss: false, + } + ); +} diff --git a/src/core/server/saved_objects/migrationsv2/migrations_state_action_machine.test.ts b/src/core/server/saved_objects/migrationsv2/migrations_state_action_machine.test.ts index d5de9b11db1c8..d4862cddf2666 100644 --- a/src/core/server/saved_objects/migrationsv2/migrations_state_action_machine.test.ts +++ b/src/core/server/saved_objects/migrationsv2/migrations_state_action_machine.test.ts @@ -259,6 +259,7 @@ describe('migrationsStateActionMachine', () => { batchSize: 1000, controlState: 'LEGACY_DELETE', currentAlias: '.my-so-index', + excludeFromUpgradeFilterHooks: {}, indexPrefix: '.my-so-index', kibanaVersion: '7.11.0', knownTypes: [], @@ -305,6 +306,7 @@ describe('migrationsStateActionMachine', () => { batchSize: 1000, controlState: 'FATAL', currentAlias: '.my-so-index', + excludeFromUpgradeFilterHooks: {}, indexPrefix: '.my-so-index', kibanaVersion: '7.11.0', knownTypes: [], @@ -444,6 +446,7 @@ describe('migrationsStateActionMachine', () => { batchSize: 1000, controlState: 'LEGACY_REINDEX', currentAlias: '.my-so-index', + excludeFromUpgradeFilterHooks: {}, indexPrefix: '.my-so-index', kibanaVersion: '7.11.0', knownTypes: [], @@ -484,6 +487,7 @@ describe('migrationsStateActionMachine', () => { batchSize: 1000, controlState: 'LEGACY_DELETE', currentAlias: '.my-so-index', + excludeFromUpgradeFilterHooks: {}, indexPrefix: '.my-so-index', kibanaVersion: '7.11.0', knownTypes: [], diff --git a/src/core/server/saved_objects/migrationsv2/model/model.test.ts b/src/core/server/saved_objects/migrationsv2/model/model.test.ts index 3058f586efb0c..0837183b2a116 100644 --- a/src/core/server/saved_objects/migrationsv2/model/model.test.ts +++ b/src/core/server/saved_objects/migrationsv2/model/model.test.ts @@ -39,6 +39,7 @@ import type { TransformedDocumentsBulkIndex, ReindexSourceToTempIndexBulk, CheckUnknownDocumentsState, + CalculateExcludeFiltersState, } from '../types'; import { SavedObjectsRawDoc } from '../../serialization'; import { TransformErrorObjects, TransformSavedObjectDocumentError } from '../../migrations/core'; @@ -91,6 +92,7 @@ describe('migrations v2 model', () => { }, }, knownTypes: ['dashboard', 'config'], + excludeFromUpgradeFilterHooks: {}, }; describe('exponential retry delays for retryable_es_client_error', () => { @@ -839,17 +841,72 @@ describe('migrations v2 model', () => { expect(newState.retryCount).toEqual(1); expect(newState.retryDelay).toEqual(2000); }); - test('SET_SOURCE_WRITE_BLOCK -> CREATE_REINDEX_TEMP if action succeeds with set_write_block_succeeded', () => { + test('SET_SOURCE_WRITE_BLOCK -> CALCULATE_EXCLUDE_FILTERS if action succeeds with set_write_block_succeeded', () => { const res: ResponseType<'SET_SOURCE_WRITE_BLOCK'> = Either.right( 'set_write_block_succeeded' ); const newState = model(setWriteBlockState, res); - expect(newState.controlState).toEqual('CREATE_REINDEX_TEMP'); + expect(newState.controlState).toEqual('CALCULATE_EXCLUDE_FILTERS'); expect(newState.retryCount).toEqual(0); expect(newState.retryDelay).toEqual(0); }); }); + describe('CALCULATE_EXCLUDE_FILTERS', () => { + const state: CalculateExcludeFiltersState = { + ...baseState, + controlState: 'CALCULATE_EXCLUDE_FILTERS', + versionIndexReadyActions: Option.none, + sourceIndex: Option.some('.kibana') as Option.Some, + targetIndex: '.kibana_7.11.0_001', + tempIndexMappings: { properties: {} }, + }; + test('CALCULATE_EXCLUDE_FILTERS -> CALCULATE_EXCLUDE_FILTERS if action fails with retryable error', () => { + const res: ResponseType<'CALCULATE_EXCLUDE_FILTERS'> = Either.left({ + type: 'retryable_es_client_error', + message: 'Something temporarily broke!', + }); + const newState = model(state, res); + expect(newState.controlState).toEqual('CALCULATE_EXCLUDE_FILTERS'); + }); + test('CALCULATE_EXCLUDE_FILTERS -> CREATE_REINDEX_TEMP if action succeeds with filters', () => { + const res: ResponseType<'CALCULATE_EXCLUDE_FILTERS'> = Either.right({ + excludeFilter: { bool: { must: { term: { fieldA: 'abc' } } } }, + errorsByType: { type1: new Error('an error!') }, + }); + const newState = model(state, res); + expect(newState.controlState).toEqual('CREATE_REINDEX_TEMP'); + expect(newState.unusedTypesQuery).toEqual({ + // New filter should be combined unused type query and filter from response + bool: { + filter: [ + { + bool: { + must_not: [ + { + term: { + type: 'unused-fleet-agent-events', + }, + }, + ], + }, + }, + { + bool: { must: { term: { fieldA: 'abc' } } }, + }, + ], + }, + }); + // Logs should be added for any errors encountered from excludeOnUpgrade hooks + expect(newState.logs).toEqual([ + { + level: 'warning', + message: `Ignoring excludeOnUpgrade hook on type [type1] that failed with error: "Error: an error!"`, + }, + ]); + }); + }); + describe('CREATE_REINDEX_TEMP', () => { const state: CreateReindexTempState = { ...baseState, diff --git a/src/core/server/saved_objects/migrationsv2/model/model.ts b/src/core/server/saved_objects/migrationsv2/model/model.ts index a78457fa891f7..474fe56ed64a4 100644 --- a/src/core/server/saved_objects/migrationsv2/model/model.ts +++ b/src/core/server/saved_objects/migrationsv2/model/model.ts @@ -9,6 +9,7 @@ import * as Either from 'fp-ts/lib/Either'; import * as Option from 'fp-ts/lib/Option'; +import { estypes } from '@elastic/elasticsearch'; import { AliasAction, isLeftTypeof } from '../actions'; import { AllActionStates, MigrationLog, State } from '../types'; import type { ResponseType } from '../next'; @@ -362,7 +363,7 @@ export const model = (currentState: State, resW: ResponseType): // If the write block is successfully in place, proceed to the next step. return { ...stateP, - controlState: 'CREATE_REINDEX_TEMP', + controlState: 'CALCULATE_EXCLUDE_FILTERS', }; } else if (isLeftTypeof(res.left, 'index_not_found_exception')) { // We don't handle the following errors as the migration algorithm @@ -372,6 +373,31 @@ export const model = (currentState: State, resW: ResponseType): } else { return throwBadResponse(stateP, res.left); } + } else if (stateP.controlState === 'CALCULATE_EXCLUDE_FILTERS') { + const res = resW as ExcludeRetryableEsError>; + + if (Either.isRight(res)) { + const unusedTypesQuery: estypes.QueryDslQueryContainer = { + bool: { + filter: [stateP.unusedTypesQuery, res.right.excludeFilter], + }, + }; + + return { + ...stateP, + controlState: 'CREATE_REINDEX_TEMP', + unusedTypesQuery, + logs: [ + ...stateP.logs, + ...Object.entries(res.right.errorsByType).map(([soType, error]) => ({ + level: 'warning' as const, + message: `Ignoring excludeOnUpgrade hook on type [${soType}] that failed with error: "${error.toString()}"`, + })), + ], + }; + } else { + return throwBadResponse(stateP, res); + } } else if (stateP.controlState === 'CREATE_REINDEX_TEMP') { const res = resW as ExcludeRetryableEsError>; if (Either.isRight(res)) { diff --git a/src/core/server/saved_objects/migrationsv2/next.ts b/src/core/server/saved_objects/migrationsv2/next.ts index a7475a95c096d..9b091b6fc8509 100644 --- a/src/core/server/saved_objects/migrationsv2/next.ts +++ b/src/core/server/saved_objects/migrationsv2/next.ts @@ -40,6 +40,7 @@ import type { RefreshTarget, OutdatedDocumentsRefresh, CheckUnknownDocumentsState, + CalculateExcludeFiltersState, } from './types'; import * as Actions from './actions'; import { ElasticsearchClient } from '../../elasticsearch'; @@ -71,6 +72,11 @@ export const nextActionMap = (client: ElasticsearchClient, transformRawDocs: Tra }), SET_SOURCE_WRITE_BLOCK: (state: SetSourceWriteBlockState) => Actions.setWriteBlock({ client, index: state.sourceIndex.value }), + CALCULATE_EXCLUDE_FILTERS: (state: CalculateExcludeFiltersState) => + Actions.calculateExcludeFilters({ + client, + excludeFromUpgradeFilterHooks: state.excludeFromUpgradeFilterHooks, + }), CREATE_NEW_TARGET: (state: CreateNewTargetState) => Actions.createIndex({ client, diff --git a/src/core/server/saved_objects/migrationsv2/types.ts b/src/core/server/saved_objects/migrationsv2/types.ts index 576e3a4412184..ea03b64e03dc8 100644 --- a/src/core/server/saved_objects/migrationsv2/types.ts +++ b/src/core/server/saved_objects/migrationsv2/types.ts @@ -18,6 +18,7 @@ import { DocumentsTransformFailed, DocumentsTransformSuccess, } from '../migrations/core/migrate_raw_docs'; +import { SavedObjectTypeExcludeFromUpgradeFilterHook } from '../types'; export type MigrationLogLevel = 'error' | 'info' | 'warning'; @@ -118,6 +119,13 @@ export interface BaseState extends ControlState { * The list of known SO types that are registered. */ readonly knownTypes: string[]; + /** + * All exclude filter hooks registered for types on this index. Keyed by type name. + */ + readonly excludeFromUpgradeFilterHooks: Record< + string, + SavedObjectTypeExcludeFromUpgradeFilterHook + >; } export interface InitState extends BaseState { @@ -172,6 +180,11 @@ export interface SetSourceWriteBlockState extends PostInitState { readonly sourceIndex: Option.Some; } +export interface CalculateExcludeFiltersState extends PostInitState { + readonly controlState: 'CALCULATE_EXCLUDE_FILTERS'; + readonly sourceIndex: Option.Some; +} + export interface CreateNewTargetState extends PostInitState { /** Blank ES cluster, create a new version-specific target index */ readonly controlState: 'CREATE_NEW_TARGET'; @@ -401,6 +414,7 @@ export type State = Readonly< | WaitForYellowSourceState | CheckUnknownDocumentsState | SetSourceWriteBlockState + | CalculateExcludeFiltersState | CreateNewTargetState | CreateReindexTempState | ReindexSourceToTempOpenPit diff --git a/src/core/server/saved_objects/types.ts b/src/core/server/saved_objects/types.ts index 1bb214de701e2..2dbc402e46269 100644 --- a/src/core/server/saved_objects/types.ts +++ b/src/core/server/saved_objects/types.ts @@ -29,6 +29,7 @@ export type { } from './import/types'; import { SavedObject } from '../../types'; +import { ElasticsearchClient } from '../elasticsearch'; type KueryNode = any; @@ -277,6 +278,11 @@ export interface SavedObjectsType { * If defined, will be used to convert the type to an alias. */ convertToAliasScript?: string; + /** + * If defined, allows a type to exclude unneeded documents from the migration process and effectively be deleted. + * See {@link SavedObjectTypeExcludeFromUpgradeFilterHook} for more details. + */ + excludeOnUpgrade?: SavedObjectTypeExcludeFromUpgradeFilterHook; /** * The {@link SavedObjectsTypeMappingDefinition | mapping definition} for the type. */ @@ -487,3 +493,18 @@ export interface SavedObjectsTypeManagementDefinition { export type SavedObjectsExportablePredicate = ( obj: SavedObject ) => boolean; + +/** + * If defined, allows a type to run a search query and return a query filter that may match any documents which may + * be excluded from the next migration upgrade process. Useful for cleaning up large numbers of old documents which + * are no longer needed and may slow the migration process. + * + * If this hook fails, the migration will proceed without these documents having been filtered out, so this + * should not be used as a guarantee that these documents have been deleted. + * + * @public + * @alpha Experimental and subject to change + */ +export type SavedObjectTypeExcludeFromUpgradeFilterHook = (toolkit: { + readonlyEsClient: Pick; +}) => estypes.QueryDslQueryContainer | Promise; diff --git a/src/core/server/server.api.md b/src/core/server/server.api.md index 8b6574ccc8e06..6bc1bb5880e0c 100644 --- a/src/core/server/server.api.md +++ b/src/core/server/server.api.md @@ -3080,6 +3080,7 @@ export interface SavedObjectStatusMeta { export interface SavedObjectsType { convertToAliasScript?: string; convertToMultiNamespaceTypeVersion?: string; + excludeOnUpgrade?: SavedObjectTypeExcludeFromUpgradeFilterHook; hidden: boolean; indexPattern?: string; management?: SavedObjectsTypeManagementDefinition; @@ -3166,6 +3167,13 @@ export class SavedObjectsUtils { static namespaceStringToId: (namespace: string) => string | undefined; } +// Warning: (ae-extra-release-tag) The doc comment should not contain more than one release tag +// +// @public +export type SavedObjectTypeExcludeFromUpgradeFilterHook = (toolkit: { + readonlyEsClient: Pick; +}) => estypes.QueryDslQueryContainer | Promise; + // @public export class SavedObjectTypeRegistry { getAllTypes(): SavedObjectsType[]; diff --git a/x-pack/plugins/actions/server/plugin.ts b/x-pack/plugins/actions/server/plugin.ts index 2f4b1325e3df3..9e22816056618 100644 --- a/x-pack/plugins/actions/server/plugin.ts +++ b/x-pack/plugins/actions/server/plugin.ts @@ -225,7 +225,12 @@ export class ActionsPlugin implements Plugin { + const oldestIdleActionTask = await getOldestIdleActionTask( + readonlyEsClient, + taskManagerIndex + ); + return { + bool: { + must: [ + { term: { type: 'action_task_params' } }, + { range: { updated_at: { lt: oldestIdleActionTask } } }, + ], + }, + }; + }, }); encryptedSavedObjects.registerType({ type: ACTION_TASK_PARAMS_SAVED_OBJECT_TYPE, diff --git a/x-pack/plugins/task_manager/server/index.ts b/x-pack/plugins/task_manager/server/index.ts index 0a0630d82f32d..cc4217f41c5ef 100644 --- a/x-pack/plugins/task_manager/server/index.ts +++ b/x-pack/plugins/task_manager/server/index.ts @@ -28,6 +28,7 @@ export { isEphemeralTaskRejectedDueToCapacityError, } from './task_running'; export { RunNowResult } from './task_scheduling'; +export { getOldestIdleActionTask } from './queries/oldest_idle_action_task'; export { TaskManagerPlugin as TaskManager, diff --git a/x-pack/plugins/task_manager/server/queries/oldest_idle_action_task.test.ts b/x-pack/plugins/task_manager/server/queries/oldest_idle_action_task.test.ts new file mode 100644 index 0000000000000..928d993b03335 --- /dev/null +++ b/x-pack/plugins/task_manager/server/queries/oldest_idle_action_task.test.ts @@ -0,0 +1,71 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { elasticsearchServiceMock } from '../../../../../src/core/server/mocks'; +import { getOldestIdleActionTask } from './oldest_idle_action_task'; + +describe('getOldestIdleActionTask', () => { + it('calls client.search with provided index name', async () => { + const client = elasticsearchServiceMock.createElasticsearchClient(); + await getOldestIdleActionTask(client, '.index-name'); + expect(client.search).toHaveBeenCalled(); + expect(client.search.mock.calls[0][0]?.index).toEqual('.index-name'); + }); + + it('returns a default of now-24h when no results', async () => { + const client = elasticsearchServiceMock.createElasticsearchClient( + elasticsearchServiceMock.createSuccessTransportRequestPromise({ + body: { hits: { hits: [], total: 0 } }, + }) + ); + + const ts = await getOldestIdleActionTask(client, '.index-name'); + expect(ts).toEqual('now-24h'); + }); + + it('returns a default of Date.now-24h when a 404 is returned', async () => { + const client = elasticsearchServiceMock.createElasticsearchClient( + elasticsearchServiceMock.createSuccessTransportRequestPromise({ + error: { status: 404 }, + }) + ); + + const ts = await getOldestIdleActionTask(client, '.index-name'); + expect(ts).toEqual('now-24h'); + }); + + it("returns the search result's task.runAt-24h field if it exists", async () => { + const client = elasticsearchServiceMock.createElasticsearchClient( + elasticsearchServiceMock.createSuccessTransportRequestPromise({ + hits: { hits: [{ _source: { task: { runAt: '2015-01-01T12:10:30Z' } } }], total: 1 }, + }) + ); + + const ts = await getOldestIdleActionTask(client, '.index-name'); + expect(ts).toEqual('2015-01-01T12:10:30Z||-24h'); + }); + + it("fallsback to 0 if the search result's task.runAt field does not exist", async () => { + const client1 = elasticsearchServiceMock.createElasticsearchClient( + elasticsearchServiceMock.createSuccessTransportRequestPromise({ + hits: { hits: [{ _source: { task: { runAt: undefined } } }], total: 1 }, + }) + ); + + const ts1 = await getOldestIdleActionTask(client1, '.index-name'); + expect(ts1).toEqual('0'); + + const client2 = elasticsearchServiceMock.createElasticsearchClient( + elasticsearchServiceMock.createSuccessTransportRequestPromise({ + hits: { hits: [{ _source: { task: undefined } }], total: 1 }, + }) + ); + + const ts2 = await getOldestIdleActionTask(client2, '.index-name'); + expect(ts2).toEqual('0'); + }); +}); diff --git a/x-pack/plugins/task_manager/server/queries/oldest_idle_action_task.ts b/x-pack/plugins/task_manager/server/queries/oldest_idle_action_task.ts new file mode 100644 index 0000000000000..5a8d8e2c81dc9 --- /dev/null +++ b/x-pack/plugins/task_manager/server/queries/oldest_idle_action_task.ts @@ -0,0 +1,79 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import type { ElasticsearchClient } from 'kibana/server'; + +/** + * Returns the millisecond timestamp of the oldest action task that may still be executed (with a 24 hour delay). + * Useful for cleaning up related objects that may no longer be needed. + * @internal + */ +export const getOldestIdleActionTask = async ( + client: Pick, + taskManagerIndex: string +): Promise => { + // Default is now - 24h + const oneDayAgo = `now-24h`; + + const response = await client.search<{ task: { runAt: string } }>( + { + size: 1, + index: taskManagerIndex, + body: { + sort: [{ 'task.runAt': { order: 'asc' } }], + query: { + bool: { + filter: { + bool: { + must: [ + { + terms: { + 'task.taskType': [ + 'actions:.email', + 'actions:.index', + 'actions:.pagerduty', + 'actions:.swimlane', + 'actions:.server-log', + 'actions:.slack', + 'actions:.webhook', + 'actions:.servicenow', + 'actions:.servicenow-sir', + 'actions:.jira', + 'actions:.resilient', + 'actions:.teams', + ], + }, + }, + { + term: { type: 'task' }, + }, + { + term: { 'task.status': 'idle' }, + }, + ], + }, + }, + }, + }, + }, + }, + { ignore: [404] } + ); + + if ((response.body as { error?: { status: number } }).error?.status === 404) { + // If the index doesn't exist, fallback to default + return oneDayAgo; + } else if (response.body?.hits?.hits?.length > 0) { + // If there is a search result, return it's task.runAt field + // If there is a search result but it has no task.runAt, assume something has gone very wrong and return 0 as a safe value + // 0 should be safest since no docs should get filtered out + const runAt = response.body.hits.hits[0]._source?.task?.runAt; + return runAt ? `${runAt}||-24h` : `0`; + } else { + // If no results, fallback to default + return oneDayAgo; + } +}; diff --git a/x-pack/plugins/task_manager/server/saved_objects/index.ts b/x-pack/plugins/task_manager/server/saved_objects/index.ts index c032e8d658a40..d2d079c7747b1 100644 --- a/x-pack/plugins/task_manager/server/saved_objects/index.ts +++ b/x-pack/plugins/task_manager/server/saved_objects/index.ts @@ -6,9 +6,11 @@ */ import type { SavedObjectsServiceSetup, SavedObjectsTypeMappingDefinition } from 'kibana/server'; +import { estypes } from '@elastic/elasticsearch'; import mappings from './mappings.json'; import { migrations } from './migrations'; import { TaskManagerConfig } from '../config.js'; +import { getOldestIdleActionTask } from '../queries/oldest_idle_action_task'; export function setupSavedObjects( savedObjects: SavedObjectsServiceSetup, @@ -22,5 +24,51 @@ export function setupSavedObjects( mappings: mappings.task as SavedObjectsTypeMappingDefinition, migrations, indexPattern: config.index, + excludeOnUpgrade: async ({ readonlyEsClient }) => { + const oldestNeededActionParams = await getOldestIdleActionTask( + readonlyEsClient, + config.index + ); + + // Delete all action tasks that have failed and are no longer needed + return { + bool: { + must: [ + { + terms: { + 'task.taskType': [ + 'actions:.email', + 'actions:.index', + 'actions:.pagerduty', + 'actions:.swimlane', + 'actions:.server-log', + 'actions:.slack', + 'actions:.webhook', + 'actions:.servicenow', + 'actions:.servicenow-sir', + 'actions:.jira', + 'actions:.resilient', + 'actions:.teams', + ], + }, + }, + { + term: { type: 'task' }, + }, + { + term: { 'task.status': 'failed' }, + }, + { + range: { + 'task.runAt': { + // Only apply to tasks that were run before the oldest needed action + lt: oldestNeededActionParams, + }, + }, + }, + ], + }, + } as estypes.QueryDslQueryContainer; + }, }); }