From 74006997626da51847f504684b7fb9418b1e47b1 Mon Sep 17 00:00:00 2001 From: Rudolf Meijering Date: Mon, 12 Apr 2021 18:19:42 +0200 Subject: [PATCH] Migrations v2 ignore fleet agent events (#96690) * migrationsv2: ignore fleet agent events and tsvb telemetry * migrationsv1: ignore tsvb-validation-telemetry * Skip fleet test that depends on fleet-agent-events * Fix typescript errors Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> # Conflicts: # src/core/server/saved_objects/migrationsv2/integration_tests/actions.test.ts --- .../migrations/core/elastic_index.test.ts | 15 +++- .../migrations/core/elastic_index.ts | 23 ++--- .../migrations/kibana/kibana_migrator.test.ts | 6 +- .../migrationsv2/actions/index.test.ts | 3 +- .../migrationsv2/actions/index.ts | 17 +++- .../integration_tests/actions.test.ts | 84 ++++++++++++++----- .../migrations_state_action_machine.test.ts | 28 +++++++ .../saved_objects/migrationsv2/model.test.ts | 8 ++ .../saved_objects/migrationsv2/model.ts | 6 ++ .../server/saved_objects/migrationsv2/next.ts | 12 ++- .../saved_objects/migrationsv2/types.ts | 5 ++ .../apis/agents_setup.ts | 2 +- 12 files changed, 168 insertions(+), 41 deletions(-) diff --git a/src/core/server/saved_objects/migrations/core/elastic_index.test.ts b/src/core/server/saved_objects/migrations/core/elastic_index.test.ts index bfa686ac0cc47..36d3984d9cf25 100644 --- a/src/core/server/saved_objects/migrations/core/elastic_index.test.ts +++ b/src/core/server/saved_objects/migrations/core/elastic_index.test.ts @@ -436,11 +436,18 @@ describe('ElasticIndex', () => { size: 100, query: { bool: { - must_not: { - term: { - type: 'fleet-agent-events', + must_not: [ + { + term: { + type: 'fleet-agent-events', + }, }, - }, + { + term: { + type: 'tsvb-validation-telemetry', + }, + }, + ], }, }, }, diff --git a/src/core/server/saved_objects/migrations/core/elastic_index.ts b/src/core/server/saved_objects/migrations/core/elastic_index.ts index e42643565eb4f..12a2d7bbb54a3 100644 --- a/src/core/server/saved_objects/migrations/core/elastic_index.ts +++ b/src/core/server/saved_objects/migrations/core/elastic_index.ts @@ -68,16 +68,19 @@ export function reader( let scrollId: string | undefined; // When migrating from the outdated index we use a read query which excludes - // saved objects which are no longer used. These saved objects will still be - // kept in the outdated index for backup purposes, but won't be availble in - // the upgraded index. - const excludeUnusedTypes = { + // saved object types which are no longer used. These saved objects will + // still be kept in the outdated index for backup purposes, but won't be + // availble in the upgraded index. + const EXCLUDE_UNUSED_TYPES = [ + 'fleet-agent-events', // https://github.com/elastic/kibana/issues/91869 + 'tsvb-validation-telemetry', // https://github.com/elastic/kibana/issues/95617 + ]; + + const excludeUnusedTypesQuery = { bool: { - must_not: { - term: { - type: 'fleet-agent-events', // https://github.com/elastic/kibana/issues/91869 - }, - }, + must_not: EXCLUDE_UNUSED_TYPES.map((type) => ({ + term: { type }, + })), }, }; @@ -90,7 +93,7 @@ export function reader( : client.search>({ body: { size: batchSize, - query: excludeUnusedTypes, + query: excludeUnusedTypesQuery, }, index, scroll, diff --git a/src/core/server/saved_objects/migrations/kibana/kibana_migrator.test.ts b/src/core/server/saved_objects/migrations/kibana/kibana_migrator.test.ts index 6c4bba906fd4f..47ad9041d7b4c 100644 --- a/src/core/server/saved_objects/migrations/kibana/kibana_migrator.test.ts +++ b/src/core/server/saved_objects/migrations/kibana/kibana_migrator.test.ts @@ -321,7 +321,7 @@ describe('KibanaMigrator', () => { options.client.tasks.get.mockReturnValue( elasticsearchClientMock.createSuccessTransportRequestPromise({ completed: true, - error: { type: 'elatsicsearch_exception', reason: 'task failed with an error' }, + error: { type: 'elasticsearch_exception', reason: 'task failed with an error' }, failures: [], task: { description: 'task description' }, }) @@ -331,11 +331,11 @@ describe('KibanaMigrator', () => { migrator.prepareMigrations(); await expect(migrator.runMigrations()).rejects.toMatchInlineSnapshot(` [Error: Unable to complete saved object migrations for the [.my-index] index. Error: Reindex failed with the following error: - {"_tag":"Some","value":{"type":"elatsicsearch_exception","reason":"task failed with an error"}}] + {"_tag":"Some","value":{"type":"elasticsearch_exception","reason":"task failed with an error"}}] `); expect(loggingSystemMock.collect(options.logger).error[0][0]).toMatchInlineSnapshot(` [Error: Reindex failed with the following error: - {"_tag":"Some","value":{"type":"elatsicsearch_exception","reason":"task failed with an error"}}] + {"_tag":"Some","value":{"type":"elasticsearch_exception","reason":"task failed with an error"}}] `); }); }); diff --git a/src/core/server/saved_objects/migrationsv2/actions/index.test.ts b/src/core/server/saved_objects/migrationsv2/actions/index.test.ts index a21078cbe1135..9a3a663d50152 100644 --- a/src/core/server/saved_objects/migrationsv2/actions/index.test.ts +++ b/src/core/server/saved_objects/migrationsv2/actions/index.test.ts @@ -85,7 +85,8 @@ describe('actions', () => { 'my_source_index', 'my_target_index', Option.none, - false + false, + Option.none ); try { await task(); diff --git a/src/core/server/saved_objects/migrationsv2/actions/index.ts b/src/core/server/saved_objects/migrationsv2/actions/index.ts index acfd8f514b875..066ef5ec159c0 100644 --- a/src/core/server/saved_objects/migrationsv2/actions/index.ts +++ b/src/core/server/saved_objects/migrationsv2/actions/index.ts @@ -13,6 +13,7 @@ import { ElasticsearchClientError, ResponseError } from '@elastic/elasticsearch/ import { pipe } from 'fp-ts/lib/pipeable'; import { errors as EsErrors } from '@elastic/elasticsearch'; import { flow } from 'fp-ts/lib/function'; +import { QueryContainer } from '@elastic/eui/src/components/search_bar/query/ast_to_es_query_dsl'; import { ElasticsearchClient } from '../../../elasticsearch'; import { IndexMapping } from '../../mappings'; import { SavedObjectsRawDoc } from '../../serialization'; @@ -436,7 +437,12 @@ export const reindex = ( sourceIndex: string, targetIndex: string, reindexScript: Option.Option, - requireAlias: boolean + requireAlias: boolean, + /* When reindexing we use a source query to exclude saved objects types which + * are no longer used. These saved objects will still be kept in the outdated + * index for backup purposes, but won't be availble in the upgraded index. + */ + unusedTypesToExclude: Option.Option ): TaskEither.TaskEither => () => { return client .reindex({ @@ -451,6 +457,15 @@ export const reindex = ( index: sourceIndex, // Set reindex batch size size: BATCH_SIZE, + // Exclude saved object types + query: Option.fold( + () => undefined, + (types) => ({ + bool: { + must_not: types.map((type) => ({ term: { type } })), + }, + }) + )(unusedTypesToExclude), }, dest: { index: targetIndex, diff --git a/src/core/server/saved_objects/migrationsv2/integration_tests/actions.test.ts b/src/core/server/saved_objects/migrationsv2/integration_tests/actions.test.ts index 8250a1200323e..e8e6407b89a5e 100644 --- a/src/core/server/saved_objects/migrationsv2/integration_tests/actions.test.ts +++ b/src/core/server/saved_objects/migrationsv2/integration_tests/actions.test.ts @@ -66,7 +66,8 @@ describe('migration actions', () => { { _source: { title: 'doc 1' } }, { _source: { title: 'doc 2' } }, { _source: { title: 'doc 3' } }, - { _source: { title: 'saved object 4' } }, + { _source: { title: 'saved object 4', type: 'another_unused_type' } }, + { _source: { title: 'f-agent-event 5', type: 'f_agent_event' } }, ] as unknown) as SavedObjectsRawDoc[]; await bulkOverwriteTransformedDocuments(client, 'existing_index_with_docs', sourceDocs)(); @@ -344,7 +345,8 @@ describe('migration actions', () => { 'existing_index_with_docs', 'reindex_target', Option.none, - false + false, + Option.none )()) as Either.Right; const task = waitForReindexTask(client, res.right.taskId, '10s'); await expect(task()).resolves.toMatchInlineSnapshot(` @@ -365,6 +367,37 @@ describe('migration actions', () => { "doc 2", "doc 3", "saved object 4", + "f-agent-event 5", + ] + `); + }); + it('resolves right and excludes all unusedTypesToExclude documents', async () => { + const res = (await reindex( + client, + 'existing_index_with_docs', + 'reindex_target_excluded_docs', + Option.none, + false, + Option.some(['f_agent_event', 'another_unused_type']) + )()) as Either.Right; + const task = waitForReindexTask(client, res.right.taskId, '10s'); + await expect(task()).resolves.toMatchInlineSnapshot(` + Object { + "_tag": "Right", + "right": "reindex_succeeded", + } + `); + + const results = ((await searchForOutdatedDocuments(client, { + batchSize: 1000, + targetIndex: 'reindex_target_excluded_docs', + outdatedDocumentsQuery: undefined, + })()) as Either.Right).right.outdatedDocuments; + expect(results.map((doc) => doc._source.title)).toMatchInlineSnapshot(` + Array [ + "doc 1", + "doc 2", + "doc 3", ] `); }); @@ -375,7 +408,8 @@ describe('migration actions', () => { 'existing_index_with_docs', 'reindex_target_2', Option.some(`ctx._source.title = ctx._source.title + '_updated'`), - false + false, + Option.none )()) as Either.Right; const task = waitForReindexTask(client, res.right.taskId, '10s'); await expect(task()).resolves.toMatchInlineSnapshot(` @@ -395,6 +429,7 @@ describe('migration actions', () => { "doc 2_updated", "doc 3_updated", "saved object 4_updated", + "f-agent-event 5_updated", ] `); }); @@ -406,7 +441,8 @@ describe('migration actions', () => { 'existing_index_with_docs', 'reindex_target_3', Option.some(`ctx._source.title = ctx._source.title + '_updated'`), - false + false, + Option.none )()) as Either.Right; let task = waitForReindexTask(client, res.right.taskId, '10s'); await expect(task()).resolves.toMatchInlineSnapshot(` @@ -422,7 +458,8 @@ describe('migration actions', () => { 'existing_index_with_docs', 'reindex_target_3', Option.none, - false + false, + Option.none )()) as Either.Right; task = waitForReindexTask(client, res.right.taskId, '10s'); await expect(task()).resolves.toMatchInlineSnapshot(` @@ -444,6 +481,7 @@ describe('migration actions', () => { "doc 2_updated", "doc 3_updated", "saved object 4_updated", + "f-agent-event 5_updated", ] `); }); @@ -470,7 +508,8 @@ describe('migration actions', () => { 'existing_index_with_docs', 'reindex_target_4', Option.some(`ctx._source.title = ctx._source.title + '_updated'`), - false + false, + Option.none )()) as Either.Right; const task = waitForReindexTask(client, res.right.taskId, '10s'); await expect(task()).resolves.toMatchInlineSnapshot(` @@ -492,6 +531,7 @@ describe('migration actions', () => { "doc 2", "doc 3_updated", "saved object 4_updated", + "f-agent-event 5_updated", ] `); }); @@ -518,7 +558,8 @@ describe('migration actions', () => { 'existing_index_with_docs', 'reindex_target_5', Option.none, - false + false, + Option.none )()) as Either.Right; const task = waitForReindexTask(client, reindexTaskId, '10s'); @@ -552,7 +593,8 @@ describe('migration actions', () => { 'existing_index_with_docs', 'reindex_target_6', Option.none, - false + false, + Option.none )()) as Either.Right; const task = waitForReindexTask(client, reindexTaskId, '10s'); @@ -572,7 +614,8 @@ describe('migration actions', () => { 'no_such_index', 'reindex_target', Option.none, - false + false, + Option.none )()) as Either.Right; const task = waitForReindexTask(client, res.right.taskId, '10s'); await expect(task()).resolves.toMatchInlineSnapshot(` @@ -592,7 +635,8 @@ describe('migration actions', () => { 'existing_index_with_docs', 'existing_index_with_write_block', Option.none, - false + false, + Option.none )()) as Either.Right; const task = waitForReindexTask(client, res.right.taskId, '10s'); @@ -613,7 +657,8 @@ describe('migration actions', () => { 'existing_index_with_docs', 'existing_index_with_write_block', Option.none, - true + true, + Option.none )()) as Either.Right; const task = waitForReindexTask(client, res.right.taskId, '10s'); @@ -634,7 +679,8 @@ describe('migration actions', () => { 'existing_index_with_docs', 'reindex_target', Option.none, - false + false, + Option.none )()) as Either.Right; const task = waitForReindexTask(client, res.right.taskId, '0s'); @@ -660,7 +706,8 @@ describe('migration actions', () => { 'existing_index_with_docs', 'reindex_target_7', Option.none, - false + false, + Option.none )()) as Either.Right; await waitForReindexTask(client, res.right.taskId, '10s')(); @@ -710,12 +757,11 @@ describe('migration actions', () => { )()) as Either.Right).right.outdatedDocuments; expect(resultsWithQuery.length).toBe(3); - const resultsWithoutQuery = ((await searchForOutdatedDocuments( - client, - 'existing_index_with_docs', - undefined as any - )()) as Either.Right).right.outdatedDocuments; - expect(resultsWithoutQuery.length).toBe(4); + const resultsWithoutQuery = ((await searchForOutdatedDocuments(client, { + targetIndex: 'existing_index_with_docs', + outdatedDocumentsQuery: undefined, + })()) as Either.Right).right.outdatedDocuments; + expect(resultsWithoutQuery.length).toBe(5); }); it('resolves with _id, _source, _seq_no and _primary_term', async () => { expect.assertions(1); diff --git a/src/core/server/saved_objects/migrationsv2/migrations_state_action_machine.test.ts b/src/core/server/saved_objects/migrationsv2/migrations_state_action_machine.test.ts index 99c06c0a3586b..cc9a05b9c1c2e 100644 --- a/src/core/server/saved_objects/migrationsv2/migrations_state_action_machine.test.ts +++ b/src/core/server/saved_objects/migrationsv2/migrations_state_action_machine.test.ts @@ -248,6 +248,13 @@ describe('migrationsStateActionMachine', () => { }, }, }, + "unusedTypesToExclude": Object { + "_tag": "Some", + "value": Array [ + "fleet-agent-events", + "tsvb-validation-telemetry", + ], + }, "versionAlias": ".my-so-index_7.11.0", "versionIndex": ".my-so-index_7.11.0_001", }, @@ -308,6 +315,13 @@ describe('migrationsStateActionMachine', () => { }, }, }, + "unusedTypesToExclude": Object { + "_tag": "Some", + "value": Array [ + "fleet-agent-events", + "tsvb-validation-telemetry", + ], + }, "versionAlias": ".my-so-index_7.11.0", "versionIndex": ".my-so-index_7.11.0_001", }, @@ -453,6 +467,13 @@ describe('migrationsStateActionMachine', () => { }, }, }, + "unusedTypesToExclude": Object { + "_tag": "Some", + "value": Array [ + "fleet-agent-events", + "tsvb-validation-telemetry", + ], + }, "versionAlias": ".my-so-index_7.11.0", "versionIndex": ".my-so-index_7.11.0_001", }, @@ -508,6 +529,13 @@ describe('migrationsStateActionMachine', () => { }, }, }, + "unusedTypesToExclude": Object { + "_tag": "Some", + "value": Array [ + "fleet-agent-events", + "tsvb-validation-telemetry", + ], + }, "versionAlias": ".my-so-index_7.11.0", "versionIndex": ".my-so-index_7.11.0_001", }, diff --git a/src/core/server/saved_objects/migrationsv2/model.test.ts b/src/core/server/saved_objects/migrationsv2/model.test.ts index 2813f01093e95..dec742fccc619 100644 --- a/src/core/server/saved_objects/migrationsv2/model.test.ts +++ b/src/core/server/saved_objects/migrationsv2/model.test.ts @@ -68,6 +68,7 @@ describe('migrations v2 model', () => { versionAlias: '.kibana_7.11.0', versionIndex: '.kibana_7.11.0_001', tempIndex: '.kibana_7.11.0_reindex_temp', + unusedTypesToExclude: Option.some(['unused-fleet-agent-events']), }; describe('exponential retry delays for retryable_es_client_error', () => { @@ -1239,6 +1240,13 @@ describe('migrations v2 model', () => { }, }, }, + "unusedTypesToExclude": Object { + "_tag": "Some", + "value": Array [ + "fleet-agent-events", + "tsvb-validation-telemetry", + ], + }, "versionAlias": ".kibana_task_manager_8.1.0", "versionIndex": ".kibana_task_manager_8.1.0_001", } diff --git a/src/core/server/saved_objects/migrationsv2/model.ts b/src/core/server/saved_objects/migrationsv2/model.ts index 4329124d55c03..c2cb55c691b9b 100644 --- a/src/core/server/saved_objects/migrationsv2/model.ts +++ b/src/core/server/saved_objects/migrationsv2/model.ts @@ -769,6 +769,11 @@ export const createInitialState = ({ }, }; + const unusedTypesToExclude = Option.some([ + 'fleet-agent-events', // https://github.com/elastic/kibana/issues/91869 + 'tsvb-validation-telemetry', // https://github.com/elastic/kibana/issues/95617 + ]); + const initialState: InitState = { controlState: 'INIT', indexPrefix, @@ -786,6 +791,7 @@ export const createInitialState = ({ retryDelay: 0, retryAttempts: migrationsConfig.retryAttempts, logs: [], + unusedTypesToExclude, }; return initialState; }; diff --git a/src/core/server/saved_objects/migrationsv2/next.ts b/src/core/server/saved_objects/migrationsv2/next.ts index 1b594cf3d8b53..2f7a292d2003d 100644 --- a/src/core/server/saved_objects/migrationsv2/next.ts +++ b/src/core/server/saved_objects/migrationsv2/next.ts @@ -61,7 +61,14 @@ export const nextActionMap = (client: ElasticsearchClient, transformRawDocs: Tra CREATE_REINDEX_TEMP: (state: CreateReindexTempState) => Actions.createIndex(client, state.tempIndex, state.tempIndexMappings), REINDEX_SOURCE_TO_TEMP: (state: ReindexSourceToTempState) => - Actions.reindex(client, state.sourceIndex.value, state.tempIndex, Option.none, false), + Actions.reindex( + client, + state.sourceIndex.value, + state.tempIndex, + Option.none, + false, + state.unusedTypesToExclude + ), SET_TEMP_WRITE_BLOCK: (state: SetTempWriteBlock) => Actions.setWriteBlock(client, state.tempIndex), REINDEX_SOURCE_TO_TEMP_WAIT_FOR_TASK: (state: ReindexSourceToTempWaitForTaskState) => @@ -100,7 +107,8 @@ export const nextActionMap = (client: ElasticsearchClient, transformRawDocs: Tra state.legacyIndex, state.sourceIndex.value, state.preMigrationScript, - false + false, + state.unusedTypesToExclude ), LEGACY_REINDEX_WAIT_FOR_TASK: (state: LegacyReindexWaitForTaskState) => Actions.waitForReindexTask(client, state.legacyReindexTaskId, '60s'), diff --git a/src/core/server/saved_objects/migrationsv2/types.ts b/src/core/server/saved_objects/migrationsv2/types.ts index dbdd5774dfa62..f2f0fcf22f8f4 100644 --- a/src/core/server/saved_objects/migrationsv2/types.ts +++ b/src/core/server/saved_objects/migrationsv2/types.ts @@ -74,6 +74,11 @@ export interface BaseState extends ControlState { * prevents lost deletes e.g. `.kibana_7.11.0_reindex`. */ readonly tempIndex: string; + /* When reindexing we use a source query to exclude saved objects types which + * are no longer used. These saved objects will still be kept in the outdated + * index for backup purposes, but won't be availble in the upgraded index. + */ + readonly unusedTypesToExclude: Option.Option; } export type InitState = BaseState & { diff --git a/x-pack/test/fleet_api_integration/apis/agents_setup.ts b/x-pack/test/fleet_api_integration/apis/agents_setup.ts index 20112afdf76a4..9b9421274feb3 100644 --- a/x-pack/test/fleet_api_integration/apis/agents_setup.ts +++ b/x-pack/test/fleet_api_integration/apis/agents_setup.ts @@ -90,7 +90,7 @@ export default function (providerContext: FtrProviderContext) { ); }); - it('should create or update the fleet_enroll user if called multiple times with forceRecreate flag', async () => { + it.skip('should create or update the fleet_enroll user if called multiple times with forceRecreate flag', async () => { await supertest.post(`/api/fleet/agents/setup`).set('kbn-xsrf', 'xxxx').expect(200); const { body: userResponseFirstTime } = await es.security.getUser({