diff --git a/src/core/server/saved_objects/migrationsv2/migrations_state_action_machine.test.ts b/src/core/server/saved_objects/migrationsv2/migrations_state_action_machine.test.ts index 795c5ca197ac83..16f37e790a3005 100644 --- a/src/core/server/saved_objects/migrationsv2/migrations_state_action_machine.test.ts +++ b/src/core/server/saved_objects/migrationsv2/migrations_state_action_machine.test.ts @@ -306,4 +306,103 @@ describe('migrationsStateActionMachine', () => { ] `); }); + it('logs all state transitions and action responses when an action throws', async () => { + try { + await migrationStateActionMachine({ + initialState: { ...initialState, reason: 'the fatal reason' } as State, + logger: mockLogger.get(), + model: transitionModel(['LEGACY_REINDEX', 'LEGACY_DELETE', 'FATAL']), + next: (state) => { + if (state.controlState === 'LEGACY_DELETE') throw new Error('this action throws'); + return () => Promise.resolve('hello'); + }, + }); + } catch (e) { + /** ignore */ + } + // Ignore the first 4 log entries that come from our model + const executionLogLogs = loggingSystemMock.collect(mockLogger).info.slice(4); + expect(executionLogLogs).toMatchInlineSnapshot(` + Array [ + Array [ + "[.my-so-index] INIT RESPONSE", + "hello", + ], + Array [ + "[.my-so-index] INIT -> LEGACY_REINDEX", + Object { + "controlState": "LEGACY_REINDEX", + "currentAlias": ".my-so-index", + "indexPrefix": ".my-so-index", + "kibanaVersion": "7.11.0", + "legacyIndex": ".my-so-index", + "logs": Array [ + Object { + "level": "info", + "message": "Log from LEGACY_REINDEX control state", + }, + ], + "outdatedDocuments": Array [], + "outdatedDocumentsQuery": Object { + "bool": Object { + "should": Array [], + }, + }, + "preMigrationScript": Object { + "_tag": "None", + }, + "reason": "the fatal reason", + "retryCount": 0, + "retryDelay": 0, + "targetMappings": Object { + "properties": Object {}, + }, + "versionAlias": ".my-so-index_7.11.0", + "versionIndex": ".my-so-index_7.11.0_001", + }, + ], + Array [ + "[.my-so-index] LEGACY_REINDEX RESPONSE", + "hello", + ], + Array [ + "[.my-so-index] LEGACY_REINDEX -> LEGACY_DELETE", + Object { + "controlState": "LEGACY_DELETE", + "currentAlias": ".my-so-index", + "indexPrefix": ".my-so-index", + "kibanaVersion": "7.11.0", + "legacyIndex": ".my-so-index", + "logs": Array [ + Object { + "level": "info", + "message": "Log from LEGACY_REINDEX control state", + }, + Object { + "level": "info", + "message": "Log from LEGACY_DELETE control state", + }, + ], + "outdatedDocuments": Array [], + "outdatedDocumentsQuery": Object { + "bool": Object { + "should": Array [], + }, + }, + "preMigrationScript": Object { + "_tag": "None", + }, + "reason": "the fatal reason", + "retryCount": 0, + "retryDelay": 0, + "targetMappings": Object { + "properties": Object {}, + }, + "versionAlias": ".my-so-index_7.11.0", + "versionIndex": ".my-so-index_7.11.0_001", + }, + ], + ] + `); + }); }); diff --git a/src/core/server/saved_objects/migrationsv2/migrations_state_action_machine.ts b/src/core/server/saved_objects/migrationsv2/migrations_state_action_machine.ts index b894f0a6504114..eba275373f32fc 100644 --- a/src/core/server/saved_objects/migrationsv2/migrations_state_action_machine.ts +++ b/src/core/server/saved_objects/migrationsv2/migrations_state_action_machine.ts @@ -154,6 +154,7 @@ export async function migrationStateActionMachine({ throw new Error('Invalid terminating control state'); } } catch (e) { + dumpExecutionLog(logger, executionLog); logger.error(e); throw new Error( `Unable to complete saved object migrations for the [${initialState.indexPrefix}] index. Please check the health of your Elasticsearch cluster` diff --git a/src/core/server/saved_objects/migrationsv2/model.test.ts b/src/core/server/saved_objects/migrationsv2/model.test.ts index 53e5d7f4a98293..c2c627a2399d8e 100644 --- a/src/core/server/saved_objects/migrationsv2/model.test.ts +++ b/src/core/server/saved_objects/migrationsv2/model.test.ts @@ -28,7 +28,7 @@ import { LegacyReindexState, LegacyReindexWaitForTaskState, LegacyDeleteState, - CloneSourceToTargetState, + ReindexSourceToTargetState, UpdateTargetMappingsState, UpdateTargetMappingsWaitForTaskState, OutdatedDocumentsSearch, @@ -36,6 +36,8 @@ import { MarkVersionIndexReady, CreateNewTargetState, BaseState, + CreateReindexTargetState, + ReindexSourceToTargetWaitForTaskState, } from './types'; import { SavedObjectsRawDoc } from '..'; import { AliasAction, RetryableEsClientError } from './actions'; @@ -186,7 +188,7 @@ describe('migrations v2 model', () => { versionAlias: '.kibana_7.11.0', versionIndex: '.kibana_7.11.0_001', }; - test('INIT -> UPDATE_TARGET_MAPPINGS if .kibana is already pointing to the target index', () => { + test('INIT -> OUTDATED_DOCUMENTS_SEARCH if .kibana is already pointing to the target index', () => { const res: ResponseType<'INIT'> = Either.right({ '.kibana_7.11.0_001': { aliases: { @@ -212,26 +214,29 @@ describe('migrations v2 model', () => { }); const newState = model(initState, res); - expect(newState.controlState).toEqual('UPDATE_TARGET_MAPPINGS'); + expect(newState.controlState).toEqual('OUTDATED_DOCUMENTS_SEARCH'); expect(newState.targetMappings).toMatchInlineSnapshot(` - Object { - "_meta": Object { - "migrationMappingPropertyHashes": Object { - "disabled_saved_object_type": "7997cf5a56cc02bdc9c93361bde732b0", - "new_saved_object_type": "4a11183eee21e6fbad864f7a30b39ad0", - }, + Object { + "_meta": Object { + "migrationMappingPropertyHashes": Object { + "new_saved_object_type": "4a11183eee21e6fbad864f7a30b39ad0", }, - "properties": Object { - "new_saved_object_type": Object { - "properties": Object { - "value": Object { - "type": "text", - }, + }, + "properties": Object { + "disabled_saved_object_type": Object { + "dynamic": false, + "properties": Object {}, + }, + "new_saved_object_type": Object { + "properties": Object { + "value": Object { + "type": "text", }, }, }, - } - `); + }, + } + `); expect(newState.retryCount).toEqual(0); expect(newState.retryDelay).toEqual(0); }); @@ -633,40 +638,75 @@ describe('migrations v2 model', () => { expect(newState.retryCount).toEqual(1); expect(newState.retryDelay).toEqual(2000); }); - test('SET_SOURCE_WRITE_BLOCK -> CLONE_SOURCE_TO_TARGET if action fails with index_not_found_exception', () => { + test('SET_SOURCE_WRITE_BLOCK -> FATAL if action fails with index_not_found_exception', () => { const res: ResponseType<'SET_SOURCE_WRITE_BLOCK'> = Either.left({ type: 'index_not_found_exception', }); const newState = model(setWriteBlockState, res); - expect(newState.controlState).toEqual('CLONE_SOURCE_TO_TARGET'); + expect(newState.controlState).toEqual('FATAL'); expect(newState.retryCount).toEqual(0); expect(newState.retryDelay).toEqual(0); }); - test('SET_SOURCE_WRITE_BLOCK -> CLONE_SOURCE_TO_TARGET if action succeeds with set_write_block_succeeded', () => { + test('SET_SOURCE_WRITE_BLOCK -> CREATE_REINDEX_TARGET if action succeeds with set_write_block_succeeded', () => { const res: ResponseType<'SET_SOURCE_WRITE_BLOCK'> = Either.right( 'set_write_block_succeeded' ); const newState = model(setWriteBlockState, res); - expect(newState.controlState).toEqual('CLONE_SOURCE_TO_TARGET'); + expect(newState.controlState).toEqual('CREATE_REINDEX_TARGET'); + expect(newState.retryCount).toEqual(0); + expect(newState.retryDelay).toEqual(0); + }); + }); + describe('CREATE_REINDEX_TARGET', () => { + const createReindexTargetState: CreateReindexTargetState = { + ...baseState, + controlState: 'CREATE_REINDEX_TARGET', + versionIndexReadyActions: Option.none, + sourceIndex: Option.some('.kibana') as Option.Some, + targetIndex: '.kibana_7.11.0_001', + reindexTargetMappings: { properties: {} }, + }; + it('CREATE_REINDEX_TARGET -> REINDEX_SOURCE_TO_TARGET if action succeeds', () => { + const res: ResponseType<'CREATE_REINDEX_TARGET'> = Either.right('create_index_succeeded'); + const newState = model(createReindexTargetState, res); + expect(newState.controlState).toEqual('REINDEX_SOURCE_TO_TARGET'); expect(newState.retryCount).toEqual(0); expect(newState.retryDelay).toEqual(0); }); }); - describe('CLONE_SOURCE_TO_TARGET', () => { - const cloneSourceToTargetState: CloneSourceToTargetState = { + describe('REINDEX_SOURCE_TO_TARGET', () => { + const reindexSourceToTargetState: ReindexSourceToTargetState = { ...baseState, - controlState: 'CLONE_SOURCE_TO_TARGET', + controlState: 'REINDEX_SOURCE_TO_TARGET', versionIndexReadyActions: Option.none, sourceIndex: Option.some('.kibana') as Option.Some, targetIndex: '.kibana_7.11.0_001', }; - test('CLONE_SOURCE_TO_TARGET -> UPDATE_TARGET_MAPPINGS', () => { - const res: ResponseType<'CLONE_SOURCE_TO_TARGET'> = Either.right({ - acknowledged: true, - shardsAcknowledged: true, + test('REINDEX_SOURCE_TO_TARGET -> REINDEX_SOURCE_TO_TARGET_WAIT_FOR_TASK', () => { + const res: ResponseType<'REINDEX_SOURCE_TO_TARGET'> = Either.right({ + taskId: 'reindex-task-id', }); - const newState = model(cloneSourceToTargetState, res); - expect(newState.controlState).toEqual('UPDATE_TARGET_MAPPINGS'); + const newState = model(reindexSourceToTargetState, res); + expect(newState.controlState).toEqual('REINDEX_SOURCE_TO_TARGET_WAIT_FOR_TASK'); + expect(newState.retryCount).toEqual(0); + expect(newState.retryDelay).toEqual(0); + }); + }); + describe('REINDEX_SOURCE_TO_TARGET_WAIT_FOR_TASK', () => { + const reindexSourceToTargetWaitForState: ReindexSourceToTargetWaitForTaskState = { + ...baseState, + controlState: 'REINDEX_SOURCE_TO_TARGET_WAIT_FOR_TASK', + versionIndexReadyActions: Option.none, + sourceIndex: Option.some('.kibana') as Option.Some, + targetIndex: '.kibana_7.11.0_001', + reindexSourceToTargetTaskId: 'reindex-task-id', + }; + test('REINDEX_SOURCE_TO_TARGET_WAIT_FOR_TASK -> OUTDATED_DOCUMENTS_SEARCH when reindex succeeds', () => { + const res: ResponseType<'REINDEX_SOURCE_TO_TARGET_WAIT_FOR_TASK'> = Either.right( + 'reindex_succeeded' + ); + const newState = model(reindexSourceToTargetWaitForState, res); + expect(newState.controlState).toEqual('OUTDATED_DOCUMENTS_SEARCH'); expect(newState.retryCount).toEqual(0); expect(newState.retryDelay).toEqual(0); }); @@ -702,7 +742,24 @@ describe('migrations v2 model', () => { targetIndex: '.kibana_7.11.0_001', updateTargetMappingsTaskId: 'update target mappings task', }; - test('UPDATE_TARGET_MAPPINGS_WAIT_FOR_TASK -> OUTDATED_DOCUMENTS_SEARCH', () => { + test('UPDATE_TARGET_MAPPINGS_WAIT_FOR_TASK -> MARK_VERSION_INDEX_READY if sone versionIndexReadyActions', () => { + const res: ResponseType<'UPDATE_TARGET_MAPPINGS_WAIT_FOR_TASK'> = Either.right( + 'pickup_updated_mappings_succeeded' + ); + const newState = model( + { + ...updateTargetMappingsWaitForTaskState, + versionIndexReadyActions: Option.some([ + { add: { index: 'kibana-index', alias: 'my-alias' } }, + ]), + }, + res + ) as UpdateTargetMappingsWaitForTaskState; + expect(newState.controlState).toEqual('MARK_VERSION_INDEX_READY'); + expect(newState.retryCount).toEqual(0); + expect(newState.retryDelay).toEqual(0); + }); + test('UPDATE_TARGET_MAPPINGS_WAIT_FOR_TASK -> DONE if none versionIndexReadyActions', () => { const res: ResponseType<'UPDATE_TARGET_MAPPINGS_WAIT_FOR_TASK'> = Either.right( 'pickup_updated_mappings_succeeded' ); @@ -710,7 +767,7 @@ describe('migrations v2 model', () => { updateTargetMappingsWaitForTaskState, res ) as UpdateTargetMappingsWaitForTaskState; - expect(newState.controlState).toEqual('OUTDATED_DOCUMENTS_SEARCH'); + expect(newState.controlState).toEqual('DONE'); expect(newState.retryCount).toEqual(0); expect(newState.retryDelay).toEqual(0); }); @@ -736,7 +793,7 @@ describe('migrations v2 model', () => { expect(newState.retryCount).toEqual(0); expect(newState.retryDelay).toEqual(0); }); - test('OUTDATED_DOCUMENTS_SEARCH -> MARK_VERSION_INDEX_READY if none outdated documents were found and some versionIndexReadyActions', () => { + test('OUTDATED_DOCUMENTS_SEARCH -> UPDATE_TARGET_MAPPINGS if none outdated documents were found and some versionIndexReadyActions', () => { const aliasActions = ([Symbol('alias action')] as unknown) as AliasAction[]; const outdatedDocumentsSourchStateWithSomeVersionIndexReadyActions = { ...outdatedDocumentsSourchState, @@ -751,17 +808,17 @@ describe('migrations v2 model', () => { outdatedDocumentsSourchStateWithSomeVersionIndexReadyActions, res ) as MarkVersionIndexReady; - expect(newState.controlState).toEqual('MARK_VERSION_INDEX_READY'); + expect(newState.controlState).toEqual('UPDATE_TARGET_MAPPINGS'); expect(newState.versionIndexReadyActions.value).toEqual(aliasActions); expect(newState.retryCount).toEqual(0); expect(newState.retryDelay).toEqual(0); }); - test('OUTDATED_DOCUMENTS_SEARCH -> DONE if none outdated documents were found and none versionIndexReadyActions', () => { + test('OUTDATED_DOCUMENTS_SEARCH -> UPDATE_TARGET_MAPPINGS if none outdated documents were found and none versionIndexReadyActions', () => { const res: ResponseType<'OUTDATED_DOCUMENTS_SEARCH'> = Either.right({ outdatedDocuments: [], }); const newState = model(outdatedDocumentsSourchState, res); - expect(newState.controlState).toEqual('DONE'); + expect(newState.controlState).toEqual('UPDATE_TARGET_MAPPINGS'); expect(newState.retryCount).toEqual(0); expect(newState.retryDelay).toEqual(0); }); @@ -907,42 +964,30 @@ describe('migrations v2 model', () => { "should": Array [ Object { "bool": Object { - "must": Array [ - Object { - "exists": Object { - "field": "my_dashboard", - }, + "must": Object { + "term": Object { + "type": "my_dashboard", }, - Object { - "bool": Object { - "must_not": Object { - "term": Object { - "migrationVersion.my_dashboard": "7.10.1", - }, - }, - }, + }, + "must_not": Object { + "term": Object { + "migrationVersion.my_dashboard": "7.10.1", }, - ], + }, }, }, Object { "bool": Object { - "must": Array [ - Object { - "exists": Object { - "field": "my_viz", - }, + "must": Object { + "term": Object { + "type": "my_viz", }, - Object { - "bool": Object { - "must_not": Object { - "term": Object { - "migrationVersion.my_viz": "8.0.0", - }, - }, - }, + }, + "must_not": Object { + "term": Object { + "migrationVersion.my_viz": "8.0.0", }, - ], + }, }, }, ], diff --git a/src/core/server/saved_objects/migrationsv2/model.ts b/src/core/server/saved_objects/migrationsv2/model.ts index 0d5fedbc92f021..c4997fe6e71453 100644 --- a/src/core/server/saved_objects/migrationsv2/model.ts +++ b/src/core/server/saved_objects/migrationsv2/model.ts @@ -25,6 +25,7 @@ import { AllActionStates, InitState, State } from './types'; import { IndexMapping } from '../mappings'; import { ResponseType } from './next'; import { SavedObjectsMigrationVersion } from '../types'; +import { disableUnknownTypeMappingFields } from '../migrations/core/migration_context'; /** * How many times to retry a failing step. @@ -186,15 +187,15 @@ export const model = (currentState: State, resW: ResponseType): ) { stateP = { ...stateP, - // Skip to 'UPDATE_TARGET_MAPPINGS' to update the mappings and - // transform any outdated documents for in case a new plugin was - // installed / enabled. - controlState: 'UPDATE_TARGET_MAPPINGS', + // Skip to 'OUTDATED_DOCUMENTS_SEARCH' so that if a new plugin was + // installed / enabled we can transform any old documents and update + // the mappings for this plugin's types. + controlState: 'OUTDATED_DOCUMENTS_SEARCH', // Source is a none because we didn't do any migration from a source // index sourceIndex: Option.none, targetIndex: `${stateP.indexPrefix}_${stateP.kibanaVersion}_001`, - targetMappings: mergeMigrationMappingPropertyHashes( + targetMappings: disableUnknownTypeMappingFields( stateP.targetMappings, indices[aliases[stateP.currentAlias]].mappings ), @@ -418,36 +419,61 @@ export const model = (currentState: State, resW: ResponseType): return stateP; } else if (stateP.controlState === 'SET_SOURCE_WRITE_BLOCK') { const res = resW as ResponseType; + const reindexTargetMappings: IndexMapping = { + // @ts-expect-error we don't allow plugins to set `dynamic` + dynamic: false, + properties: { + type: { type: 'keyword' }, + migrationVersion: { + // @ts-expect-error we don't allow plugins to set `dynamic` + dynamic: 'true', + type: 'object', + }, + }, + }; // If the write block is successfully in place, proceed to the next step. if (Either.isRight(res)) { - stateP = { ...stateP, controlState: 'CLONE_SOURCE_TO_TARGET' }; + stateP = { + ...stateP, + controlState: 'CREATE_REINDEX_TARGET', + reindexTargetMappings, + }; } else if (Either.isLeft(res)) { if (res.left.type === 'index_not_found_exception') { // If the write block failed because the index doesn't exist, it means // another instance already completed the legacy pre-migration. Proceed // to the next step. - stateP = { ...stateP, controlState: 'CLONE_SOURCE_TO_TARGET' }; + stateP = { + ...stateP, + controlState: 'FATAL', + reason: `SET_SOURCE_WRITE_BLOCK failed because the source index [${stateP.sourceIndex.value}] does not exist.`, + }; } } else { return throwBadResponse(res); } return stateP; - } else if (stateP.controlState === 'CLONE_SOURCE_TO_TARGET') { - return { ...stateP, controlState: 'UPDATE_TARGET_MAPPINGS' }; - } else if (stateP.controlState === 'UPDATE_TARGET_MAPPINGS') { + } else if (stateP.controlState === 'CREATE_REINDEX_TARGET') { + return { ...stateP, controlState: 'REINDEX_SOURCE_TO_TARGET' }; + } else if (stateP.controlState === 'REINDEX_SOURCE_TO_TARGET') { const res = resW as ResponseType; if (Either.isRight(res)) { - stateP = { + return { ...stateP, - controlState: 'UPDATE_TARGET_MAPPINGS_WAIT_FOR_TASK', - updateTargetMappingsTaskId: res.right.taskId, + controlState: 'REINDEX_SOURCE_TO_TARGET_WAIT_FOR_TASK', + reindexSourceToTargetTaskId: res.right.taskId, + }; + } else { + return { + ...stateP, + controlState: 'FATAL', + reason: `REINDEX_SOURCE_TO_TARGET: unexpected action response: ${JSON.stringify(res)}`, }; } - return stateP; - } else if (stateP.controlState === 'UPDATE_TARGET_MAPPINGS_WAIT_FOR_TASK') { + } else if (stateP.controlState === 'REINDEX_SOURCE_TO_TARGET_WAIT_FOR_TASK') { const res = resW as ResponseType; - if (Either.isRight(res) && res.right === 'pickup_updated_mappings_succeeded') { + if (Either.isRight(res)) { return { ...stateP, controlState: 'OUTDATED_DOCUMENTS_SEARCH', @@ -456,7 +482,7 @@ export const model = (currentState: State, resW: ResponseType): return { ...stateP, controlState: 'FATAL', - reason: `UPDATE_TARGET_MAPPINGS_WAIT_FOR_TASK: unexpected action response: ${JSON.stringify( + reason: `REINDEX_SOURCE_TO_TARGET_WAIT_FOR_TASK: unexpected action response: ${JSON.stringify( res )}`, }; @@ -474,25 +500,10 @@ export const model = (currentState: State, resW: ResponseType): } else { // If there are no more results we have transformed all outdated // documents and can proceed to the next step - if (Option.isSome(stateP.versionIndexReadyActions)) { - // If there are some versionIndexReadyActions we performed a full - // migration and need to point the aliases to our newly migrated - // index. - return { - ...stateP, - controlState: 'MARK_VERSION_INDEX_READY', - versionIndexReadyActions: stateP.versionIndexReadyActions, - }; - } else { - // If there are none versionIndexReadyActions another instance - // already completed this migration and we only updated the mappings - // and transformed outdated documents for incase a new plugin was - // enabled. - return { - ...stateP, - controlState: 'DONE', - }; - } + return { + ...stateP, + controlState: 'UPDATE_TARGET_MAPPINGS', + }; } } return stateP; @@ -510,6 +521,47 @@ export const model = (currentState: State, resW: ResponseType): reason: `OUTDATED_DOCUMENTS_TRANSFORM: unexpected action response: ${JSON.stringify(res)}`, }; } + } else if (stateP.controlState === 'UPDATE_TARGET_MAPPINGS') { + const res = resW as ResponseType; + if (Either.isRight(res)) { + stateP = { + ...stateP, + controlState: 'UPDATE_TARGET_MAPPINGS_WAIT_FOR_TASK', + updateTargetMappingsTaskId: res.right.taskId, + }; + } + return stateP; + } else if (stateP.controlState === 'UPDATE_TARGET_MAPPINGS_WAIT_FOR_TASK') { + const res = resW as ResponseType; + if (Either.isRight(res) && res.right === 'pickup_updated_mappings_succeeded') { + if (Option.isSome(stateP.versionIndexReadyActions)) { + // If there are some versionIndexReadyActions we performed a full + // migration and need to point the aliases to our newly migrated + // index. + return { + ...stateP, + controlState: 'MARK_VERSION_INDEX_READY', + versionIndexReadyActions: stateP.versionIndexReadyActions, + }; + } else { + // If there are none versionIndexReadyActions another instance + // already completed this migration and we only updated the mappings + // and transformed outdated documents for incase a new plugin was + // enabled. + return { + ...stateP, + controlState: 'DONE', + }; + } + } else { + return { + ...stateP, + controlState: 'FATAL', + reason: `UPDATE_TARGET_MAPPINGS_WAIT_FOR_TASK: unexpected action response: ${JSON.stringify( + res + )}`, + }; + } } else if (stateP.controlState === 'CREATE_NEW_TARGET') { return { ...stateP, @@ -544,14 +596,8 @@ export const createInitialState = ({ bool: { should: Object.entries(migrationVersionPerType).map(([type, latestVersion]) => ({ bool: { - must: [ - { exists: { field: type } }, - { - bool: { - must_not: { term: { [`migrationVersion.${type}`]: latestVersion } }, - }, - }, - ], + must: { term: { type } }, + must_not: { term: { [`migrationVersion.${type}`]: latestVersion } }, }, })), }, diff --git a/src/core/server/saved_objects/migrationsv2/next.ts b/src/core/server/saved_objects/migrationsv2/next.ts index e3ae4bc3df53ff..ed7c9df57f41ca 100644 --- a/src/core/server/saved_objects/migrationsv2/next.ts +++ b/src/core/server/saved_objects/migrationsv2/next.ts @@ -18,11 +18,12 @@ */ import * as TaskEither from 'fp-ts/lib/TaskEither'; +import * as Option from 'fp-ts/lib/Option'; import { UnwrapPromise } from '@kbn/utility-types'; import { pipe } from 'fp-ts/lib/pipeable'; import { AllActionStates, - CloneSourceToTargetState, + ReindexSourceToTargetState, CreateNewTargetState, InitState, LegacyCreateReindexTargetState, @@ -37,6 +38,8 @@ import { State, UpdateTargetMappingsState, UpdateTargetMappingsWaitForTaskState, + CreateReindexTargetState, + ReindexSourceToTargetWaitForTaskState, } from './types'; import * as Actions from './actions'; import { ElasticsearchClient } from '../../elasticsearch'; @@ -63,8 +66,12 @@ export const nextActionMap = (client: ElasticsearchClient, transformRawDocs: Tra Actions.setWriteBlock(client, state.sourceIndex.value), CREATE_NEW_TARGET: (state: CreateNewTargetState) => Actions.createIndex(client, state.targetIndex, state.targetMappings), - CLONE_SOURCE_TO_TARGET: (state: CloneSourceToTargetState) => - Actions.cloneIndex(client, state.sourceIndex.value, state.targetIndex), + CREATE_REINDEX_TARGET: (state: CreateReindexTargetState) => + Actions.createIndex(client, state.targetIndex, state.reindexTargetMappings), + REINDEX_SOURCE_TO_TARGET: (state: ReindexSourceToTargetState) => + Actions.reindex(client, state.sourceIndex.value, state.targetIndex, Option.none), + REINDEX_SOURCE_TO_TARGET_WAIT_FOR_TASK: (state: ReindexSourceToTargetWaitForTaskState) => + Actions.waitForReindexTask(client, state.reindexSourceToTargetTaskId, '60s'), UPDATE_TARGET_MAPPINGS: (state: UpdateTargetMappingsState) => Actions.updateAndPickupMappings(client, state.targetIndex, state.targetMappings), UPDATE_TARGET_MAPPINGS_WAIT_FOR_TASK: (state: UpdateTargetMappingsWaitForTaskState) => diff --git a/src/core/server/saved_objects/migrationsv2/types.ts b/src/core/server/saved_objects/migrationsv2/types.ts index 3fb0452fd842bc..d9043a9d471a2b 100644 --- a/src/core/server/saved_objects/migrationsv2/types.ts +++ b/src/core/server/saved_objects/migrationsv2/types.ts @@ -102,10 +102,37 @@ export type CreateNewTargetState = PostInitState & { readonly versionIndexReadyActions: Option.Some; }; -export type CloneSourceToTargetState = PostInitState & { - /** Create the target index by cloning the source index */ - readonly controlState: 'CLONE_SOURCE_TO_TARGET'; +export type CreateReindexTargetState = PostInitState & { + /** + * Create a target index with mappings from the source index and registered + * plugins + */ + readonly controlState: 'CREATE_REINDEX_TARGET'; + readonly sourceIndex: Option.Some; + /** + * Special mappings set when creating the reindex target. These mappings + * have `dynamic: false` to allow for any kind of outdated document to be + * written to the index, but still define mappings for the + * `migrationVersion` and `type` fields so that we can search for and + * transform outdated documents. + */ + readonly reindexTargetMappings: IndexMapping; +}; + +export type ReindexSourceToTargetState = PostInitState & { + /** Reindex documents from the source index into the target index */ + readonly controlState: 'REINDEX_SOURCE_TO_TARGET'; + readonly sourceIndex: Option.Some; +}; + +export type ReindexSourceToTargetWaitForTaskState = PostInitState & { + /** + * Wait until reindexing documents from the source index into the target + * index has completed + */ + readonly controlState: 'REINDEX_SOURCE_TO_TARGET_WAIT_FOR_TASK'; readonly sourceIndex: Option.Some; + readonly reindexSourceToTargetTaskId: string; }; export type UpdateTargetMappingsState = PostInitState & { @@ -201,7 +228,9 @@ export type State = | DoneState | SetSourceWriteBlockState | CreateNewTargetState - | CloneSourceToTargetState + | CreateReindexTargetState + | ReindexSourceToTargetState + | ReindexSourceToTargetWaitForTaskState | UpdateTargetMappingsState | UpdateTargetMappingsWaitForTaskState | OutdatedDocumentsSearch