Skip to content

Commit

Permalink
Reindex instead of cloning for migrations
Browse files Browse the repository at this point in the history
  • Loading branch information
@rudolf
rudolf committed Dec 10, 2020
1 parent a5c92e2 commit 4f97684
Show file tree
Hide file tree
Showing 6 changed files with 342 additions and 115 deletions.
99 changes: 99 additions & 0 deletions src/core/server/saved_objects/migrationsv2/migrations_state_action_machine.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -306,4 +306,103 @@ describe('migrationsStateActionMachine', () => {
]
`);
});
it('logs all state transitions and action responses when an action throws', async () => {
try {
await migrationStateActionMachine({
initialState: { ...initialState, reason: 'the fatal reason' } as State,
logger: mockLogger.get(),
model: transitionModel(['LEGACY_REINDEX', 'LEGACY_DELETE', 'FATAL']),
next: (state) => {
if (state.controlState === 'LEGACY_DELETE') throw new Error('this action throws');
return () => Promise.resolve('hello');
},
});
} catch (e) {
/** ignore */
}
// Ignore the first 4 log entries that come from our model
const executionLogLogs = loggingSystemMock.collect(mockLogger).info.slice(4);
expect(executionLogLogs).toMatchInlineSnapshot(`
Array [
Array [
"[.my-so-index] INIT RESPONSE",
"hello",
],
Array [
"[.my-so-index] INIT -> LEGACY_REINDEX",
Object {
"controlState": "LEGACY_REINDEX",
"currentAlias": ".my-so-index",
"indexPrefix": ".my-so-index",
"kibanaVersion": "7.11.0",
"legacyIndex": ".my-so-index",
"logs": Array [
Object {
"level": "info",
"message": "Log from LEGACY_REINDEX control state",
},
],
"outdatedDocuments": Array [],
"outdatedDocumentsQuery": Object {
"bool": Object {
"should": Array [],
},
},
"preMigrationScript": Object {
"_tag": "None",
},
"reason": "the fatal reason",
"retryCount": 0,
"retryDelay": 0,
"targetMappings": Object {
"properties": Object {},
},
"versionAlias": ".my-so-index_7.11.0",
"versionIndex": ".my-so-index_7.11.0_001",
},
],
Array [
"[.my-so-index] LEGACY_REINDEX RESPONSE",
"hello",
],
Array [
"[.my-so-index] LEGACY_REINDEX -> LEGACY_DELETE",
Object {
"controlState": "LEGACY_DELETE",
"currentAlias": ".my-so-index",
"indexPrefix": ".my-so-index",
"kibanaVersion": "7.11.0",
"legacyIndex": ".my-so-index",
"logs": Array [
Object {
"level": "info",
"message": "Log from LEGACY_REINDEX control state",
},
Object {
"level": "info",
"message": "Log from LEGACY_DELETE control state",
},
],
"outdatedDocuments": Array [],
"outdatedDocumentsQuery": Object {
"bool": Object {
"should": Array [],
},
},
"preMigrationScript": Object {
"_tag": "None",
},
"reason": "the fatal reason",
"retryCount": 0,
"retryDelay": 0,
"targetMappings": Object {
"properties": Object {},
},
"versionAlias": ".my-so-index_7.11.0",
"versionIndex": ".my-so-index_7.11.0_001",
},
],
]
`);
});
});
1 change: 1 addition & 0 deletions src/core/server/saved_objects/migrationsv2/migrations_state_action_machine.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -154,6 +154,7 @@ export async function migrationStateActionMachine({
throw new Error('Invalid terminating control state');
}
} catch (e) {
dumpExecutionLog(logger, executionLog);
logger.error(e);
throw new Error(
`Unable to complete saved object migrations for the [${initialState.indexPrefix}] index. Please check the health of your Elasticsearch cluster`
Expand Down
173 changes: 109 additions & 64 deletions src/core/server/saved_objects/migrationsv2/model.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,14 +28,16 @@ import {
LegacyReindexState,
LegacyReindexWaitForTaskState,
LegacyDeleteState,
CloneSourceToTargetState,
ReindexSourceToTargetState,
UpdateTargetMappingsState,
UpdateTargetMappingsWaitForTaskState,
OutdatedDocumentsSearch,
OutdatedDocumentsTransform,
MarkVersionIndexReady,
CreateNewTargetState,
BaseState,
CreateReindexTargetState,
ReindexSourceToTargetWaitForTaskState,
} from './types';
import { SavedObjectsRawDoc } from '..';
import { AliasAction, RetryableEsClientError } from './actions';
Expand Down Expand Up @@ -186,7 +188,7 @@ describe('migrations v2 model', () => {
versionAlias: '.kibana_7.11.0',
versionIndex: '.kibana_7.11.0_001',
};
test('INIT -> UPDATE_TARGET_MAPPINGS if .kibana is already pointing to the target index', () => {
test('INIT -> OUTDATED_DOCUMENTS_SEARCH if .kibana is already pointing to the target index', () => {
const res: ResponseType<'INIT'> = Either.right({
'.kibana_7.11.0_001': {
aliases: {
Expand All @@ -212,26 +214,29 @@ describe('migrations v2 model', () => {
});
const newState = model(initState, res);

expect(newState.controlState).toEqual('UPDATE_TARGET_MAPPINGS');
expect(newState.controlState).toEqual('OUTDATED_DOCUMENTS_SEARCH');
expect(newState.targetMappings).toMatchInlineSnapshot(`
Object {
"_meta": Object {
"migrationMappingPropertyHashes": Object {
"disabled_saved_object_type": "7997cf5a56cc02bdc9c93361bde732b0",
"new_saved_object_type": "4a11183eee21e6fbad864f7a30b39ad0",
},
Object {
"_meta": Object {
"migrationMappingPropertyHashes": Object {
"new_saved_object_type": "4a11183eee21e6fbad864f7a30b39ad0",
},
"properties": Object {
"new_saved_object_type": Object {
"properties": Object {
"value": Object {
"type": "text",
},
},
"properties": Object {
"disabled_saved_object_type": Object {
"dynamic": false,
"properties": Object {},
},
"new_saved_object_type": Object {
"properties": Object {
"value": Object {
"type": "text",
},
},
},
}
`);
},
}
`);
expect(newState.retryCount).toEqual(0);
expect(newState.retryDelay).toEqual(0);
});
Expand Down Expand Up @@ -633,40 +638,75 @@ describe('migrations v2 model', () => {
expect(newState.retryCount).toEqual(1);
expect(newState.retryDelay).toEqual(2000);
});
test('SET_SOURCE_WRITE_BLOCK -> CLONE_SOURCE_TO_TARGET if action fails with index_not_found_exception', () => {
test('SET_SOURCE_WRITE_BLOCK -> FATAL if action fails with index_not_found_exception', () => {
const res: ResponseType<'SET_SOURCE_WRITE_BLOCK'> = Either.left({
type: 'index_not_found_exception',
});
const newState = model(setWriteBlockState, res);
expect(newState.controlState).toEqual('CLONE_SOURCE_TO_TARGET');
expect(newState.controlState).toEqual('FATAL');
expect(newState.retryCount).toEqual(0);
expect(newState.retryDelay).toEqual(0);
});
test('SET_SOURCE_WRITE_BLOCK -> CLONE_SOURCE_TO_TARGET if action succeeds with set_write_block_succeeded', () => {
test('SET_SOURCE_WRITE_BLOCK -> CREATE_REINDEX_TARGET if action succeeds with set_write_block_succeeded', () => {
const res: ResponseType<'SET_SOURCE_WRITE_BLOCK'> = Either.right(
'set_write_block_succeeded'
);
const newState = model(setWriteBlockState, res);
expect(newState.controlState).toEqual('CLONE_SOURCE_TO_TARGET');
expect(newState.controlState).toEqual('CREATE_REINDEX_TARGET');
expect(newState.retryCount).toEqual(0);
expect(newState.retryDelay).toEqual(0);
});
});
describe('CREATE_REINDEX_TARGET', () => {
const createReindexTargetState: CreateReindexTargetState = {
...baseState,
controlState: 'CREATE_REINDEX_TARGET',
versionIndexReadyActions: Option.none,
sourceIndex: Option.some('.kibana') as Option.Some<string>,
targetIndex: '.kibana_7.11.0_001',
reindexTargetMappings: { properties: {} },
};
it('CREATE_REINDEX_TARGET -> REINDEX_SOURCE_TO_TARGET if action succeeds', () => {
const res: ResponseType<'CREATE_REINDEX_TARGET'> = Either.right('create_index_succeeded');
const newState = model(createReindexTargetState, res);
expect(newState.controlState).toEqual('REINDEX_SOURCE_TO_TARGET');
expect(newState.retryCount).toEqual(0);
expect(newState.retryDelay).toEqual(0);
});
});
describe('CLONE_SOURCE_TO_TARGET', () => {
const cloneSourceToTargetState: CloneSourceToTargetState = {
describe('REINDEX_SOURCE_TO_TARGET', () => {
const reindexSourceToTargetState: ReindexSourceToTargetState = {
...baseState,
controlState: 'CLONE_SOURCE_TO_TARGET',
controlState: 'REINDEX_SOURCE_TO_TARGET',
versionIndexReadyActions: Option.none,
sourceIndex: Option.some('.kibana') as Option.Some<string>,
targetIndex: '.kibana_7.11.0_001',
};
test('CLONE_SOURCE_TO_TARGET -> UPDATE_TARGET_MAPPINGS', () => {
const res: ResponseType<'CLONE_SOURCE_TO_TARGET'> = Either.right({
acknowledged: true,
shardsAcknowledged: true,
test('REINDEX_SOURCE_TO_TARGET -> REINDEX_SOURCE_TO_TARGET_WAIT_FOR_TASK', () => {
const res: ResponseType<'REINDEX_SOURCE_TO_TARGET'> = Either.right({
taskId: 'reindex-task-id',
});
const newState = model(cloneSourceToTargetState, res);
expect(newState.controlState).toEqual('UPDATE_TARGET_MAPPINGS');
const newState = model(reindexSourceToTargetState, res);
expect(newState.controlState).toEqual('REINDEX_SOURCE_TO_TARGET_WAIT_FOR_TASK');
expect(newState.retryCount).toEqual(0);
expect(newState.retryDelay).toEqual(0);
});
});
describe('REINDEX_SOURCE_TO_TARGET_WAIT_FOR_TASK', () => {
const reindexSourceToTargetWaitForState: ReindexSourceToTargetWaitForTaskState = {
...baseState,
controlState: 'REINDEX_SOURCE_TO_TARGET_WAIT_FOR_TASK',
versionIndexReadyActions: Option.none,
sourceIndex: Option.some('.kibana') as Option.Some<string>,
targetIndex: '.kibana_7.11.0_001',
reindexSourceToTargetTaskId: 'reindex-task-id',
};
test('REINDEX_SOURCE_TO_TARGET_WAIT_FOR_TASK -> OUTDATED_DOCUMENTS_SEARCH when reindex succeeds', () => {
const res: ResponseType<'REINDEX_SOURCE_TO_TARGET_WAIT_FOR_TASK'> = Either.right(
'reindex_succeeded'
);
const newState = model(reindexSourceToTargetWaitForState, res);
expect(newState.controlState).toEqual('OUTDATED_DOCUMENTS_SEARCH');
expect(newState.retryCount).toEqual(0);
expect(newState.retryDelay).toEqual(0);
});
Expand Down Expand Up @@ -702,15 +742,32 @@ describe('migrations v2 model', () => {
targetIndex: '.kibana_7.11.0_001',
updateTargetMappingsTaskId: 'update target mappings task',
};
test('UPDATE_TARGET_MAPPINGS_WAIT_FOR_TASK -> OUTDATED_DOCUMENTS_SEARCH', () => {
test('UPDATE_TARGET_MAPPINGS_WAIT_FOR_TASK -> MARK_VERSION_INDEX_READY if sone versionIndexReadyActions', () => {
const res: ResponseType<'UPDATE_TARGET_MAPPINGS_WAIT_FOR_TASK'> = Either.right(
'pickup_updated_mappings_succeeded'
);
const newState = model(
{
...updateTargetMappingsWaitForTaskState,
versionIndexReadyActions: Option.some([
{ add: { index: 'kibana-index', alias: 'my-alias' } },
]),
},
res
) as UpdateTargetMappingsWaitForTaskState;
expect(newState.controlState).toEqual('MARK_VERSION_INDEX_READY');
expect(newState.retryCount).toEqual(0);
expect(newState.retryDelay).toEqual(0);
});
test('UPDATE_TARGET_MAPPINGS_WAIT_FOR_TASK -> DONE if none versionIndexReadyActions', () => {
const res: ResponseType<'UPDATE_TARGET_MAPPINGS_WAIT_FOR_TASK'> = Either.right(
'pickup_updated_mappings_succeeded'
);
const newState = model(
updateTargetMappingsWaitForTaskState,
res
) as UpdateTargetMappingsWaitForTaskState;
expect(newState.controlState).toEqual('OUTDATED_DOCUMENTS_SEARCH');
expect(newState.controlState).toEqual('DONE');
expect(newState.retryCount).toEqual(0);
expect(newState.retryDelay).toEqual(0);
});
Expand All @@ -736,7 +793,7 @@ describe('migrations v2 model', () => {
expect(newState.retryCount).toEqual(0);
expect(newState.retryDelay).toEqual(0);
});
test('OUTDATED_DOCUMENTS_SEARCH -> MARK_VERSION_INDEX_READY if none outdated documents were found and some versionIndexReadyActions', () => {
test('OUTDATED_DOCUMENTS_SEARCH -> UPDATE_TARGET_MAPPINGS if none outdated documents were found and some versionIndexReadyActions', () => {
const aliasActions = ([Symbol('alias action')] as unknown) as AliasAction[];
const outdatedDocumentsSourchStateWithSomeVersionIndexReadyActions = {
...outdatedDocumentsSourchState,
Expand All @@ -751,17 +808,17 @@ describe('migrations v2 model', () => {
outdatedDocumentsSourchStateWithSomeVersionIndexReadyActions,
res
) as MarkVersionIndexReady;
expect(newState.controlState).toEqual('MARK_VERSION_INDEX_READY');
expect(newState.controlState).toEqual('UPDATE_TARGET_MAPPINGS');
expect(newState.versionIndexReadyActions.value).toEqual(aliasActions);
expect(newState.retryCount).toEqual(0);
expect(newState.retryDelay).toEqual(0);
});
test('OUTDATED_DOCUMENTS_SEARCH -> DONE if none outdated documents were found and none versionIndexReadyActions', () => {
test('OUTDATED_DOCUMENTS_SEARCH -> UPDATE_TARGET_MAPPINGS if none outdated documents were found and none versionIndexReadyActions', () => {
const res: ResponseType<'OUTDATED_DOCUMENTS_SEARCH'> = Either.right({
outdatedDocuments: [],
});
const newState = model(outdatedDocumentsSourchState, res);
expect(newState.controlState).toEqual('DONE');
expect(newState.controlState).toEqual('UPDATE_TARGET_MAPPINGS');
expect(newState.retryCount).toEqual(0);
expect(newState.retryDelay).toEqual(0);
});
Expand Down Expand Up @@ -907,42 +964,30 @@ describe('migrations v2 model', () => {
"should": Array [
Object {
"bool": Object {
"must": Array [
Object {
"exists": Object {
"field": "my_dashboard",
},
"must": Object {
"term": Object {
"type": "my_dashboard",
},
Object {
"bool": Object {
"must_not": Object {
"term": Object {
"migrationVersion.my_dashboard": "7.10.1",
},
},
},
},
"must_not": Object {
"term": Object {
"migrationVersion.my_dashboard": "7.10.1",
},
],
},
},
},
Object {
"bool": Object {
"must": Array [
Object {
"exists": Object {
"field": "my_viz",
},
"must": Object {
"term": Object {
"type": "my_viz",
},
Object {
"bool": Object {
"must_not": Object {
"term": Object {
"migrationVersion.my_viz": "8.0.0",
},
},
},
},
"must_not": Object {
"term": Object {
"migrationVersion.my_viz": "8.0.0",
},
],
},
},
},
],
Expand Down
Loading

0 comments on commit 4f97684

Please sign in to comment.