Skip to content

Commit

Permalink
Remove executionLog to reduce memory consumption
Browse files Browse the repository at this point in the history
  • Loading branch information
@rudolf
rudolf committed Sep 10, 2021
1 parent e174e94 commit ce86dcf
Show file tree
Hide file tree
Showing 4 changed files with 74 additions and 315 deletions.
1 change: 1 addition & 0 deletions src/core/server/saved_objects/migrationsv2/integration_tests/cleanup.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ function createRoot() {
{
name: 'root',
appenders: ['file'],
level: 'debug', // DEBUG logs are required to retrieve the PIT _id from the action response logs
},
],
},
Expand Down
298 changes: 60 additions & 238 deletions src/core/server/saved_objects/migrationsv2/migrations_state_action_machine.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ describe('migrationsStateActionMachine', () => {
};
};

it('logs state transitions, messages in state.logs and action responses', async () => {
it('logs state transitions, messages in state.logs and action responses when reaching DONE', async () => {
await migrationStateActionMachine({
initialState,
logger: mockLogger.get(),
Expand Down Expand Up @@ -155,6 +155,60 @@ describe('migrationsStateActionMachine', () => {
`);
});

it('logs state transitions, messages in state.logs and action responses when reaching FATAL', async () => {
await migrationStateActionMachine({
initialState: {
...initialState,
reason: 'the fatal reason',
outdatedDocuments: [{ _id: '1234', password: 'sensitive password' }],
transformedDocBatches: [[{ _id: '1234', password: 'sensitive transformed password' }]],
} as State,
logger: mockLogger.get(),
model: transitionModel(['LEGACY_DELETE', 'FATAL']),
next,
client: esClient,
}).catch((err) => err);
expect(loggingSystemMock.collect(mockLogger)).toMatchInlineSnapshot(`
Object {
"debug": Array [
Array [
"[.my-so-index] INIT RESPONSE",
Object {
"_tag": "Right",
"right": "response",
},
],
Array [
"[.my-so-index] LEGACY_DELETE RESPONSE",
Object {
"_tag": "Right",
"right": "response",
},
],
],
"error": Array [],
"fatal": Array [],
"info": Array [
Array [
"[.my-so-index] Log from LEGACY_DELETE control state",
],
Array [
"[.my-so-index] INIT -> LEGACY_DELETE. took: 0ms.",
],
Array [
"[.my-so-index] Log from FATAL control state",
],
Array [
"[.my-so-index] LEGACY_DELETE -> FATAL. took: 0ms.",
],
],
"log": Array [],
"trace": Array [],
"warn": Array [],
}
`);
});

// see https://github.com/elastic/kibana/issues/98406
it('correctly logs state transition when using a logger adapter', async () => {
const underlyingLogger = mockLogger.get();
Expand Down Expand Up @@ -196,6 +250,7 @@ describe('migrationsStateActionMachine', () => {
})
).resolves.toEqual(expect.anything());
});

it('resolves with migrated status if some sourceIndex in the DONE state', async () => {
await expect(
migrationStateActionMachine({
Expand All @@ -207,6 +262,7 @@ describe('migrationsStateActionMachine', () => {
})
).resolves.toEqual(expect.objectContaining({ status: 'migrated' }));
});

it('resolves with patched status if none sourceIndex in the DONE state', async () => {
await expect(
migrationStateActionMachine({
Expand All @@ -218,6 +274,7 @@ describe('migrationsStateActionMachine', () => {
})
).resolves.toEqual(expect.objectContaining({ status: 'patched' }));
});

it('rejects with error message when reaching the FATAL state', async () => {
await expect(
migrationStateActionMachine({
Expand All @@ -231,127 +288,8 @@ describe('migrationsStateActionMachine', () => {
`[Error: Unable to complete saved object migrations for the [.my-so-index] index: the fatal reason]`
);
});
it('logs all state transitions and action responses when reaching the FATAL state', async () => {
await migrationStateActionMachine({
initialState: {
...initialState,
reason: 'the fatal reason',
outdatedDocuments: [{ _id: '1234', password: 'sensitive password' }],
transformedDocBatches: [[{ _id: '1234', password: 'sensitive transformed password' }]],
} as State,
logger: mockLogger.get(),
model: transitionModel(['LEGACY_DELETE', 'FATAL']),
next,
client: esClient,
}).catch((err) => err);
// Ignore the first 4 log entries that come from our model
const executionLogLogs = loggingSystemMock.collect(mockLogger).info.slice(4);
expect(executionLogLogs).toEqual([
[
'[.my-so-index] INIT RESPONSE',
{
_tag: 'Right',
right: 'response',
},
],
[
'[.my-so-index] INIT -> LEGACY_DELETE',
{
kibana: {
migrationState: {
batchSize: 1000,
maxBatchSizeBytes: 1e8,
controlState: 'LEGACY_DELETE',
currentAlias: '.my-so-index',
excludeFromUpgradeFilterHooks: {},
indexPrefix: '.my-so-index',
kibanaVersion: '7.11.0',
knownTypes: [],
legacyIndex: '.my-so-index',
logs: [
{
level: 'info',
message: 'Log from LEGACY_DELETE control state',
},
],
outdatedDocuments: [{ _id: '1234' }],
outdatedDocumentsQuery: expect.any(Object),
preMigrationScript: {
_tag: 'None',
},
reason: 'the fatal reason',
retryAttempts: 5,
retryCount: 0,
retryDelay: 0,
targetIndexMappings: {
properties: {},
},
tempIndex: '.my-so-index_7.11.0_reindex_temp',
tempIndexMappings: expect.any(Object),
transformedDocBatches: [[{ _id: '1234' }]],
unusedTypesQuery: expect.any(Object),
versionAlias: '.my-so-index_7.11.0',
versionIndex: '.my-so-index_7.11.0_001',
},
},
},
],
[
'[.my-so-index] LEGACY_DELETE RESPONSE',
{
_tag: 'Right',
right: 'response',
},
],
[
'[.my-so-index] LEGACY_DELETE -> FATAL',
{
kibana: {
migrationState: {
batchSize: 1000,
maxBatchSizeBytes: 1e8,
controlState: 'FATAL',
currentAlias: '.my-so-index',
excludeFromUpgradeFilterHooks: {},
indexPrefix: '.my-so-index',
kibanaVersion: '7.11.0',
knownTypes: [],
legacyIndex: '.my-so-index',
logs: [
{
level: 'info',
message: 'Log from LEGACY_DELETE control state',
},
{
level: 'info',
message: 'Log from FATAL control state',
},
],
outdatedDocuments: [{ _id: '1234' }],
outdatedDocumentsQuery: expect.any(Object),
preMigrationScript: {
_tag: 'None',
},
reason: 'the fatal reason',
retryAttempts: 5,
retryCount: 0,
retryDelay: 0,
targetIndexMappings: {
properties: {},
},
tempIndex: '.my-so-index_7.11.0_reindex_temp',
tempIndexMappings: expect.any(Object),
transformedDocBatches: [[{ _id: '1234' }]],
unusedTypesQuery: expect.any(Object),
versionAlias: '.my-so-index_7.11.0',
versionIndex: '.my-so-index_7.11.0_001',
},
},
},
],
]);
});
it('rejects and logs the error when an action throws with an ResponseError', async () => {

it('rejects and logs the error when an action throws with a ResponseError', async () => {
await expect(
migrationStateActionMachine({
initialState: { ...initialState, reason: 'the fatal reason' } as State,
Expand Down Expand Up @@ -384,9 +322,6 @@ describe('migrationsStateActionMachine', () => {
Array [
"[.my-so-index] Unexpected Elasticsearch ResponseError: statusCode: 200, method: POST, url: /mock error: [snapshot_in_progress_exception]: Cannot delete indices that are being snapshotted,",
],
Array [
"[.my-so-index] migration failed, dumping execution log:",
],
],
"fatal": Array [],
"info": Array [],
Expand Down Expand Up @@ -417,9 +352,6 @@ describe('migrationsStateActionMachine', () => {
Array [
[Error: this action throws],
],
Array [
"[.my-so-index] migration failed, dumping execution log:",
],
],
"fatal": Array [],
"info": Array [],
Expand All @@ -429,116 +361,6 @@ describe('migrationsStateActionMachine', () => {
}
`);
});
it('logs all state transitions and action responses when an action throws', async () => {
try {
await migrationStateActionMachine({
initialState: { ...initialState, reason: 'the fatal reason' } as State,
logger: mockLogger.get(),
model: transitionModel(['LEGACY_REINDEX', 'LEGACY_DELETE', 'FATAL']),
next: (state) => {
if (state.controlState === 'LEGACY_DELETE') throw new Error('this action throws');
return () => Promise.resolve('hello');
},
client: esClient,
});
} catch (e) {
/** ignore */
}
// Ignore the first 4 log entries that come from our model
const executionLogLogs = loggingSystemMock.collect(mockLogger).info.slice(4);
expect(executionLogLogs).toEqual([
['[.my-so-index] INIT RESPONSE', 'hello'],
[
'[.my-so-index] INIT -> LEGACY_REINDEX',
{
kibana: {
migrationState: {
batchSize: 1000,
maxBatchSizeBytes: 1e8,
controlState: 'LEGACY_REINDEX',
currentAlias: '.my-so-index',
excludeFromUpgradeFilterHooks: {},
indexPrefix: '.my-so-index',
kibanaVersion: '7.11.0',
knownTypes: [],
legacyIndex: '.my-so-index',
logs: [
{
level: 'info',
message: 'Log from LEGACY_REINDEX control state',
},
],
outdatedDocuments: [],
outdatedDocumentsQuery: expect.any(Object),
preMigrationScript: {
_tag: 'None',
},
reason: 'the fatal reason',
retryAttempts: 5,
retryCount: 0,
retryDelay: 0,
targetIndexMappings: {
properties: {},
},
tempIndex: '.my-so-index_7.11.0_reindex_temp',
tempIndexMappings: expect.any(Object),
transformedDocBatches: [],
unusedTypesQuery: expect.any(Object),
versionAlias: '.my-so-index_7.11.0',
versionIndex: '.my-so-index_7.11.0_001',
},
},
},
],
['[.my-so-index] LEGACY_REINDEX RESPONSE', 'hello'],
[
'[.my-so-index] LEGACY_REINDEX -> LEGACY_DELETE',
{
kibana: {
migrationState: {
batchSize: 1000,
maxBatchSizeBytes: 1e8,
controlState: 'LEGACY_DELETE',
currentAlias: '.my-so-index',
excludeFromUpgradeFilterHooks: {},
indexPrefix: '.my-so-index',
kibanaVersion: '7.11.0',
knownTypes: [],
legacyIndex: '.my-so-index',
logs: [
{
level: 'info',
message: 'Log from LEGACY_REINDEX control state',
},
{
level: 'info',
message: 'Log from LEGACY_DELETE control state',
},
],
outdatedDocuments: [],
outdatedDocumentsQuery: expect.any(Object),
preMigrationScript: {
_tag: 'None',
},
reason: 'the fatal reason',
retryAttempts: 5,
retryCount: 0,
retryDelay: 0,
targetIndexMappings: {
properties: {},
},
tempIndex: '.my-so-index_7.11.0_reindex_temp',
tempIndexMappings: expect.any(Object),
transformedDocBatches: [],
unusedTypesQuery: expect.any(Object),
versionAlias: '.my-so-index_7.11.0',
versionIndex: '.my-so-index_7.11.0_001',
},
},
},
],
]);
});
describe('cleanup', () => {
beforeEach(() => {
cleanupMock.mockClear();
Expand Down
Loading

0 comments on commit ce86dcf

Please sign in to comment.