Skip to content

Commit

Permalink
[Reporting] ILM policy for managing reporting indices (elastic#100130)
Browse files Browse the repository at this point in the history
* wip; added logic for creating ILM policy at start up

* added log when ilm policy is not created

* added test for start function

* updated ilm policy to not delete data

* actually update jest snapshots and remove unused import

* updated the ilm policy, removed the min_age for the hot phase

* update jest snapshot

* removed TODO comment

* debug log -> info log

Co-authored-by: Kibana Machine <[email protected]>
  • Loading branch information
jloleysens and kibanamachine committed Jun 14, 2021
1 parent 80b109f commit c75f58c
Show file tree
Hide file tree
Showing 4 changed files with 114 additions and 12 deletions.
18 changes: 18 additions & 0 deletions x-pack/plugins/reporting/server/lib/store/report_ilm_policy.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/

import { PutLifecycleRequest } from '@elastic/elasticsearch/api/types';

export const reportingIlmPolicy: PutLifecycleRequest['body'] = {
policy: {
phases: {
hot: {
actions: {},
},
},
},
};
39 changes: 39 additions & 0 deletions x-pack/plugins/reporting/server/lib/store/store.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@

import type { DeeplyMockedKeys } from '@kbn/utility-types/jest';
import { ElasticsearchClient } from 'src/core/server';
import { elasticsearchServiceMock } from 'src/core/server/mocks';
import { ReportingCore } from '../../';
import {
createMockConfigSchema,
Expand All @@ -16,6 +17,8 @@ import {
import { Report, ReportDocument } from './report';
import { ReportingStore } from './store';

const { createApiResponse } = elasticsearchServiceMock;

describe('ReportingStore', () => {
const mockLogger = createMockLevelLogger();
let mockCore: ReportingCore;
Expand Down Expand Up @@ -403,4 +406,40 @@ describe('ReportingStore', () => {
]
`);
});

describe('start', () => {
it('creates an ILM policy for managing reporting indices if there is not already one', async () => {
mockEsClient.ilm.getLifecycle.mockRejectedValueOnce(createApiResponse({ statusCode: 404 }));
mockEsClient.ilm.putLifecycle.mockResolvedValueOnce(createApiResponse());

const store = new ReportingStore(mockCore, mockLogger);
await store.start();

expect(mockEsClient.ilm.getLifecycle).toHaveBeenCalledWith({ policy: 'kibana-reporting' });
expect(mockEsClient.ilm.putLifecycle.mock.calls[0][0]).toMatchInlineSnapshot(`
Object {
"body": Object {
"policy": Object {
"phases": Object {
"hot": Object {
"actions": Object {},
},
},
},
},
"policy": "kibana-reporting",
}
`);
});

it('does not create an ILM policy for managing reporting indices if one already exists', async () => {
mockEsClient.ilm.getLifecycle.mockResolvedValueOnce(createApiResponse());

const store = new ReportingStore(mockCore, mockLogger);
await store.start();

expect(mockEsClient.ilm.getLifecycle).toHaveBeenCalledWith({ policy: 'kibana-reporting' });
expect(mockEsClient.ilm.putLifecycle).not.toHaveBeenCalled();
});
});
});
66 changes: 54 additions & 12 deletions x-pack/plugins/reporting/server/lib/store/store.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import { ReportTaskParams } from '../tasks';
import { indexTimestamp } from './index_timestamp';
import { mapping } from './mapping';
import { Report, ReportDocument, ReportSource } from './report';
import { reportingIlmPolicy } from './report_ilm_policy';

/*
* When searching for long-pending reports, we get a subset of fields
Expand Down Expand Up @@ -71,19 +72,22 @@ export class ReportingStore {
return exists;
}

const indexSettings = {
number_of_shards: 1,
auto_expand_replicas: '0-1',
};
const body = {
settings: indexSettings,
mappings: {
properties: mapping,
},
};

try {
await client.indices.create({ index: indexName, body });
await client.indices.create({
index: indexName,
body: {
settings: {
number_of_shards: 1,
auto_expand_replicas: '0-1',
lifecycle: {
name: this.ilmPolicyName,
},
},
mappings: {
properties: mapping,
},
},
});

return true;
} catch (error) {
Expand Down Expand Up @@ -130,6 +134,44 @@ export class ReportingStore {
return client.indices.refresh({ index });
}

private readonly ilmPolicyName = 'kibana-reporting';

private async doesIlmPolicyExist(): Promise<boolean> {
const client = await this.getClient();
try {
await client.ilm.getLifecycle({ policy: this.ilmPolicyName });
return true;
} catch (e) {
if (e.statusCode === 404) {
return false;
}
throw e;
}
}

/**
* Function to be called during plugin start phase. This ensures the environment is correctly
* configured for storage of reports.
*/
public async start() {
const client = await this.getClient();
try {
if (await this.doesIlmPolicyExist()) {
this.logger.debug(`Found ILM policy ${this.ilmPolicyName}; skipping creation.`);
return;
}
this.logger.info(`Creating ILM policy for managing reporting indices: ${this.ilmPolicyName}`);
await client.ilm.putLifecycle({
policy: this.ilmPolicyName,
body: reportingIlmPolicy,
});
} catch (e) {
this.logger.error('Error in start phase');
this.logger.error(e.body.error);
throw e;
}
}

public async addReport(report: Report): Promise<Report> {
let index = report._index;
if (!index) {
Expand Down
3 changes: 3 additions & 0 deletions x-pack/plugins/reporting/server/plugin.ts
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,9 @@ export class ReportingPlugin
logger: this.logger,
});

// Note: this must be called after ReportingCore.pluginStart
await store.start();

this.logger.debug('Start complete');
})().catch((e) => {
this.logger.error(`Error in Reporting start, reporting may not function properly`);
Expand Down

0 comments on commit c75f58c

Please sign in to comment.