From 944c541c0c68dd539afedc34ab7e69fb82d25925 Mon Sep 17 00:00:00 2001 From: Jean-Louis Leysens Date: Fri, 14 May 2021 17:10:55 +0200 Subject: [PATCH 1/9] wip; added logic for creating ILM policy at start up --- .../server/lib/store/report_ilm_policy.ts | 26 +++++++ .../reporting/server/lib/store/store.ts | 67 +++++++++++++++---- x-pack/plugins/reporting/server/plugin.ts | 3 + 3 files changed, 84 insertions(+), 12 deletions(-) create mode 100644 x-pack/plugins/reporting/server/lib/store/report_ilm_policy.ts diff --git a/x-pack/plugins/reporting/server/lib/store/report_ilm_policy.ts b/x-pack/plugins/reporting/server/lib/store/report_ilm_policy.ts new file mode 100644 index 0000000000000..ff89ef26d890d --- /dev/null +++ b/x-pack/plugins/reporting/server/lib/store/report_ilm_policy.ts @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { PutLifecycleRequest } from '@elastic/elasticsearch/api/types'; + +// TODO: Review the default/starting policy +export const reportingIlmPolicy: PutLifecycleRequest['body'] = { + policy: { + phases: { + hot: { + actions: {}, + min_age: '0ms', + }, + delete: { + min_age: '180d', + actions: { + delete: {}, + }, + }, + }, + }, +}; diff --git a/x-pack/plugins/reporting/server/lib/store/store.ts b/x-pack/plugins/reporting/server/lib/store/store.ts index fc7bd9c23d769..4c53f89816dce 100644 --- a/x-pack/plugins/reporting/server/lib/store/store.ts +++ b/x-pack/plugins/reporting/server/lib/store/store.ts @@ -14,6 +14,7 @@ import { ReportTaskParams } from '../tasks'; import { indexTimestamp } from './index_timestamp'; import { mapping } from './mapping'; import { Report, ReportDocument, ReportSource } from './report'; +import { reportingIlmPolicy } from './report_ilm_policy'; /* * When searching for long-pending reports, we get a subset of fields @@ -71,19 +72,22 @@ export class ReportingStore { return exists; } - const indexSettings = { - number_of_shards: 1, - auto_expand_replicas: '0-1', - }; - const body = { - settings: indexSettings, - mappings: { - properties: mapping, - }, - }; - try { - await client.indices.create({ index: indexName, body }); + await client.indices.create({ + index: indexName, + body: { + settings: { + number_of_shards: 1, + auto_expand_replicas: '0-1', + lifecycle: { + name: this.ilmPolicyName, + }, + }, + mappings: { + properties: mapping, + }, + }, + }); return true; } catch (error) { @@ -130,6 +134,45 @@ export class ReportingStore { return client.indices.refresh({ index }); } + private readonly ilmPolicyName = 'kibana-reporting'; + + private async doesIlmPolicyExist(): Promise { + const client = await this.getClient(); + try { + await client.ilm.getLifecycle({ policy: this.ilmPolicyName }); + return true; + } catch (e) { + if (e.statusCode === 404) { + return false; + } + throw e; + } + } + + /** + * Function to be called during plugin start phase. This ensures the environment is correctly + * configured for storage of reports. + */ + public async start() { + const client = await this.getClient(); + try { + if (await this.doesIlmPolicyExist()) { + return; + } + this.logger.debug( + `Creating ILM policy for managing reporting indices: ${this.ilmPolicyName}` + ); + await client.ilm.putLifecycle({ + policy: this.ilmPolicyName, + body: reportingIlmPolicy, + }); + } catch (e) { + this.logger.error('Error in start phase'); + this.logger.error(e.body.error); + throw e; + } + } + public async addReport(report: Report): Promise { let index = report._index; if (!index) { diff --git a/x-pack/plugins/reporting/server/plugin.ts b/x-pack/plugins/reporting/server/plugin.ts index 26a9be2b15c3f..e0ac39ed45b68 100644 --- a/x-pack/plugins/reporting/server/plugin.ts +++ b/x-pack/plugins/reporting/server/plugin.ts @@ -107,6 +107,9 @@ export class ReportingPlugin logger: this.logger, }); + // Note: this must be called after ReportingCore.pluginStart + await store.start(); + this.logger.debug('Start complete'); })().catch((e) => { this.logger.error(`Error in Reporting start, reporting may not function properly`); From c94974febe5da4093cf547f0c3d203da7fc8d482 Mon Sep 17 00:00:00 2001 From: Jean-Louis Leysens Date: Mon, 17 May 2021 12:04:10 +0200 Subject: [PATCH 2/9] added log when ilm policy is not created --- x-pack/plugins/reporting/server/lib/store/store.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugins/reporting/server/lib/store/store.ts b/x-pack/plugins/reporting/server/lib/store/store.ts index 4c53f89816dce..b6a5d3dfe94cc 100644 --- a/x-pack/plugins/reporting/server/lib/store/store.ts +++ b/x-pack/plugins/reporting/server/lib/store/store.ts @@ -157,6 +157,7 @@ export class ReportingStore { const client = await this.getClient(); try { if (await this.doesIlmPolicyExist()) { + this.logger.debug(`Found ILM policy ${this.ilmPolicyName}; skipping creation.`); return; } this.logger.debug( From cc739ba867e67f48313202ec5d173860ac80e926 Mon Sep 17 00:00:00 2001 From: Jean-Louis Leysens Date: Mon, 17 May 2021 12:25:26 +0200 Subject: [PATCH 3/9] added test for start function --- .../reporting/server/lib/store/store.test.ts | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/x-pack/plugins/reporting/server/lib/store/store.test.ts b/x-pack/plugins/reporting/server/lib/store/store.test.ts index 7f96433fcc6ce..52471a105bd4f 100644 --- a/x-pack/plugins/reporting/server/lib/store/store.test.ts +++ b/x-pack/plugins/reporting/server/lib/store/store.test.ts @@ -7,6 +7,7 @@ import type { DeeplyMockedKeys } from '@kbn/utility-types/jest'; import { ElasticsearchClient } from 'src/core/server'; +import { elasticsearchServiceMock } from 'src/core/server/mocks'; import { ReportingCore } from '../../'; import { createMockConfigSchema, @@ -15,6 +16,9 @@ import { } from '../../test_helpers'; import { Report, ReportDocument } from './report'; import { ReportingStore } from './store'; +import { reportingIlmPolicy } from './report_ilm_policy'; + +const { createApiResponse } = elasticsearchServiceMock; describe('ReportingStore', () => { const mockLogger = createMockLevelLogger(); @@ -403,4 +407,47 @@ describe('ReportingStore', () => { ] `); }); + + describe('start', () => { + it('creates an ILM policy for managing reporting indices if there is not already one', async () => { + mockEsClient.ilm.getLifecycle.mockRejectedValueOnce(createApiResponse({ statusCode: 404 })); + mockEsClient.ilm.putLifecycle.mockResolvedValueOnce(createApiResponse()); + + const store = new ReportingStore(mockCore, mockLogger); + await store.start(); + + expect(mockEsClient.ilm.getLifecycle).toHaveBeenCalledWith({ policy: 'kibana-reporting' }); + expect(mockEsClient.ilm.putLifecycle.mock.calls[0][0]).toMatchInlineSnapshot(` + Object { + "body": Object { + "policy": Object { + "phases": Object { + "delete": Object { + "actions": Object { + "delete": Object {}, + }, + "min_age": "180d", + }, + "hot": Object { + "actions": Object {}, + "min_age": "0ms", + }, + }, + }, + }, + "policy": "kibana-reporting", + } + `); + }); + + it('does not create an ILM policy for managing reporting indices if one already exists', async () => { + mockEsClient.ilm.getLifecycle.mockResolvedValueOnce(createApiResponse()); + + const store = new ReportingStore(mockCore, mockLogger); + await store.start(); + + expect(mockEsClient.ilm.getLifecycle).toHaveBeenCalledWith({ policy: 'kibana-reporting' }); + expect(mockEsClient.ilm.putLifecycle).not.toHaveBeenCalled(); + }); + }); }); From e9a27f7ad516eeb7abcddb7146a71922c7585988 Mon Sep 17 00:00:00 2001 From: Jean-Louis Leysens Date: Mon, 17 May 2021 13:37:33 +0200 Subject: [PATCH 4/9] updated ilm policy to not delete data --- .../plugins/reporting/server/lib/store/report_ilm_policy.ts | 6 ------ 1 file changed, 6 deletions(-) diff --git a/x-pack/plugins/reporting/server/lib/store/report_ilm_policy.ts b/x-pack/plugins/reporting/server/lib/store/report_ilm_policy.ts index ff89ef26d890d..9b37d3e293cf0 100644 --- a/x-pack/plugins/reporting/server/lib/store/report_ilm_policy.ts +++ b/x-pack/plugins/reporting/server/lib/store/report_ilm_policy.ts @@ -15,12 +15,6 @@ export const reportingIlmPolicy: PutLifecycleRequest['body'] = { actions: {}, min_age: '0ms', }, - delete: { - min_age: '180d', - actions: { - delete: {}, - }, - }, }, }, }; From 385e1700b6447eaa0923b1597bef4a2e2bc79cb2 Mon Sep 17 00:00:00 2001 From: Jean-Louis Leysens Date: Mon, 17 May 2021 13:39:33 +0200 Subject: [PATCH 5/9] actually update jest snapshots and remove unused import --- x-pack/plugins/reporting/server/lib/store/store.test.ts | 7 ------- 1 file changed, 7 deletions(-) diff --git a/x-pack/plugins/reporting/server/lib/store/store.test.ts b/x-pack/plugins/reporting/server/lib/store/store.test.ts index 52471a105bd4f..0f00c9279d0bf 100644 --- a/x-pack/plugins/reporting/server/lib/store/store.test.ts +++ b/x-pack/plugins/reporting/server/lib/store/store.test.ts @@ -16,7 +16,6 @@ import { } from '../../test_helpers'; import { Report, ReportDocument } from './report'; import { ReportingStore } from './store'; -import { reportingIlmPolicy } from './report_ilm_policy'; const { createApiResponse } = elasticsearchServiceMock; @@ -422,12 +421,6 @@ describe('ReportingStore', () => { "body": Object { "policy": Object { "phases": Object { - "delete": Object { - "actions": Object { - "delete": Object {}, - }, - "min_age": "180d", - }, "hot": Object { "actions": Object {}, "min_age": "0ms", From bcabc5c143277dd46927e45fce11f37f65f4ee25 Mon Sep 17 00:00:00 2001 From: Jean-Louis Leysens Date: Tue, 18 May 2021 16:18:12 +0200 Subject: [PATCH 6/9] updated the ilm policy, removed the min_age for the hot phase --- x-pack/plugins/reporting/server/lib/store/report_ilm_policy.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugins/reporting/server/lib/store/report_ilm_policy.ts b/x-pack/plugins/reporting/server/lib/store/report_ilm_policy.ts index 9b37d3e293cf0..7e5718a76e7ca 100644 --- a/x-pack/plugins/reporting/server/lib/store/report_ilm_policy.ts +++ b/x-pack/plugins/reporting/server/lib/store/report_ilm_policy.ts @@ -13,7 +13,6 @@ export const reportingIlmPolicy: PutLifecycleRequest['body'] = { phases: { hot: { actions: {}, - min_age: '0ms', }, }, }, From 4f2aa3f64a5835ab657fe3e04366ba19caad95f1 Mon Sep 17 00:00:00 2001 From: Jean-Louis Leysens Date: Thu, 20 May 2021 12:11:45 +0200 Subject: [PATCH 7/9] update jest snapshot --- x-pack/plugins/reporting/server/lib/store/store.test.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugins/reporting/server/lib/store/store.test.ts b/x-pack/plugins/reporting/server/lib/store/store.test.ts index 0f00c9279d0bf..fa35240dfc8fb 100644 --- a/x-pack/plugins/reporting/server/lib/store/store.test.ts +++ b/x-pack/plugins/reporting/server/lib/store/store.test.ts @@ -423,7 +423,6 @@ describe('ReportingStore', () => { "phases": Object { "hot": Object { "actions": Object {}, - "min_age": "0ms", }, }, }, From c74cf0eebb09f2a0b6f40802a4aee36062c426bb Mon Sep 17 00:00:00 2001 From: Jean-Louis Leysens Date: Thu, 20 May 2021 12:24:16 +0200 Subject: [PATCH 8/9] removed TODO comment --- x-pack/plugins/reporting/server/lib/store/report_ilm_policy.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugins/reporting/server/lib/store/report_ilm_policy.ts b/x-pack/plugins/reporting/server/lib/store/report_ilm_policy.ts index 7e5718a76e7ca..f4cd69a0331d7 100644 --- a/x-pack/plugins/reporting/server/lib/store/report_ilm_policy.ts +++ b/x-pack/plugins/reporting/server/lib/store/report_ilm_policy.ts @@ -7,7 +7,6 @@ import { PutLifecycleRequest } from '@elastic/elasticsearch/api/types'; -// TODO: Review the default/starting policy export const reportingIlmPolicy: PutLifecycleRequest['body'] = { policy: { phases: { From 15f2d3497d1afe20a35117bf09f35257920dc299 Mon Sep 17 00:00:00 2001 From: Jean-Louis Leysens Date: Tue, 25 May 2021 11:16:16 +0200 Subject: [PATCH 9/9] debug log -> info log --- x-pack/plugins/reporting/server/lib/store/store.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/x-pack/plugins/reporting/server/lib/store/store.ts b/x-pack/plugins/reporting/server/lib/store/store.ts index b6a5d3dfe94cc..9fb203fd5627a 100644 --- a/x-pack/plugins/reporting/server/lib/store/store.ts +++ b/x-pack/plugins/reporting/server/lib/store/store.ts @@ -160,9 +160,7 @@ export class ReportingStore { this.logger.debug(`Found ILM policy ${this.ilmPolicyName}; skipping creation.`); return; } - this.logger.debug( - `Creating ILM policy for managing reporting indices: ${this.ilmPolicyName}` - ); + this.logger.info(`Creating ILM policy for managing reporting indices: ${this.ilmPolicyName}`); await client.ilm.putLifecycle({ policy: this.ilmPolicyName, body: reportingIlmPolicy,