From bcdf60a71a47102defdd50ed937c66518e1fcd7c Mon Sep 17 00:00:00 2001 From: Tim Sullivan <tsullivan@users.noreply.github.com> Date: Thu, 25 Jun 2020 14:52:30 -0700 Subject: [PATCH] [Reporting] ReportingStore module (#69426) * Add store class * fix tests * fix the createIndex bug * add reportingstore test * change function args * nits * add test for automatic index creation failure recovery # Conflicts: # x-pack/plugins/reporting/server/lib/esqueue/job.js --- x-pack/plugins/reporting/server/core.ts | 2 + .../reporting/server/lib/create_queue.ts | 16 +- .../reporting/server/lib/enqueue_job.ts | 49 +- .../esqueue/__tests__/helpers/create_index.js | 100 ----- .../__tests__/helpers/index_timestamp.js | 93 ---- .../server/lib/esqueue/__tests__/job.js | 420 ------------------ .../reporting/server/lib/esqueue/index.js | 24 +- .../reporting/server/lib/esqueue/job.js | 142 ------ .../reporting/server/lib/esqueue/worker.js | 12 +- x-pack/plugins/reporting/server/lib/index.ts | 1 + .../reporting/server/lib/store/index.ts | 8 + .../index_timestamp.ts} | 9 +- .../reporting/server/lib/store/mapping.ts | 65 +++ .../reporting/server/lib/store/report.test.ts | 77 ++++ .../reporting/server/lib/store/report.ts | 85 ++++ .../reporting/server/lib/store/store.test.ts | 166 +++++++ .../reporting/server/lib/store/store.ts | 169 +++++++ x-pack/plugins/reporting/server/plugin.ts | 15 +- .../test_helpers/create_mock_levellogger.ts | 23 + .../create_mock_reportingplugin.ts | 27 +- .../reporting_api_integration/services.ts | 3 +- 21 files changed, 665 insertions(+), 841 deletions(-) delete mode 100644 x-pack/plugins/reporting/server/lib/esqueue/__tests__/helpers/create_index.js delete mode 100644 x-pack/plugins/reporting/server/lib/esqueue/__tests__/helpers/index_timestamp.js delete mode 100644 x-pack/plugins/reporting/server/lib/esqueue/__tests__/job.js delete mode 100644 x-pack/plugins/reporting/server/lib/esqueue/job.js create mode 100644 x-pack/plugins/reporting/server/lib/store/index.ts rename x-pack/plugins/reporting/server/lib/{esqueue/helpers/index_timestamp.js => store/index_timestamp.ts} (80%) create mode 100644 x-pack/plugins/reporting/server/lib/store/mapping.ts create mode 100644 x-pack/plugins/reporting/server/lib/store/report.test.ts create mode 100644 x-pack/plugins/reporting/server/lib/store/report.ts create mode 100644 x-pack/plugins/reporting/server/lib/store/store.test.ts create mode 100644 x-pack/plugins/reporting/server/lib/store/store.ts create mode 100644 x-pack/plugins/reporting/server/test_helpers/create_mock_levellogger.ts diff --git a/x-pack/plugins/reporting/server/core.ts b/x-pack/plugins/reporting/server/core.ts index 9acd359fa0db4..eccd6c7db1698 100644 --- a/x-pack/plugins/reporting/server/core.ts +++ b/x-pack/plugins/reporting/server/core.ts @@ -24,6 +24,7 @@ import { screenshotsObservableFactory } from './export_types/common/lib/screensh import { checkLicense, getExportTypesRegistry } from './lib'; import { ESQueueInstance } from './lib/create_queue'; import { EnqueueJobFn } from './lib/enqueue_job'; +import { ReportingStore } from './lib/store'; export interface ReportingInternalSetup { elasticsearch: ElasticsearchServiceSetup; @@ -37,6 +38,7 @@ export interface ReportingInternalStart { browserDriverFactory: HeadlessChromiumDriverFactory; enqueueJob: EnqueueJobFn; esqueue: ESQueueInstance; + store: ReportingStore; savedObjects: SavedObjectsServiceStart; uiSettings: UiSettingsServiceStart; } diff --git a/x-pack/plugins/reporting/server/lib/create_queue.ts b/x-pack/plugins/reporting/server/lib/create_queue.ts index 5d09af312a41b..a8dcb92c55b2d 100644 --- a/x-pack/plugins/reporting/server/lib/create_queue.ts +++ b/x-pack/plugins/reporting/server/lib/create_queue.ts @@ -8,17 +8,16 @@ import { ReportingCore } from '../core'; import { JobSource, TaskRunResult } from '../types'; import { createTaggedLogger } from './create_tagged_logger'; // TODO remove createTaggedLogger once esqueue is removed import { createWorkerFactory } from './create_worker'; -import { Job } from './enqueue_job'; // @ts-ignore import { Esqueue } from './esqueue'; import { LevelLogger } from './level_logger'; +import { ReportingStore } from './store'; interface ESQueueWorker { on: (event: string, handler: any) => void; } export interface ESQueueInstance { - addJob: (type: string, payload: unknown, options: object) => Job; registerWorker: <JobParamsType>( pluginId: string, workerFn: GenericWorkerFn<JobParamsType>, @@ -37,26 +36,25 @@ type GenericWorkerFn<JobParamsType> = ( ...workerRestArgs: any[] ) => void | Promise<TaskRunResult>; -export async function createQueueFactory<JobParamsType, JobPayloadType>( +export async function createQueueFactory( reporting: ReportingCore, + store: ReportingStore, logger: LevelLogger ): Promise<ESQueueInstance> { const config = reporting.getConfig(); - const queueIndexInterval = config.get('queue', 'indexInterval'); + + // esqueue-related const queueTimeout = config.get('queue', 'timeout'); - const queueIndex = config.get('index'); const isPollingEnabled = config.get('queue', 'pollEnabled'); - const elasticsearch = await reporting.getElasticsearchService(); + const elasticsearch = reporting.getElasticsearchService(); const queueOptions = { - interval: queueIndexInterval, timeout: queueTimeout, - dateSeparator: '.', client: elasticsearch.legacy.client, logger: createTaggedLogger(logger, ['esqueue', 'queue-worker']), }; - const queue: ESQueueInstance = new Esqueue(queueIndex, queueOptions); + const queue: ESQueueInstance = new Esqueue(store, queueOptions); if (isPollingEnabled) { // create workers to poll the index for idle jobs waiting to be claimed and executed diff --git a/x-pack/plugins/reporting/server/lib/enqueue_job.ts b/x-pack/plugins/reporting/server/lib/enqueue_job.ts index 625da90f3b4f2..d1554a03b9389 100644 --- a/x-pack/plugins/reporting/server/lib/enqueue_job.ts +++ b/x-pack/plugins/reporting/server/lib/enqueue_job.ts @@ -4,39 +4,24 @@ * you may not use this file except in compliance with the Elastic License. */ -import { EventEmitter } from 'events'; import { KibanaRequest, RequestHandlerContext } from 'src/core/server'; import { AuthenticatedUser } from '../../../security/server'; import { ESQueueCreateJobFn } from '../../server/types'; import { ReportingCore } from '../core'; -// @ts-ignore -import { events as esqueueEvents } from './esqueue'; -import { LevelLogger } from './level_logger'; +import { LevelLogger } from './'; +import { ReportingStore, Report } from './store'; -interface ConfirmedJob { - id: string; - index: string; - _seq_no: number; - _primary_term: number; -} - -export type Job = EventEmitter & { - id: string; - toJSON: () => { - id: string; - }; -}; - -export type EnqueueJobFn = <JobParamsType>( +export type EnqueueJobFn = ( exportTypeId: string, - jobParams: JobParamsType, + jobParams: unknown, user: AuthenticatedUser | null, context: RequestHandlerContext, request: KibanaRequest -) => Promise<Job>; +) => Promise<Report>; export function enqueueJobFactory( reporting: ReportingCore, + store: ReportingStore, parentLogger: LevelLogger ): EnqueueJobFn { const config = reporting.getConfig(); @@ -45,16 +30,16 @@ export function enqueueJobFactory( const maxAttempts = config.get('capture', 'maxAttempts'); const logger = parentLogger.clone(['queue-job']); - return async function enqueueJob<JobParamsType>( + return async function enqueueJob( exportTypeId: string, - jobParams: JobParamsType, + jobParams: unknown, user: AuthenticatedUser | null, context: RequestHandlerContext, request: KibanaRequest - ): Promise<Job> { - type ScheduleTaskFnType = ESQueueCreateJobFn<JobParamsType>; + ) { + type ScheduleTaskFnType = ESQueueCreateJobFn<unknown>; + const username = user ? user.username : false; - const esqueue = await reporting.getEsqueue(); const exportType = reporting.getExportTypesRegistry().getById(exportTypeId); if (exportType == null) { @@ -71,16 +56,6 @@ export function enqueueJobFactory( max_attempts: maxAttempts, }; - return new Promise((resolve, reject) => { - const job = esqueue.addJob(exportType.jobType, payload, options); - - job.on(esqueueEvents.EVENT_JOB_CREATED, (createdJob: ConfirmedJob) => { - if (createdJob.id === job.id) { - logger.info(`Successfully queued job: ${createdJob.id}`); - resolve(job); - } - }); - job.on(esqueueEvents.EVENT_JOB_CREATE_ERROR, reject); - }); + return await store.addReport(exportType.jobType, payload, options); }; } diff --git a/x-pack/plugins/reporting/server/lib/esqueue/__tests__/helpers/create_index.js b/x-pack/plugins/reporting/server/lib/esqueue/__tests__/helpers/create_index.js deleted file mode 100644 index 691bd4f618a1c..0000000000000 --- a/x-pack/plugins/reporting/server/lib/esqueue/__tests__/helpers/create_index.js +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import expect from '@kbn/expect'; -import sinon from 'sinon'; -import { createIndex } from '../../helpers/create_index'; -import { ClientMock } from '../fixtures/legacy_elasticsearch'; -import { constants } from '../../constants'; - -describe('Create Index', function () { - describe('Does not exist', function () { - let client; - let createSpy; - - beforeEach(function () { - client = new ClientMock(); - createSpy = sinon.spy(client, 'callAsInternalUser').withArgs('indices.create'); - }); - - it('should return true', function () { - const indexName = 'test-index'; - const result = createIndex(client, indexName); - - return result.then((exists) => expect(exists).to.be(true)); - }); - - it('should create the index with mappings and default settings', function () { - const indexName = 'test-index'; - const settings = constants.DEFAULT_SETTING_INDEX_SETTINGS; - const result = createIndex(client, indexName); - - return result.then(function () { - const payload = createSpy.getCall(0).args[1]; - sinon.assert.callCount(createSpy, 1); - expect(payload).to.have.property('index', indexName); - expect(payload).to.have.property('body'); - expect(payload.body).to.have.property('settings'); - expect(payload.body.settings).to.eql(settings); - expect(payload.body).to.have.property('mappings'); - expect(payload.body.mappings).to.have.property('properties'); - }); - }); - - it('should create the index with custom settings', function () { - const indexName = 'test-index'; - const settings = { - ...constants.DEFAULT_SETTING_INDEX_SETTINGS, - auto_expand_replicas: false, - number_of_shards: 3000, - number_of_replicas: 1, - format: '3000', - }; - const result = createIndex(client, indexName, settings); - - return result.then(function () { - const payload = createSpy.getCall(0).args[1]; - sinon.assert.callCount(createSpy, 1); - expect(payload).to.have.property('index', indexName); - expect(payload).to.have.property('body'); - expect(payload.body).to.have.property('settings'); - expect(payload.body.settings).to.eql(settings); - expect(payload.body).to.have.property('mappings'); - expect(payload.body.mappings).to.have.property('properties'); - }); - }); - }); - - describe('Does exist', function () { - let client; - let createSpy; - - beforeEach(function () { - client = new ClientMock(); - sinon - .stub(client, 'callAsInternalUser') - .withArgs('indices.exists') - .callsFake(() => Promise.resolve(true)); - createSpy = client.callAsInternalUser.withArgs('indices.create'); - }); - - it('should return true', function () { - const indexName = 'test-index'; - const result = createIndex(client, indexName); - - return result.then((exists) => expect(exists).to.be(true)); - }); - - it('should not create the index', function () { - const indexName = 'test-index'; - const result = createIndex(client, indexName); - - return result.then(function () { - sinon.assert.callCount(createSpy, 0); - }); - }); - }); -}); diff --git a/x-pack/plugins/reporting/server/lib/esqueue/__tests__/helpers/index_timestamp.js b/x-pack/plugins/reporting/server/lib/esqueue/__tests__/helpers/index_timestamp.js deleted file mode 100644 index 71dc8a363e429..0000000000000 --- a/x-pack/plugins/reporting/server/lib/esqueue/__tests__/helpers/index_timestamp.js +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import expect from '@kbn/expect'; -import sinon from 'sinon'; -import moment from 'moment'; -import { constants } from '../../constants'; -import { indexTimestamp } from '../../helpers/index_timestamp'; - -const anchor = '2016-04-02T01:02:03.456'; // saturday - -describe('Index timestamp interval', function () { - describe('construction', function () { - it('should throw given an invalid interval', function () { - const init = () => indexTimestamp('bananas'); - expect(init).to.throwException(/invalid.+interval/i); - }); - }); - - describe('timestamps', function () { - let clock; - let separator; - - beforeEach(function () { - separator = constants.DEFAULT_SETTING_DATE_SEPARATOR; - clock = sinon.useFakeTimers(moment(anchor).valueOf()); - }); - - afterEach(function () { - clock.restore(); - }); - - describe('formats', function () { - it('should return the year', function () { - const timestamp = indexTimestamp('year'); - const str = `2016`; - expect(timestamp).to.equal(str); - }); - - it('should return the year and month', function () { - const timestamp = indexTimestamp('month'); - const str = `2016${separator}04`; - expect(timestamp).to.equal(str); - }); - - it('should return the year, month, and first day of the week', function () { - const timestamp = indexTimestamp('week'); - const str = `2016${separator}03${separator}27`; - expect(timestamp).to.equal(str); - }); - - it('should return the year, month, and day of the week', function () { - const timestamp = indexTimestamp('day'); - const str = `2016${separator}04${separator}02`; - expect(timestamp).to.equal(str); - }); - - it('should return the year, month, day and hour', function () { - const timestamp = indexTimestamp('hour'); - const str = `2016${separator}04${separator}02${separator}01`; - expect(timestamp).to.equal(str); - }); - - it('should return the year, month, day, hour and minute', function () { - const timestamp = indexTimestamp('minute'); - const str = `2016${separator}04${separator}02${separator}01${separator}02`; - expect(timestamp).to.equal(str); - }); - }); - - describe('date separator', function () { - it('should be customizable', function () { - const separators = ['-', '.', '_']; - separators.forEach((customSep) => { - const str = `2016${customSep}04${customSep}02${customSep}01${customSep}02`; - const timestamp = indexTimestamp('minute', customSep); - expect(timestamp).to.equal(str); - }); - }); - - it('should throw if a letter is used', function () { - const separators = ['a', 'B', 'YYYY']; - separators.forEach((customSep) => { - const fn = () => indexTimestamp('minute', customSep); - expect(fn).to.throwException(); - }); - }); - }); - }); -}); diff --git a/x-pack/plugins/reporting/server/lib/esqueue/__tests__/job.js b/x-pack/plugins/reporting/server/lib/esqueue/__tests__/job.js deleted file mode 100644 index 955eed8d65722..0000000000000 --- a/x-pack/plugins/reporting/server/lib/esqueue/__tests__/job.js +++ /dev/null @@ -1,420 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import events from 'events'; -import expect from '@kbn/expect'; -import sinon from 'sinon'; -import proxyquire from 'proxyquire'; -import { QueueMock } from './fixtures/queue'; -import { ClientMock } from './fixtures/legacy_elasticsearch'; -import { constants } from '../constants'; - -const createIndexMock = sinon.stub(); -const { Job } = proxyquire.noPreserveCache()('../job', { - './helpers/create_index': { createIndex: createIndexMock }, -}); - -const maxPriority = 20; -const minPriority = -20; -const defaultPriority = 10; -const defaultCreatedBy = false; - -function validateDoc(spy) { - sinon.assert.callCount(spy, 1); - const spyCall = spy.getCall(0); - return spyCall.args[1]; -} - -describe('Job Class', function () { - let mockQueue; - let client; - let index; - - let type; - let payload; - let options; - - beforeEach(function () { - createIndexMock.resetHistory(); - createIndexMock.returns(Promise.resolve('mock')); - index = 'test'; - - client = new ClientMock(); - mockQueue = new QueueMock(); - mockQueue.setClient(client); - }); - - it('should be an event emitter', function () { - const job = new Job(mockQueue, index, 'test', {}); - expect(job).to.be.an(events.EventEmitter); - }); - - describe('invalid construction', function () { - it('should throw with a missing type', function () { - const init = () => new Job(mockQueue, index); - expect(init).to.throwException(/type.+string/i); - }); - - it('should throw with an invalid type', function () { - const init = () => new Job(mockQueue, index, { 'not a string': true }); - expect(init).to.throwException(/type.+string/i); - }); - - it('should throw with an invalid payload', function () { - const init = () => new Job(mockQueue, index, 'type1', [1, 2, 3]); - expect(init).to.throwException(/plain.+object/i); - }); - - it(`should throw error if invalid maxAttempts`, function () { - const init = () => new Job(mockQueue, index, 'type1', { id: '123' }, { max_attempts: -1 }); - expect(init).to.throwException(/invalid.+max_attempts/i); - }); - }); - - describe('construction', function () { - let indexSpy; - beforeEach(function () { - type = 'type1'; - payload = { id: '123' }; - indexSpy = sinon.spy(client, 'callAsInternalUser').withArgs('index'); - }); - - it('should create the target index', function () { - const job = new Job(mockQueue, index, type, payload, options); - return job.ready.then(() => { - sinon.assert.calledOnce(createIndexMock); - const args = createIndexMock.getCall(0).args; - expect(args[0]).to.equal(client); - expect(args[1]).to.equal(index); - }); - }); - - it('should index the payload', function () { - const job = new Job(mockQueue, index, type, payload); - return job.ready.then(() => { - const indexArgs = validateDoc(indexSpy); - expect(indexArgs).to.have.property('index', index); - expect(indexArgs).to.have.property('body'); - expect(indexArgs.body).to.have.property('payload', payload); - }); - }); - - it('should index the job type', function () { - const job = new Job(mockQueue, index, type, payload); - return job.ready.then(() => { - const indexArgs = validateDoc(indexSpy); - expect(indexArgs).to.have.property('index', index); - expect(indexArgs).to.have.property('body'); - expect(indexArgs.body).to.have.property('jobtype', type); - }); - }); - - it('should set event creation time', function () { - const job = new Job(mockQueue, index, type, payload); - return job.ready.then(() => { - const indexArgs = validateDoc(indexSpy); - expect(indexArgs.body).to.have.property('created_at'); - }); - }); - - it('should refresh the index', function () { - const refreshSpy = client.callAsInternalUser.withArgs('indices.refresh'); - - const job = new Job(mockQueue, index, type, payload); - return job.ready.then(() => { - sinon.assert.calledOnce(refreshSpy); - const spyCall = refreshSpy.getCall(0); - expect(spyCall.args[1]).to.have.property('index', index); - }); - }); - - it('should emit the job information on success', function (done) { - const job = new Job(mockQueue, index, type, payload); - job.once(constants.EVENT_JOB_CREATED, (jobDoc) => { - try { - expect(jobDoc).to.have.property('id'); - expect(jobDoc).to.have.property('index'); - expect(jobDoc).to.have.property('_seq_no'); - expect(jobDoc).to.have.property('_primary_term'); - done(); - } catch (e) { - done(e); - } - }); - }); - - it('should emit error on index creation failure', function (done) { - const errMsg = 'test index creation failure'; - - createIndexMock.returns(Promise.reject(new Error(errMsg))); - const job = new Job(mockQueue, index, type, payload); - - job.once(constants.EVENT_JOB_CREATE_ERROR, (err) => { - try { - expect(err.message).to.equal(errMsg); - done(); - } catch (e) { - done(e); - } - }); - }); - - it('should emit error on client index failure', function (done) { - const errMsg = 'test document index failure'; - - client.callAsInternalUser.restore(); - sinon - .stub(client, 'callAsInternalUser') - .withArgs('index') - .callsFake(() => Promise.reject(new Error(errMsg))); - const job = new Job(mockQueue, index, type, payload); - - job.once(constants.EVENT_JOB_CREATE_ERROR, (err) => { - try { - expect(err.message).to.equal(errMsg); - done(); - } catch (e) { - done(e); - } - }); - }); - }); - - describe('event emitting', function () { - it('should trigger events on the queue instance', function (done) { - const eventName = 'test event'; - const payload1 = { - test: true, - deep: { object: 'ok' }, - }; - const payload2 = 'two'; - const payload3 = new Error('test error'); - - const job = new Job(mockQueue, index, type, payload, options); - - mockQueue.on(eventName, (...args) => { - try { - expect(args[0]).to.equal(payload1); - expect(args[1]).to.equal(payload2); - expect(args[2]).to.equal(payload3); - done(); - } catch (e) { - done(e); - } - }); - - job.emit(eventName, payload1, payload2, payload3); - }); - }); - - describe('default values', function () { - let indexSpy; - beforeEach(function () { - type = 'type1'; - payload = { id: '123' }; - indexSpy = sinon.spy(client, 'callAsInternalUser').withArgs('index'); - }); - - it('should set attempt count to 0', function () { - const job = new Job(mockQueue, index, type, payload); - return job.ready.then(() => { - const indexArgs = validateDoc(indexSpy); - expect(indexArgs.body).to.have.property('attempts', 0); - }); - }); - - it('should index default created_by value', function () { - const job = new Job(mockQueue, index, type, payload); - return job.ready.then(() => { - const indexArgs = validateDoc(indexSpy); - expect(indexArgs.body).to.have.property('created_by', defaultCreatedBy); - }); - }); - - it('should set an expired process_expiration time', function () { - const now = new Date().getTime(); - const job = new Job(mockQueue, index, type, payload); - return job.ready.then(() => { - const indexArgs = validateDoc(indexSpy); - expect(indexArgs.body).to.have.property('process_expiration'); - expect(indexArgs.body.process_expiration.getTime()).to.be.lessThan(now); - }); - }); - - it('should set status as pending', function () { - const job = new Job(mockQueue, index, type, payload); - return job.ready.then(() => { - const indexArgs = validateDoc(indexSpy); - expect(indexArgs.body).to.have.property('status', constants.JOB_STATUS_PENDING); - }); - }); - - it('should have a default priority of 10', function () { - const job = new Job(mockQueue, index, type, payload, options); - return job.ready.then(() => { - const indexArgs = validateDoc(indexSpy); - expect(indexArgs.body).to.have.property('priority', defaultPriority); - }); - }); - - it('should set a browser type', function () { - const job = new Job(mockQueue, index, type, payload); - return job.ready.then(() => { - const indexArgs = validateDoc(indexSpy); - expect(indexArgs.body).to.have.property('browser_type'); - }); - }); - }); - - describe('option passing', function () { - let indexSpy; - beforeEach(function () { - type = 'type1'; - payload = { id: '123' }; - options = { - timeout: 4567, - max_attempts: 9, - headers: { - authorization: 'Basic cXdlcnR5', - }, - }; - indexSpy = sinon.spy(client, 'callAsInternalUser').withArgs('index'); - }); - - it('should index the created_by value', function () { - const createdBy = 'user_identifier'; - const job = new Job(mockQueue, index, type, payload, { - created_by: createdBy, - ...options, - }); - return job.ready.then(() => { - const indexArgs = validateDoc(indexSpy); - expect(indexArgs.body).to.have.property('created_by', createdBy); - }); - }); - - it('should index timeout value from options', function () { - const job = new Job(mockQueue, index, type, payload, options); - return job.ready.then(() => { - const indexArgs = validateDoc(indexSpy); - expect(indexArgs.body).to.have.property('timeout', options.timeout); - }); - }); - - it('should set max attempt count', function () { - const job = new Job(mockQueue, index, type, payload, options); - return job.ready.then(() => { - const indexArgs = validateDoc(indexSpy); - expect(indexArgs.body).to.have.property('max_attempts', options.max_attempts); - }); - }); - - it('should add headers to the request params', function () { - const job = new Job(mockQueue, index, type, payload, options); - return job.ready.then(() => { - const indexArgs = validateDoc(indexSpy); - expect(indexArgs).to.have.property('headers', options.headers); - }); - }); - - it(`should use upper priority of ${maxPriority}`, function () { - const job = new Job(mockQueue, index, type, payload, { priority: maxPriority * 2 }); - return job.ready.then(() => { - const indexArgs = validateDoc(indexSpy); - expect(indexArgs.body).to.have.property('priority', maxPriority); - }); - }); - - it(`should use lower priority of ${minPriority}`, function () { - const job = new Job(mockQueue, index, type, payload, { priority: minPriority * 2 }); - return job.ready.then(() => { - const indexArgs = validateDoc(indexSpy); - expect(indexArgs.body).to.have.property('priority', minPriority); - }); - }); - }); - - describe('get method', function () { - beforeEach(function () { - type = 'type2'; - payload = { id: '123' }; - }); - - it('should return the job document', function () { - const job = new Job(mockQueue, index, type, payload); - - return job.get().then((doc) => { - const jobDoc = job.document; // document should be resolved - expect(doc).to.have.property('index', index); - expect(doc).to.have.property('id', jobDoc.id); - expect(doc).to.have.property('_seq_no', jobDoc._seq_no); - expect(doc).to.have.property('_primary_term', jobDoc._primary_term); - expect(doc).to.have.property('created_by', defaultCreatedBy); - - expect(doc).to.have.property('payload'); - expect(doc).to.have.property('jobtype'); - expect(doc).to.have.property('priority'); - expect(doc).to.have.property('timeout'); - }); - }); - - it('should contain optional data', function () { - const optionals = { - created_by: 'some_ident', - }; - - const job = new Job(mockQueue, index, type, payload, optionals); - return Promise.resolve(client.callAsInternalUser('get', {}, optionals)) - .then((doc) => { - sinon.stub(client, 'callAsInternalUser').withArgs('get').returns(Promise.resolve(doc)); - }) - .then(() => { - return job.get().then((doc) => { - expect(doc).to.have.property('created_by', optionals.created_by); - }); - }); - }); - }); - - describe('toJSON method', function () { - beforeEach(function () { - type = 'type2'; - payload = { id: '123' }; - options = { - timeout: 4567, - max_attempts: 9, - priority: 8, - }; - }); - - it('should return the static information about the job', function () { - const job = new Job(mockQueue, index, type, payload, options); - - // toJSON is sync, should work before doc is written to elasticsearch - expect(job.document).to.be(undefined); - - const doc = job.toJSON(); - expect(doc).to.have.property('index', index); - expect(doc).to.have.property('jobtype', type); - expect(doc).to.have.property('created_by', defaultCreatedBy); - expect(doc).to.have.property('timeout', options.timeout); - expect(doc).to.have.property('max_attempts', options.max_attempts); - expect(doc).to.have.property('priority', options.priority); - expect(doc).to.have.property('id'); - expect(doc).to.not.have.property('version'); - }); - - it('should contain optional data', function () { - const optionals = { - created_by: 'some_ident', - }; - - const job = new Job(mockQueue, index, type, payload, optionals); - const doc = job.toJSON(); - expect(doc).to.have.property('created_by', optionals.created_by); - }); - }); -}); diff --git a/x-pack/plugins/reporting/server/lib/esqueue/index.js b/x-pack/plugins/reporting/server/lib/esqueue/index.js index 735d19f8f6c47..0fbcb54c673dd 100644 --- a/x-pack/plugins/reporting/server/lib/esqueue/index.js +++ b/x-pack/plugins/reporting/server/lib/esqueue/index.js @@ -5,20 +5,17 @@ */ import { EventEmitter } from 'events'; -import { Job } from './job'; import { Worker } from './worker'; import { constants } from './constants'; -import { indexTimestamp } from './helpers/index_timestamp'; import { omit } from 'lodash'; export { events } from './constants/events'; export class Esqueue extends EventEmitter { - constructor(index, options = {}) { - if (!index) throw new Error('Must specify an index to write to'); - + constructor(store, options = {}) { super(); - this.index = index; + this.store = store; // for updating jobs in ES + this.index = this.store.indexPrefix; // for polling for pending jobs this.settings = { interval: constants.DEFAULT_SETTING_INTERVAL, timeout: constants.DEFAULT_SETTING_TIMEOUT, @@ -40,21 +37,6 @@ export class Esqueue extends EventEmitter { }); } - addJob(jobtype, payload, opts = {}) { - const timestamp = indexTimestamp(this.settings.interval, this.settings.dateSeparator); - const index = `${this.index}-${timestamp}`; - const defaults = { - timeout: this.settings.timeout, - }; - - const options = Object.assign(defaults, opts, { - indexSettings: this.settings.indexSettings, - logger: this._logger, - }); - - return new Job(this, index, jobtype, payload, options); - } - registerWorker(type, workerFn, opts) { const worker = new Worker(this, type, workerFn, { ...opts, logger: this._logger }); this._workers.push(worker); diff --git a/x-pack/plugins/reporting/server/lib/esqueue/job.js b/x-pack/plugins/reporting/server/lib/esqueue/job.js deleted file mode 100644 index 776ec64103e42..0000000000000 --- a/x-pack/plugins/reporting/server/lib/esqueue/job.js +++ /dev/null @@ -1,142 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import events from 'events'; -import Puid from 'puid'; -import { constants } from './constants'; -import { createIndex } from './helpers/create_index'; -import { isPlainObject } from 'lodash'; - -const puid = new Puid(); - -export class Job extends events.EventEmitter { - constructor(queue, index, jobtype, payload, options = {}) { - if (typeof jobtype !== 'string') throw new Error('Jobtype must be a string'); - if (!isPlainObject(payload)) throw new Error('Payload must be a plain object'); - - super(); - - this.queue = queue; - this._client = this.queue.client; - this.id = puid.generate(); - this.index = index; - this.jobtype = jobtype; - this.payload = payload; - this.created_by = options.created_by || false; - this.timeout = options.timeout || 10000; - this.maxAttempts = options.max_attempts || 3; - this.priority = Math.max(Math.min(options.priority || 10, 20), -20); - this.indexSettings = options.indexSettings || {}; - this.browser_type = options.browser_type; - - if (typeof this.maxAttempts !== 'number' || this.maxAttempts < 1) { - throw new Error(`Invalid max_attempts: ${this.maxAttempts}`); - } - - this.debug = (msg, err) => { - const logger = options.logger || function () {}; - const message = `${this.id} - ${msg}`; - const tags = ['debug']; - - if (err) { - logger(`${message}: ${err}`, tags); - return; - } - - logger(message, tags); - }; - - const indexParams = { - index: this.index, - id: this.id, - body: { - jobtype: this.jobtype, - meta: { - // We are copying these values out of payload because these fields are indexed and can be aggregated on - // for tracking stats, while payload contents are not. - objectType: payload.type, - layout: payload.layout ? payload.layout.id : 'none', - }, - payload: this.payload, - priority: this.priority, - created_by: this.created_by, - timeout: this.timeout, - process_expiration: new Date(0), // use epoch so the job query works - created_at: new Date(), - attempts: 0, - max_attempts: this.maxAttempts, - status: constants.JOB_STATUS_PENDING, - browser_type: this.browser_type, - }, - }; - - if (options.headers) { - indexParams.headers = options.headers; - } - - this.ready = createIndex(this._client, this.index, this.indexSettings) - .then(() => this._client.callAsInternalUser('index', indexParams)) - .then((doc) => { - this.document = { - id: doc._id, - index: doc._index, - _seq_no: doc._seq_no, - _primary_term: doc._primary_term, - }; - this.debug(`Job created in index ${this.index}`); - - return this._client - .callAsInternalUser('indices.refresh', { - index: this.index, - }) - .then(() => { - this.debug(`Job index refreshed ${this.index}`); - this.emit(constants.EVENT_JOB_CREATED, this.document); - }); - }) - .catch((err) => { - this.debug('Job creation failed', err); - this.emit(constants.EVENT_JOB_CREATE_ERROR, err); - }); - } - - emit(name, ...args) { - super.emit(name, ...args); - this.queue.emit(name, ...args); - } - - get() { - return this.ready - .then(() => { - return this._client.callAsInternalUser('get', { - index: this.index, - id: this.id, - }); - }) - .then((doc) => { - return Object.assign(doc._source, { - index: doc._index, - id: doc._id, - _seq_no: doc._seq_no, - _primary_term: doc._primary_term, - }); - }); - } - - toJSON() { - return { - id: this.id, - index: this.index, - jobtype: this.jobtype, - created_by: this.created_by, - payload: this.payload, - timeout: this.timeout, - max_attempts: this.maxAttempts, - priority: this.priority, - browser_type: this.browser_type, - }; - } -} diff --git a/x-pack/plugins/reporting/server/lib/esqueue/worker.js b/x-pack/plugins/reporting/server/lib/esqueue/worker.js index ab462d8c6e7c3..f24cb6cd55307 100644 --- a/x-pack/plugins/reporting/server/lib/esqueue/worker.js +++ b/x-pack/plugins/reporting/server/lib/esqueue/worker.js @@ -158,8 +158,8 @@ export class Worker extends events.EventEmitter { kibana_name: this.kibanaName, }; - return this._client - .callAsInternalUser('update', { + return this.queue.store + .updateReport({ index: job._index, id: job._id, if_seq_no: job._seq_no, @@ -197,8 +197,8 @@ export class Worker extends events.EventEmitter { output: docOutput, }); - return this._client - .callAsInternalUser('update', { + return this.queue.store + .updateReport({ index: job._index, id: job._id, if_seq_no: job._seq_no, @@ -294,8 +294,8 @@ export class Worker extends events.EventEmitter { output: docOutput, }; - return this._client - .callAsInternalUser('update', { + return this.queue.store + .updateReport({ index: job._index, id: job._id, if_seq_no: job._seq_no, diff --git a/x-pack/plugins/reporting/server/lib/index.ts b/x-pack/plugins/reporting/server/lib/index.ts index 0e9c49b170887..f5a50fca28b7a 100644 --- a/x-pack/plugins/reporting/server/lib/index.ts +++ b/x-pack/plugins/reporting/server/lib/index.ts @@ -12,3 +12,4 @@ export { enqueueJobFactory } from './enqueue_job'; export { getExportTypesRegistry } from './export_types_registry'; export { runValidations } from './validate'; export { startTrace } from './trace'; +export { ReportingStore } from './store'; diff --git a/x-pack/plugins/reporting/server/lib/store/index.ts b/x-pack/plugins/reporting/server/lib/store/index.ts new file mode 100644 index 0000000000000..a88d36d3fdf9a --- /dev/null +++ b/x-pack/plugins/reporting/server/lib/store/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export { Report } from './report'; +export { ReportingStore } from './store'; diff --git a/x-pack/plugins/reporting/server/lib/esqueue/helpers/index_timestamp.js b/x-pack/plugins/reporting/server/lib/store/index_timestamp.ts similarity index 80% rename from x-pack/plugins/reporting/server/lib/esqueue/helpers/index_timestamp.js rename to x-pack/plugins/reporting/server/lib/store/index_timestamp.ts index ceb4ef43b2d9d..71ce0b1e572f8 100644 --- a/x-pack/plugins/reporting/server/lib/esqueue/helpers/index_timestamp.js +++ b/x-pack/plugins/reporting/server/lib/store/index_timestamp.ts @@ -4,19 +4,20 @@ * you may not use this file except in compliance with the Elastic License. */ -import moment from 'moment'; +import moment, { unitOfTime } from 'moment'; export const intervals = ['year', 'month', 'week', 'day', 'hour', 'minute']; // TODO: This helper function can be removed by using `schema.duration` objects in the reporting config schema -export function indexTimestamp(intervalStr, separator = '-') { +export function indexTimestamp(intervalStr: string, separator = '-') { + const startOf = intervalStr as unitOfTime.StartOf; if (separator.match(/[a-z]/i)) throw new Error('Interval separator can not be a letter'); const index = intervals.indexOf(intervalStr); - if (index === -1) throw new Error('Invalid index interval: ', intervalStr); + if (index === -1) throw new Error('Invalid index interval: ' + intervalStr); const m = moment(); - m.startOf(intervalStr); + m.startOf(startOf); let dateString; switch (intervalStr) { diff --git a/x-pack/plugins/reporting/server/lib/store/mapping.ts b/x-pack/plugins/reporting/server/lib/store/mapping.ts new file mode 100644 index 0000000000000..a819923e2f105 --- /dev/null +++ b/x-pack/plugins/reporting/server/lib/store/mapping.ts @@ -0,0 +1,65 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export const mapping = { + meta: { + // We are indexing these properties with both text and keyword fields because that's what will be auto generated + // when an index already exists. This schema is only used when a reporting index doesn't exist. This way existing + // reporting indexes and new reporting indexes will look the same and the data can be queried in the same + // manner. + properties: { + /** + * Type of object that is triggering this report. Should be either search, visualization or dashboard. + * Used for job listing and telemetry stats only. + */ + objectType: { + type: 'text', + fields: { + keyword: { + type: 'keyword', + ignore_above: 256, + }, + }, + }, + /** + * Can be either preserve_layout, print or none (in the case of csv export). + * Used for phone home stats only. + */ + layout: { + type: 'text', + fields: { + keyword: { + type: 'keyword', + ignore_above: 256, + }, + }, + }, + }, + }, + browser_type: { type: 'keyword' }, + jobtype: { type: 'keyword' }, + payload: { type: 'object', enabled: false }, + priority: { type: 'byte' }, + timeout: { type: 'long' }, + process_expiration: { type: 'date' }, + created_by: { type: 'keyword' }, + created_at: { type: 'date' }, + started_at: { type: 'date' }, + completed_at: { type: 'date' }, + attempts: { type: 'short' }, + max_attempts: { type: 'short' }, + kibana_name: { type: 'keyword' }, + kibana_id: { type: 'keyword' }, + status: { type: 'keyword' }, + output: { + type: 'object', + properties: { + content_type: { type: 'keyword' }, + size: { type: 'long' }, + content: { type: 'object', enabled: false }, + }, + }, +}; diff --git a/x-pack/plugins/reporting/server/lib/store/report.test.ts b/x-pack/plugins/reporting/server/lib/store/report.test.ts new file mode 100644 index 0000000000000..83444494e61d3 --- /dev/null +++ b/x-pack/plugins/reporting/server/lib/store/report.test.ts @@ -0,0 +1,77 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { Report } from './report'; + +describe('Class Report', () => { + it('constructs Report instance', () => { + const opts = { + index: '.reporting-test-index-12345', + jobtype: 'test-report', + created_by: 'created_by_test_string', + browser_type: 'browser_type_test_string', + max_attempts: 50, + payload: { payload_test_field: 1 }, + timeout: 30000, + priority: 1, + }; + const report = new Report(opts); + expect(report.toJSON()).toMatchObject({ + _primary_term: undefined, + _seq_no: undefined, + browser_type: 'browser_type_test_string', + created_by: 'created_by_test_string', + jobtype: 'test-report', + max_attempts: 50, + payload: { + payload_test_field: 1, + }, + priority: 1, + timeout: 30000, + }); + + expect(report.id).toBeDefined(); + }); + + it('updateWithDoc method syncs takes fields to sync ES metadata', () => { + const opts = { + index: '.reporting-test-index-12345', + jobtype: 'test-report', + created_by: 'created_by_test_string', + browser_type: 'browser_type_test_string', + max_attempts: 50, + payload: { payload_test_field: 1 }, + timeout: 30000, + priority: 1, + }; + const report = new Report(opts); + + const metadata = { + _index: '.reporting-test-update', + _id: '12342p9o387549o2345', + _primary_term: 77, + _seq_no: 99, + }; + report.updateWithDoc(metadata); + + expect(report.toJSON()).toMatchObject({ + index: '.reporting-test-update', + _primary_term: 77, + _seq_no: 99, + browser_type: 'browser_type_test_string', + created_by: 'created_by_test_string', + jobtype: 'test-report', + max_attempts: 50, + payload: { + payload_test_field: 1, + }, + priority: 1, + timeout: 30000, + }); + + expect(report._id).toBe('12342p9o387549o2345'); + }); +}); diff --git a/x-pack/plugins/reporting/server/lib/store/report.ts b/x-pack/plugins/reporting/server/lib/store/report.ts new file mode 100644 index 0000000000000..cc9967e64b6eb --- /dev/null +++ b/x-pack/plugins/reporting/server/lib/store/report.ts @@ -0,0 +1,85 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +// @ts-ignore no module definition +import Puid from 'puid'; + +interface Payload { + id?: string; + index: string; + jobtype: string; + created_by: string | boolean; + payload: unknown; + browser_type: string; + priority: number; + max_attempts: number; + timeout: number; +} + +const puid = new Puid(); + +export class Report { + public readonly jobtype: string; + public readonly created_by: string | boolean; + public readonly payload: unknown; + public readonly browser_type: string; + public readonly id: string; + + public readonly priority: number; + // queue stuff, to be removed with Task Manager integration + public readonly max_attempts: number; + public readonly timeout: number; + + public _index: string; + public _id?: string; // set by ES + public _primary_term?: unknown; // set by ES + public _seq_no: unknown; // set by ES + + /* + * Create an unsaved report + */ + constructor(opts: Payload) { + this.jobtype = opts.jobtype; + this.created_by = opts.created_by; + this.payload = opts.payload; + this.browser_type = opts.browser_type; + this.priority = opts.priority; + this.max_attempts = opts.max_attempts; + this.timeout = opts.timeout; + this.id = puid.generate(); + + this._index = opts.index; + } + + /* + * Update the report with "live" storage metadata + */ + updateWithDoc(doc: Partial<Report>) { + if (doc._index) { + this._index = doc._index; // can not be undefined + } + + this._id = doc._id; + this._primary_term = doc._primary_term; + this._seq_no = doc._seq_no; + } + + toJSON() { + return { + id: this.id, + index: this._index, + _seq_no: this._seq_no, + _primary_term: this._primary_term, + jobtype: this.jobtype, + created_by: this.created_by, + payload: this.payload, + timeout: this.timeout, + max_attempts: this.max_attempts, + priority: this.priority, + browser_type: this.browser_type, + }; + } +} diff --git a/x-pack/plugins/reporting/server/lib/store/store.test.ts b/x-pack/plugins/reporting/server/lib/store/store.test.ts new file mode 100644 index 0000000000000..4868a1dfdd8f3 --- /dev/null +++ b/x-pack/plugins/reporting/server/lib/store/store.test.ts @@ -0,0 +1,166 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import sinon from 'sinon'; +import { ReportingConfig, ReportingCore } from '../..'; +import { createMockReportingCore } from '../../test_helpers'; +import { createMockLevelLogger } from '../../test_helpers/create_mock_levellogger'; +import { ReportingStore } from './store'; +import { ElasticsearchServiceSetup } from 'src/core/server'; + +const getMockConfig = (mockConfigGet: sinon.SinonStub) => ({ + get: mockConfigGet, + kbnConfig: { get: mockConfigGet }, +}); + +describe('ReportingStore', () => { + const mockLogger = createMockLevelLogger(); + let mockConfig: ReportingConfig; + let mockCore: ReportingCore; + + const callClusterStub = sinon.stub(); + const mockElasticsearch = { legacy: { client: { callAsInternalUser: callClusterStub } } }; + + beforeEach(async () => { + const mockConfigGet = sinon.stub(); + mockConfigGet.withArgs('index').returns('.reporting-test'); + mockConfigGet.withArgs('queue', 'indexInterval').returns('week'); + mockConfig = getMockConfig(mockConfigGet); + mockCore = await createMockReportingCore(mockConfig); + + callClusterStub.withArgs('indices.exists').resolves({}); + callClusterStub.withArgs('indices.create').resolves({}); + callClusterStub.withArgs('index').resolves({}); + callClusterStub.withArgs('indices.refresh').resolves({}); + callClusterStub.withArgs('update').resolves({}); + + mockCore.getElasticsearchService = () => + (mockElasticsearch as unknown) as ElasticsearchServiceSetup; + }); + + describe('addReport', () => { + it('returns Report object', async () => { + const store = new ReportingStore(mockCore, mockLogger); + const reportType = 'unknowntype'; + const reportPayload = {}; + const reportOptions = { + timeout: 10000, + created_by: 'created_by_string', + browser_type: 'browser_type_string', + max_attempts: 1, + }; + await expect( + store.addReport(reportType, reportPayload, reportOptions) + ).resolves.toMatchObject({ + _primary_term: undefined, + _seq_no: undefined, + browser_type: 'browser_type_string', + created_by: 'created_by_string', + jobtype: 'unknowntype', + max_attempts: 1, + payload: {}, + priority: 10, + timeout: 10000, + }); + }); + + it('throws if options has invalid indexInterval', async () => { + const mockConfigGet = sinon.stub(); + mockConfigGet.withArgs('index').returns('.reporting-test'); + mockConfigGet.withArgs('queue', 'indexInterval').returns('centurially'); + mockConfig = getMockConfig(mockConfigGet); + mockCore = await createMockReportingCore(mockConfig); + + const store = new ReportingStore(mockCore, mockLogger); + const reportType = 'unknowntype'; + const reportPayload = {}; + const reportOptions = { + timeout: 10000, + created_by: 'created_by_string', + browser_type: 'browser_type_string', + max_attempts: 1, + }; + expect( + store.addReport(reportType, reportPayload, reportOptions) + ).rejects.toMatchInlineSnapshot(`[Error: Invalid index interval: centurially]`); + }); + + it('handles error creating the index', async () => { + // setup + callClusterStub.withArgs('indices.exists').resolves(false); + callClusterStub.withArgs('indices.create').rejects(new Error('error')); + + const store = new ReportingStore(mockCore, mockLogger); + const reportType = 'unknowntype'; + const reportPayload = {}; + const reportOptions = { + timeout: 10000, + created_by: 'created_by_string', + browser_type: 'browser_type_string', + max_attempts: 1, + }; + await expect( + store.addReport(reportType, reportPayload, reportOptions) + ).rejects.toMatchInlineSnapshot(`[Error: error]`); + }); + + /* Creating the index will fail, if there were multiple jobs staged in + * parallel and creation completed from another Kibana instance. Only the + * first request in line can successfully create it. + * In spite of that race condition, adding the new job in Elasticsearch is + * fine. + */ + it('ignores index creation error if the index already exists and continues adding the report', async () => { + // setup + callClusterStub.withArgs('indices.exists').resolves(false); + callClusterStub.withArgs('indices.create').rejects(new Error('error')); + + const store = new ReportingStore(mockCore, mockLogger); + const reportType = 'unknowntype'; + const reportPayload = {}; + const reportOptions = { + timeout: 10000, + created_by: 'created_by_string', + browser_type: 'browser_type_string', + max_attempts: 1, + }; + await expect( + store.addReport(reportType, reportPayload, reportOptions) + ).rejects.toMatchInlineSnapshot(`[Error: error]`); + }); + + it('skips creating the index if already exists', async () => { + // setup + callClusterStub.withArgs('indices.exists').resolves(false); + callClusterStub + .withArgs('indices.create') + .rejects(new Error('resource_already_exists_exception')); // will be triggered but ignored + + const store = new ReportingStore(mockCore, mockLogger); + const reportType = 'unknowntype'; + const reportPayload = {}; + const reportOptions = { + timeout: 10000, + created_by: 'created_by_string', + browser_type: 'browser_type_string', + max_attempts: 1, + }; + await expect( + store.addReport(reportType, reportPayload, reportOptions) + ).resolves.toMatchObject({ + _primary_term: undefined, + _seq_no: undefined, + browser_type: 'browser_type_string', + created_by: 'created_by_string', + jobtype: 'unknowntype', + max_attempts: 1, + payload: {}, + priority: 10, + timeout: 10000, + }); + }); + }); +}); diff --git a/x-pack/plugins/reporting/server/lib/store/store.ts b/x-pack/plugins/reporting/server/lib/store/store.ts new file mode 100644 index 0000000000000..5d8ab2bcb5261 --- /dev/null +++ b/x-pack/plugins/reporting/server/lib/store/store.ts @@ -0,0 +1,169 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { ElasticsearchServiceSetup } from 'src/core/server'; +import { LevelLogger } from '../'; +import { ReportingCore } from '../../'; +import { LayoutInstance } from '../../export_types/common/layouts'; +import { indexTimestamp } from './index_timestamp'; +import { mapping } from './mapping'; +import { Report } from './report'; + +export const statuses = { + JOB_STATUS_PENDING: 'pending', + JOB_STATUS_PROCESSING: 'processing', + JOB_STATUS_COMPLETED: 'completed', + JOB_STATUS_WARNINGS: 'completed_with_warnings', + JOB_STATUS_FAILED: 'failed', + JOB_STATUS_CANCELLED: 'cancelled', +}; + +interface AddReportOpts { + timeout: number; + created_by: string | boolean; + browser_type: string; + max_attempts: number; +} + +interface UpdateQuery { + index: string; + id: string; + if_seq_no: unknown; + if_primary_term: unknown; + body: { doc: Partial<Report> }; +} + +/* + * A class to give an interface to historical reports in the reporting.index + * - track the state: pending, processing, completed, etc + * - handle updates and deletes to the reporting document + * - interface for downloading the report + */ +export class ReportingStore { + public readonly indexPrefix: string; + public readonly indexInterval: string; + + private client: ElasticsearchServiceSetup['legacy']['client']; + private logger: LevelLogger; + + constructor(reporting: ReportingCore, logger: LevelLogger) { + const config = reporting.getConfig(); + const elasticsearch = reporting.getElasticsearchService(); + + this.client = elasticsearch.legacy.client; + this.indexPrefix = config.get('index'); + this.indexInterval = config.get('queue', 'indexInterval'); + + this.logger = logger; + } + + private async createIndex(indexName: string) { + return this.client + .callAsInternalUser('indices.exists', { + index: indexName, + }) + .then((exists) => { + if (exists) { + return exists; + } + + const indexSettings = { + number_of_shards: 1, + auto_expand_replicas: '0-1', + }; + const body = { + settings: indexSettings, + mappings: { + properties: mapping, + }, + }; + + return this.client + .callAsInternalUser('indices.create', { + index: indexName, + body, + }) + .then(() => true) + .catch((err: Error) => { + const isIndexExistsError = err.message.match(/resource_already_exists_exception/); + if (isIndexExistsError) { + // Do not fail a job if the job runner hits the race condition. + this.logger.warn(`Automatic index creation failed: index already exists: ${err}`); + return; + } + + throw err; + }); + }); + } + + private async saveReport(report: Report) { + const payload = report.payload as { objectType: string; layout: LayoutInstance }; + + const indexParams = { + index: report._index, + id: report.id, + body: { + jobtype: report.jobtype, + meta: { + // We are copying these values out of payload because these fields are indexed and can be aggregated on + // for tracking stats, while payload contents are not. + objectType: payload.type, + layout: payload.layout ? payload.layout.id : 'none', + }, + payload: report.payload, + created_by: report.created_by, + timeout: report.timeout, + process_expiration: new Date(0), // use epoch so the job query works + created_at: new Date(), + attempts: 0, + max_attempts: report.max_attempts, + status: statuses.JOB_STATUS_PENDING, + browser_type: report.browser_type, + }, + }; + return this.client.callAsInternalUser('index', indexParams); + } + + private async refreshIndex(index: string) { + return this.client.callAsInternalUser('indices.refresh', { index }); + } + + public async addReport(type: string, payload: unknown, options: AddReportOpts): Promise<Report> { + const timestamp = indexTimestamp(this.indexInterval); + const index = `${this.indexPrefix}-${timestamp}`; + await this.createIndex(index); + + const report = new Report({ + index, + payload, + jobtype: type, + created_by: options.created_by, + browser_type: options.browser_type, + max_attempts: options.max_attempts, + timeout: options.timeout, + priority: 10, // unused + }); + + const doc = await this.saveReport(report); + report.updateWithDoc(doc); + + await this.refreshIndex(index); + this.logger.info(`Successfully queued pending job: ${report._index}/${report.id}`); + + return report; + } + + public async updateReport(query: UpdateQuery): Promise<Report> { + return this.client.callAsInternalUser('update', { + index: query.index, + id: query.id, + if_seq_no: query.if_seq_no, + if_primary_term: query.if_primary_term, + body: { doc: query.body.doc }, + }); + } +} diff --git a/x-pack/plugins/reporting/server/plugin.ts b/x-pack/plugins/reporting/server/plugin.ts index 693b0917603fc..cedc9dc14a237 100644 --- a/x-pack/plugins/reporting/server/plugin.ts +++ b/x-pack/plugins/reporting/server/plugin.ts @@ -8,7 +8,13 @@ import { CoreSetup, CoreStart, Plugin, PluginInitializerContext } from 'src/core import { ReportingCore } from './'; import { initializeBrowserDriverFactory } from './browsers'; import { buildConfig, ReportingConfigType } from './config'; -import { createQueueFactory, enqueueJobFactory, LevelLogger, runValidations } from './lib'; +import { + createQueueFactory, + enqueueJobFactory, + LevelLogger, + runValidations, + ReportingStore, +} from './lib'; import { registerRoutes } from './routes'; import { setFieldFormats } from './services'; import { ReportingSetup, ReportingSetupDeps, ReportingStart, ReportingStartDeps } from './types'; @@ -86,9 +92,9 @@ export class ReportingPlugin const config = reportingCore.getConfig(); const browserDriverFactory = await initializeBrowserDriverFactory(config, logger); - - const esqueue = await createQueueFactory(reportingCore, logger); // starts polling for pending jobs - const enqueueJob = enqueueJobFactory(reportingCore, logger); // called from generation routes + const store = new ReportingStore(reportingCore, logger); + const esqueue = await createQueueFactory(reportingCore, store, logger); // starts polling for pending jobs + const enqueueJob = enqueueJobFactory(reportingCore, store, logger); // called from generation routes reportingCore.pluginStart({ browserDriverFactory, @@ -96,6 +102,7 @@ export class ReportingPlugin uiSettings: core.uiSettings, esqueue, enqueueJob, + store, }); // run self-check validations diff --git a/x-pack/plugins/reporting/server/test_helpers/create_mock_levellogger.ts b/x-pack/plugins/reporting/server/test_helpers/create_mock_levellogger.ts new file mode 100644 index 0000000000000..f5e9a44281cb6 --- /dev/null +++ b/x-pack/plugins/reporting/server/test_helpers/create_mock_levellogger.ts @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { LevelLogger } from '../lib'; + +export function createMockLevelLogger() { + // eslint-disable-next-line no-console + const consoleLogger = (tag: string) => (message: unknown) => console.log(tag, message); + const innerLogger = { + get: () => innerLogger, + debug: consoleLogger('debug'), + info: consoleLogger('info'), + warn: consoleLogger('warn'), + trace: consoleLogger('trace'), + error: consoleLogger('error'), + fatal: consoleLogger('fatal'), + log: consoleLogger('log'), + }; + return new LevelLogger(innerLogger); +} diff --git a/x-pack/plugins/reporting/server/test_helpers/create_mock_reportingplugin.ts b/x-pack/plugins/reporting/server/test_helpers/create_mock_reportingplugin.ts index 579035a46f615..427a6362a7258 100644 --- a/x-pack/plugins/reporting/server/test_helpers/create_mock_reportingplugin.ts +++ b/x-pack/plugins/reporting/server/test_helpers/create_mock_reportingplugin.ts @@ -20,6 +20,8 @@ import { } from '../browsers'; import { ReportingInternalSetup, ReportingInternalStart } from '../core'; import { ReportingStartDeps } from '../types'; +import { ReportingStore } from '../lib'; +import { createMockLevelLogger } from './create_mock_levellogger'; (initializeBrowserDriverFactory as jest.Mock< Promise<HeadlessChromiumDriverFactory> @@ -37,13 +39,19 @@ const createMockPluginSetup = (setupMock?: any): ReportingInternalSetup => { }; }; -const createMockPluginStart = (startMock?: any): ReportingInternalStart => { +const createMockPluginStart = ( + mockReportingCore: ReportingCore, + startMock?: any +): ReportingInternalStart => { + const logger = createMockLevelLogger(); + const store = new ReportingStore(mockReportingCore, logger); return { browserDriverFactory: startMock.browserDriverFactory, enqueueJob: startMock.enqueueJob, esqueue: startMock.esqueue, savedObjects: startMock.savedObjects || { getScopedClient: jest.fn() }, uiSettings: startMock.uiSettings || { asScopedToClient: () => ({ get: jest.fn() }) }, + store, }; }; @@ -60,9 +68,22 @@ export const createMockStartDeps = (startMock?: any): ReportingStartDeps => ({ export const createMockReportingCore = async ( config: ReportingConfig, - setupDepsMock: ReportingInternalSetup | undefined = createMockPluginSetup({}), - startDepsMock: ReportingInternalStart | undefined = createMockPluginStart({}) + setupDepsMock: ReportingInternalSetup | undefined = undefined, + startDepsMock: ReportingInternalStart | undefined = undefined ) => { + if (!setupDepsMock) { + setupDepsMock = createMockPluginSetup({}); + } + + const mockReportingCore = { + getConfig: () => config, + getElasticsearchService: () => setupDepsMock?.elasticsearch, + } as ReportingCore; + + if (!startDepsMock) { + startDepsMock = createMockPluginStart(mockReportingCore, {}); + } + config = config || {}; const core = new ReportingCore(); diff --git a/x-pack/test/reporting_api_integration/services.ts b/x-pack/test/reporting_api_integration/services.ts index dadb466d45982..85f5a98c69b2e 100644 --- a/x-pack/test/reporting_api_integration/services.ts +++ b/x-pack/test/reporting_api_integration/services.ts @@ -7,8 +7,7 @@ import expect from '@kbn/expect'; import * as Rx from 'rxjs'; import { filter, first, mapTo, switchMap, timeout } from 'rxjs/operators'; -// @ts-ignore no module definition -import { indexTimestamp } from '../../plugins/reporting/server/lib/esqueue/helpers/index_timestamp'; +import { indexTimestamp } from '../../plugins/reporting/server/lib/store/index_timestamp'; import { services as xpackServices } from '../functional/services'; import { FtrProviderContext } from './ftr_provider_context';