From 98c5713f27d7adbf6ecef43c6dd3f72cbe912088 Mon Sep 17 00:00:00 2001 From: James Gowdy Date: Fri, 9 Aug 2019 17:09:51 +0100 Subject: [PATCH] [ML] Adding job overrides to the module setup endpoint (#42946) (#43026) * [ML] Adding job overrides to the module setup endpoint * removing text code * throw error for incompatible data * changes based on review --- .../ml/common/util/__tests__/job_utils.js | 6 +- .../plugins/ml/common/util/job_utils.js | 2 +- .../models/data_recognizer/data_recognizer.js | 96 +++++++++++++++++++ .../plugins/ml/server/routes/modules.js | 10 +- 4 files changed, 109 insertions(+), 5 deletions(-) diff --git a/x-pack/legacy/plugins/ml/common/util/__tests__/job_utils.js b/x-pack/legacy/plugins/ml/common/util/__tests__/job_utils.js index ae450ab82f7ca..737dd62543166 100644 --- a/x-pack/legacy/plugins/ml/common/util/__tests__/job_utils.js +++ b/x-pack/legacy/plugins/ml/common/util/__tests__/job_utils.js @@ -521,12 +521,12 @@ describe('ML - job utils', () => { describe('prefixDatafeedId', () => { - it('returns datafeed-prefix-job"', () => { + it('returns datafeed-prefix-job from datafeed-job"', () => { expect(prefixDatafeedId('datafeed-job', 'prefix-')).to.be('datafeed-prefix-job'); }); - it('returns prefix-job"', () => { - expect(prefixDatafeedId('job', 'prefix-')).to.be('prefix-job'); + it('returns datafeed-prefix-job from job"', () => { + expect(prefixDatafeedId('job', 'prefix-')).to.be('datafeed-prefix-job'); }); }); diff --git a/x-pack/legacy/plugins/ml/common/util/job_utils.js b/x-pack/legacy/plugins/ml/common/util/job_utils.js index 03d55e9d824b2..b3d3e182ee426 100644 --- a/x-pack/legacy/plugins/ml/common/util/job_utils.js +++ b/x-pack/legacy/plugins/ml/common/util/job_utils.js @@ -239,7 +239,7 @@ export const ML_DATA_PREVIEW_COUNT = 10; export function prefixDatafeedId(datafeedId, prefix) { return (datafeedId.match(/^datafeed-/)) ? datafeedId.replace(/^datafeed-/, `datafeed-${prefix}`) : - `${prefix}${datafeedId}`; + `datafeed-${prefix}${datafeedId}`; } // Returns a name which is safe to use in elasticsearch aggregations for the supplied diff --git a/x-pack/legacy/plugins/ml/server/models/data_recognizer/data_recognizer.js b/x-pack/legacy/plugins/ml/server/models/data_recognizer/data_recognizer.js index 3878bc54327d6..98a8655645a26 100644 --- a/x-pack/legacy/plugins/ml/server/models/data_recognizer/data_recognizer.js +++ b/x-pack/legacy/plugins/ml/server/models/data_recognizer/data_recognizer.js @@ -8,6 +8,7 @@ import fs from 'fs'; import Boom from 'boom'; +import { merge } from 'lodash'; import { getLatestDataOrBucketTimestamp, prefixDatafeedId } from '../../../common/util/job_utils'; import { mlLog } from '../../client/log'; import { jobServiceProvider } from '../job_service'; @@ -261,6 +262,8 @@ export class DataRecognizer { startDatafeed, start, end, + jobOverrides, + datafeedOverrides, request ) { @@ -300,6 +303,9 @@ export class DataRecognizer { datafeeds: [], savedObjects: [] }; + + this.applyJobConfigOverrides(moduleConfig, jobOverrides, jobPrefix); + this.applyDatafeedConfigOverrides(moduleConfig, datafeedOverrides, jobPrefix); this.updateDatafeedIndices(moduleConfig); this.updateJobUrlIndexPatterns(moduleConfig); @@ -754,4 +760,94 @@ export class DataRecognizer { return false; } + applyJobConfigOverrides(moduleConfig, jobOverrides, jobPrefix = '') { + if(jobOverrides !== undefined && jobOverrides !== null) { + if (typeof jobOverrides !== 'object') { + throw Boom.badRequest( + `Incompatible jobOverrides type (${typeof jobOverrides}). It needs to be an object or array of objects.` + ); + } + + // jobOverrides could be a single object or an array of objects. + // if single, convert to an array + const overrides = Array.isArray(jobOverrides) ? jobOverrides : [jobOverrides]; + const { jobs } = moduleConfig; + + // separate all the overrides. + // the overrides which don't contain a job id will be applied to all jobs in the module + const generalOverrides = []; + const jobSpecificOverrides = []; + overrides.forEach(o => { + if (o.job_id === undefined) { + generalOverrides.push(o); + } else { + jobSpecificOverrides.push(o); + } + }); + + generalOverrides.forEach(o => { + jobs.forEach(({ config }) => merge(config, o)); + }); + + jobSpecificOverrides.forEach(o => { + // for each override, find the relevant job. + // note, the job id already has the prefix prepended to it + const job = jobs.find(j => j.id === `${jobPrefix}${o.job_id}`); + if (job !== undefined) { + // delete the job_id in the override as this shouldn't be overridden + delete o.job_id; + merge(job.config, o); + } + }); + } + } + + applyDatafeedConfigOverrides(moduleConfig, datafeedOverrides, jobPrefix = '') { + if(datafeedOverrides !== undefined && datafeedOverrides !== null) { + if (typeof datafeedOverrides !== 'object') { + throw Boom.badRequest( + `Incompatible datafeedOverrides type (${typeof datafeedOverrides}). It needs to be an object or array of objects.` + ); + } + + // jobOverrides could be a single object or an array of objects. + // if single, convert to an array + const overrides = Array.isArray(datafeedOverrides) ? datafeedOverrides : [datafeedOverrides]; + const { datafeeds } = moduleConfig; + + // separate all the overrides. + // the overrides which don't contain a datafeed id or a job id will be applied to all jobs in the module + const generalOverrides = []; + const datafeedSpecificOverrides = []; + overrides.forEach(o => { + if (o.datafeed_id === undefined && o.job_id === undefined) { + generalOverrides.push(o); + } else { + datafeedSpecificOverrides.push(o); + } + }); + + generalOverrides.forEach(o => { + datafeeds.forEach(({ config }) => { + merge(config, o); + }); + }); + + // collect all the overrides which contain either a job id or a datafeed id + datafeedSpecificOverrides.forEach(o => { + // either a job id or datafeed id has been specified, so create a new id + // containing either one plus the prefix + const tempId = o.datafeed_id !== undefined ? o.datafeed_id : o.job_id; + const dId = prefixDatafeedId(tempId, jobPrefix); + + const datafeed = datafeeds.find(d => d.id === dId); + if (datafeed !== undefined) { + delete o.job_id; + delete o.datafeed_id; + merge(datafeed.config, o); + } + }); + } + } + } diff --git a/x-pack/legacy/plugins/ml/server/routes/modules.js b/x-pack/legacy/plugins/ml/server/routes/modules.js index 85d6ab581b970..c4b0c41f1cf11 100644 --- a/x-pack/legacy/plugins/ml/server/routes/modules.js +++ b/x-pack/legacy/plugins/ml/server/routes/modules.js @@ -36,6 +36,8 @@ function saveModuleItems( startDatafeed, start, end, + jobOverrides, + datafeedOverrides, request ) { const dr = new DataRecognizer(callWithRequest); @@ -49,6 +51,8 @@ function saveModuleItems( startDatafeed, start, end, + jobOverrides, + datafeedOverrides, request); } @@ -107,7 +111,9 @@ export function dataRecognizer({ commonRouteConfig, elasticsearchPlugin, route } useDedicatedIndex, startDatafeed, start, - end + end, + jobOverrides, + datafeedOverrides, } = request.payload; return saveModuleItems( @@ -121,6 +127,8 @@ export function dataRecognizer({ commonRouteConfig, elasticsearchPlugin, route } startDatafeed, start, end, + jobOverrides, + datafeedOverrides, request ) .catch(resp => wrapError(resp));