From 663af72004c709b4fc7d5cb0d87e8de1a50552db Mon Sep 17 00:00:00 2001 From: James Gowdy Date: Thu, 8 Aug 2019 16:21:08 +0100 Subject: [PATCH 1/4] [ML] Adding job overrides to the module setup endpoint --- .../ml/common/util/__tests__/job_utils.js | 6 +- .../plugins/ml/common/util/job_utils.js | 2 +- .../models/data_recognizer/data_recognizer.js | 73 ++++++++++++++++++- .../plugins/ml/server/routes/modules.js | 10 ++- 4 files changed, 84 insertions(+), 7 deletions(-) diff --git a/x-pack/legacy/plugins/ml/common/util/__tests__/job_utils.js b/x-pack/legacy/plugins/ml/common/util/__tests__/job_utils.js index ae450ab82f7ca..737dd62543166 100644 --- a/x-pack/legacy/plugins/ml/common/util/__tests__/job_utils.js +++ b/x-pack/legacy/plugins/ml/common/util/__tests__/job_utils.js @@ -521,12 +521,12 @@ describe('ML - job utils', () => { describe('prefixDatafeedId', () => { - it('returns datafeed-prefix-job"', () => { + it('returns datafeed-prefix-job from datafeed-job"', () => { expect(prefixDatafeedId('datafeed-job', 'prefix-')).to.be('datafeed-prefix-job'); }); - it('returns prefix-job"', () => { - expect(prefixDatafeedId('job', 'prefix-')).to.be('prefix-job'); + it('returns datafeed-prefix-job from job"', () => { + expect(prefixDatafeedId('job', 'prefix-')).to.be('datafeed-prefix-job'); }); }); diff --git a/x-pack/legacy/plugins/ml/common/util/job_utils.js b/x-pack/legacy/plugins/ml/common/util/job_utils.js index 03d55e9d824b2..b3d3e182ee426 100644 --- a/x-pack/legacy/plugins/ml/common/util/job_utils.js +++ b/x-pack/legacy/plugins/ml/common/util/job_utils.js @@ -239,7 +239,7 @@ export const ML_DATA_PREVIEW_COUNT = 10; export function prefixDatafeedId(datafeedId, prefix) { return (datafeedId.match(/^datafeed-/)) ? datafeedId.replace(/^datafeed-/, `datafeed-${prefix}`) : - `${prefix}${datafeedId}`; + `datafeed-${prefix}${datafeedId}`; } // Returns a name which is safe to use in elasticsearch aggregations for the supplied diff --git a/x-pack/legacy/plugins/ml/server/models/data_recognizer/data_recognizer.js b/x-pack/legacy/plugins/ml/server/models/data_recognizer/data_recognizer.js index 3878bc54327d6..f7a1d6dfd5644 100644 --- a/x-pack/legacy/plugins/ml/server/models/data_recognizer/data_recognizer.js +++ b/x-pack/legacy/plugins/ml/server/models/data_recognizer/data_recognizer.js @@ -8,6 +8,7 @@ import fs from 'fs'; import Boom from 'boom'; +import { merge } from 'lodash'; import { getLatestDataOrBucketTimestamp, prefixDatafeedId } from '../../../common/util/job_utils'; import { mlLog } from '../../client/log'; import { jobServiceProvider } from '../job_service'; @@ -261,6 +262,8 @@ export class DataRecognizer { startDatafeed, start, end, + jobOverrides, + datafeedOverrides, request ) { @@ -279,7 +282,7 @@ export class DataRecognizer { // the module's jobs contain custom URLs which require an index patten id // but there is no corresponding index pattern, throw an error - if (this.indexPatternId === undefined && this.doJobUrlsContainIndexPatternId(moduleConfig)) { + if (false && this.indexPatternId === undefined && this.doJobUrlsContainIndexPatternId(moduleConfig)) { throw Boom.badRequest( `Module's jobs contain custom URLs which require a kibana index pattern (${this.indexPatternName}) which cannot be found.` ); @@ -287,7 +290,7 @@ export class DataRecognizer { // the module's saved objects require an index patten id // but there is no corresponding index pattern, throw an error - if (this.indexPatternId === undefined && this.doSavedObjectsContainIndexPatternId(moduleConfig)) { + if (false && this.indexPatternId === undefined && this.doSavedObjectsContainIndexPatternId(moduleConfig)) { throw Boom.badRequest( `Module's saved objects contain custom URLs which require a kibana index pattern (${this.indexPatternName}) which cannot be found.` ); @@ -300,6 +303,9 @@ export class DataRecognizer { datafeeds: [], savedObjects: [] }; + + this.applyJobConfigOverrides(moduleConfig, jobOverrides, jobPrefix); + this.applyDatafeedConfigOverrides(moduleConfig, datafeedOverrides, jobPrefix); this.updateDatafeedIndices(moduleConfig); this.updateJobUrlIndexPatterns(moduleConfig); @@ -754,4 +760,67 @@ export class DataRecognizer { return false; } + applyJobConfigOverrides(moduleConfig, jobOverrides, jobPrefix = '') { + if(jobOverrides !== undefined) { + // jobOverrides could be a single object or an array of objects. + // if single, convert to an array + const overrides = Array.isArray(jobOverrides) ? jobOverrides : [jobOverrides]; + const { jobs } = moduleConfig; + + // first collect the overrides which don't contain a job id + // these will be applied to all jobs in the module + const generalOverrides = overrides.filter(o => o.job_id === undefined); + generalOverrides.forEach(o => { + jobs.forEach(({ config }) => merge(config, o)); + }); + + // collect all job specific overrides + const jobSpecificOverrides = overrides.filter(o => o.job_id !== undefined); + jobSpecificOverrides.forEach(o => { + // for each override, find the relevant job. + // note, the job id already has the prefix prepended to it + const job = jobs.find(j => j.id === `${jobPrefix}${o.job_id}`); + if (job !== undefined) { + // delete the job_id in the override as this shouldn't be overridden + delete o.job_id; + merge(job.config, o); + } + }); + } + } + + applyDatafeedConfigOverrides(moduleConfig, datafeedOverrides, jobPrefix = '') { + if(datafeedOverrides !== undefined) { + // jobOverrides could be a single object or an array of objects. + // if single, convert to an array + const overrides = Array.isArray(datafeedOverrides) ? datafeedOverrides : [datafeedOverrides]; + const { datafeeds } = moduleConfig; + + // first collect the overrides which don't contain a datafeed id or a job id + // these will be applied to all jobs in the module + const generalOverrides = overrides.filter(o => o.datafeed_id === undefined && o.job_id === undefined); + generalOverrides.forEach(o => { + datafeeds.forEach(({ config }) => { + merge(config, o); + }); + }); + + // collect all the overrides which contain either a job id or a datafeed id + const datafeedSpecificOverrides = overrides.filter(o => (o.datafeed_id !== undefined || o.job_id !== undefined)); + datafeedSpecificOverrides.forEach(o => { + // either a job id or datafeed id has been specified, so create a new id + // containing either one plus the prefix + const tempId = o.datafeed_id !== undefined ? o.datafeed_id : o.job_id; + const dId = prefixDatafeedId(tempId, jobPrefix); + + const datafeed = datafeeds.find(d => d.id === dId); + if (datafeed !== undefined) { + delete o.job_id; + delete o.datafeed_id; + merge(datafeed.config, o); + } + }); + } + } + } diff --git a/x-pack/legacy/plugins/ml/server/routes/modules.js b/x-pack/legacy/plugins/ml/server/routes/modules.js index 85d6ab581b970..c4b0c41f1cf11 100644 --- a/x-pack/legacy/plugins/ml/server/routes/modules.js +++ b/x-pack/legacy/plugins/ml/server/routes/modules.js @@ -36,6 +36,8 @@ function saveModuleItems( startDatafeed, start, end, + jobOverrides, + datafeedOverrides, request ) { const dr = new DataRecognizer(callWithRequest); @@ -49,6 +51,8 @@ function saveModuleItems( startDatafeed, start, end, + jobOverrides, + datafeedOverrides, request); } @@ -107,7 +111,9 @@ export function dataRecognizer({ commonRouteConfig, elasticsearchPlugin, route } useDedicatedIndex, startDatafeed, start, - end + end, + jobOverrides, + datafeedOverrides, } = request.payload; return saveModuleItems( @@ -121,6 +127,8 @@ export function dataRecognizer({ commonRouteConfig, elasticsearchPlugin, route } startDatafeed, start, end, + jobOverrides, + datafeedOverrides, request ) .catch(resp => wrapError(resp)); From 9912a7e28703ae965bf15ebfcc9ef663903ddf9e Mon Sep 17 00:00:00 2001 From: James Gowdy Date: Thu, 8 Aug 2019 16:23:03 +0100 Subject: [PATCH 2/4] removing text code --- .../ml/server/models/data_recognizer/data_recognizer.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/legacy/plugins/ml/server/models/data_recognizer/data_recognizer.js b/x-pack/legacy/plugins/ml/server/models/data_recognizer/data_recognizer.js index f7a1d6dfd5644..aad626eae9fe6 100644 --- a/x-pack/legacy/plugins/ml/server/models/data_recognizer/data_recognizer.js +++ b/x-pack/legacy/plugins/ml/server/models/data_recognizer/data_recognizer.js @@ -282,7 +282,7 @@ export class DataRecognizer { // the module's jobs contain custom URLs which require an index patten id // but there is no corresponding index pattern, throw an error - if (false && this.indexPatternId === undefined && this.doJobUrlsContainIndexPatternId(moduleConfig)) { + if (this.indexPatternId === undefined && this.doJobUrlsContainIndexPatternId(moduleConfig)) { throw Boom.badRequest( `Module's jobs contain custom URLs which require a kibana index pattern (${this.indexPatternName}) which cannot be found.` ); @@ -290,7 +290,7 @@ export class DataRecognizer { // the module's saved objects require an index patten id // but there is no corresponding index pattern, throw an error - if (false && this.indexPatternId === undefined && this.doSavedObjectsContainIndexPatternId(moduleConfig)) { + if (this.indexPatternId === undefined && this.doSavedObjectsContainIndexPatternId(moduleConfig)) { throw Boom.badRequest( `Module's saved objects contain custom URLs which require a kibana index pattern (${this.indexPatternName}) which cannot be found.` ); From 311ef39d4343b4927d3825b846023d1e345fe580 Mon Sep 17 00:00:00 2001 From: James Gowdy Date: Thu, 8 Aug 2019 20:56:58 +0100 Subject: [PATCH 3/4] throw error for incompatible data --- .../server/models/data_recognizer/data_recognizer.js | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/x-pack/legacy/plugins/ml/server/models/data_recognizer/data_recognizer.js b/x-pack/legacy/plugins/ml/server/models/data_recognizer/data_recognizer.js index aad626eae9fe6..863e1a4375a5b 100644 --- a/x-pack/legacy/plugins/ml/server/models/data_recognizer/data_recognizer.js +++ b/x-pack/legacy/plugins/ml/server/models/data_recognizer/data_recognizer.js @@ -762,6 +762,12 @@ export class DataRecognizer { applyJobConfigOverrides(moduleConfig, jobOverrides, jobPrefix = '') { if(jobOverrides !== undefined) { + if (typeof jobOverrides !== 'object') { + throw Boom.badRequest( + `Incompatible jobOverrides type (${typeof jobOverrides}). It needs to be an object or array of objects.` + ); + } + // jobOverrides could be a single object or an array of objects. // if single, convert to an array const overrides = Array.isArray(jobOverrides) ? jobOverrides : [jobOverrides]; @@ -791,6 +797,12 @@ export class DataRecognizer { applyDatafeedConfigOverrides(moduleConfig, datafeedOverrides, jobPrefix = '') { if(datafeedOverrides !== undefined) { + if (typeof datafeedOverrides !== 'object') { + throw Boom.badRequest( + `Incompatible datafeedOverrides type (${typeof datafeedOverrides}). It needs to be an object or array of objects.` + ); + } + // jobOverrides could be a single object or an array of objects. // if single, convert to an array const overrides = Array.isArray(datafeedOverrides) ? datafeedOverrides : [datafeedOverrides]; From 4b6cab68f5fc3615591e27356450350ef1df4881 Mon Sep 17 00:00:00 2001 From: James Gowdy Date: Fri, 9 Aug 2019 08:44:53 +0100 Subject: [PATCH 4/4] changes based on review --- .../models/data_recognizer/data_recognizer.js | 37 +++++++++++++------ 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/x-pack/legacy/plugins/ml/server/models/data_recognizer/data_recognizer.js b/x-pack/legacy/plugins/ml/server/models/data_recognizer/data_recognizer.js index 863e1a4375a5b..98a8655645a26 100644 --- a/x-pack/legacy/plugins/ml/server/models/data_recognizer/data_recognizer.js +++ b/x-pack/legacy/plugins/ml/server/models/data_recognizer/data_recognizer.js @@ -761,7 +761,7 @@ export class DataRecognizer { } applyJobConfigOverrides(moduleConfig, jobOverrides, jobPrefix = '') { - if(jobOverrides !== undefined) { + if(jobOverrides !== undefined && jobOverrides !== null) { if (typeof jobOverrides !== 'object') { throw Boom.badRequest( `Incompatible jobOverrides type (${typeof jobOverrides}). It needs to be an object or array of objects.` @@ -773,15 +773,22 @@ export class DataRecognizer { const overrides = Array.isArray(jobOverrides) ? jobOverrides : [jobOverrides]; const { jobs } = moduleConfig; - // first collect the overrides which don't contain a job id - // these will be applied to all jobs in the module - const generalOverrides = overrides.filter(o => o.job_id === undefined); + // separate all the overrides. + // the overrides which don't contain a job id will be applied to all jobs in the module + const generalOverrides = []; + const jobSpecificOverrides = []; + overrides.forEach(o => { + if (o.job_id === undefined) { + generalOverrides.push(o); + } else { + jobSpecificOverrides.push(o); + } + }); + generalOverrides.forEach(o => { jobs.forEach(({ config }) => merge(config, o)); }); - // collect all job specific overrides - const jobSpecificOverrides = overrides.filter(o => o.job_id !== undefined); jobSpecificOverrides.forEach(o => { // for each override, find the relevant job. // note, the job id already has the prefix prepended to it @@ -796,7 +803,7 @@ export class DataRecognizer { } applyDatafeedConfigOverrides(moduleConfig, datafeedOverrides, jobPrefix = '') { - if(datafeedOverrides !== undefined) { + if(datafeedOverrides !== undefined && datafeedOverrides !== null) { if (typeof datafeedOverrides !== 'object') { throw Boom.badRequest( `Incompatible datafeedOverrides type (${typeof datafeedOverrides}). It needs to be an object or array of objects.` @@ -808,9 +815,18 @@ export class DataRecognizer { const overrides = Array.isArray(datafeedOverrides) ? datafeedOverrides : [datafeedOverrides]; const { datafeeds } = moduleConfig; - // first collect the overrides which don't contain a datafeed id or a job id - // these will be applied to all jobs in the module - const generalOverrides = overrides.filter(o => o.datafeed_id === undefined && o.job_id === undefined); + // separate all the overrides. + // the overrides which don't contain a datafeed id or a job id will be applied to all jobs in the module + const generalOverrides = []; + const datafeedSpecificOverrides = []; + overrides.forEach(o => { + if (o.datafeed_id === undefined && o.job_id === undefined) { + generalOverrides.push(o); + } else { + datafeedSpecificOverrides.push(o); + } + }); + generalOverrides.forEach(o => { datafeeds.forEach(({ config }) => { merge(config, o); @@ -818,7 +834,6 @@ export class DataRecognizer { }); // collect all the overrides which contain either a job id or a datafeed id - const datafeedSpecificOverrides = overrides.filter(o => (o.datafeed_id !== undefined || o.job_id !== undefined)); datafeedSpecificOverrides.forEach(o => { // either a job id or datafeed id has been specified, so create a new id // containing either one plus the prefix