diff --git a/x-pack/legacy/plugins/ml/common/util/job_utils.d.ts b/x-pack/legacy/plugins/ml/common/util/job_utils.d.ts
index e2ea6c163b6ce..23152afe0af2f 100644
--- a/x-pack/legacy/plugins/ml/common/util/job_utils.d.ts
+++ b/x-pack/legacy/plugins/ml/common/util/job_utils.d.ts
@@ -32,6 +32,10 @@ export const ML_DATA_PREVIEW_COUNT: number;
export function isJobIdValid(jobId: string): boolean;
+export function validateModelMemoryLimitUnits(
+ modelMemoryLimit: string
+): { valid: boolean; messages: any[]; contains: () => boolean; find: () => void };
+
export function processCreatedBy(customSettings: { created_by?: string }): void;
export function mlFunctionToESAggregation(functionName: string): string | null;
diff --git a/x-pack/legacy/plugins/ml/common/util/job_utils.js b/x-pack/legacy/plugins/ml/common/util/job_utils.js
index 7b7a698cafb9c..877ab8df66ca2 100644
--- a/x-pack/legacy/plugins/ml/common/util/job_utils.js
+++ b/x-pack/legacy/plugins/ml/common/util/job_utils.js
@@ -405,10 +405,11 @@ export function basicJobValidation(job, fields, limits, skipMmlChecks = false) {
if (skipMmlChecks === false) {
// model memory limit
+ const mml = job.analysis_limits && job.analysis_limits.model_memory_limit;
const {
messages: mmlUnitMessages,
valid: mmlUnitValid,
- } = validateModelMemoryLimitUnits(job);
+ } = validateModelMemoryLimitUnits(mml);
messages.push(...mmlUnitMessages);
valid = (valid && mmlUnitValid);
@@ -494,12 +495,12 @@ export function validateModelMemoryLimit(job, limits) {
};
}
-export function validateModelMemoryLimitUnits(job) {
+export function validateModelMemoryLimitUnits(modelMemoryLimit) {
const messages = [];
let valid = true;
- if (typeof job.analysis_limits !== 'undefined' && typeof job.analysis_limits.model_memory_limit !== 'undefined') {
- const mml = job.analysis_limits.model_memory_limit.toUpperCase();
+ if (modelMemoryLimit !== undefined) {
+ const mml = modelMemoryLimit.toUpperCase();
const mmlSplit = mml.match(/\d+(\w+)$/);
const unit = (mmlSplit && mmlSplit.length === 2) ? mmlSplit[1] : null;
diff --git a/x-pack/legacy/plugins/ml/public/data_frame_analytics/pages/analytics_management/components/create_analytics_form/create_analytics_form.test.tsx b/x-pack/legacy/plugins/ml/public/data_frame_analytics/pages/analytics_management/components/create_analytics_form/create_analytics_form.test.tsx
index 1371eacef799b..846397aa93929 100644
--- a/x-pack/legacy/plugins/ml/public/data_frame_analytics/pages/analytics_management/components/create_analytics_form/create_analytics_form.test.tsx
+++ b/x-pack/legacy/plugins/ml/public/data_frame_analytics/pages/analytics_management/components/create_analytics_form/create_analytics_form.test.tsx
@@ -45,7 +45,7 @@ describe('Data Frame Analytics: ', () => {
);
const euiFormRows = wrapper.find('EuiFormRow');
- expect(euiFormRows.length).toBe(6);
+ expect(euiFormRows.length).toBe(7);
const row1 = euiFormRows.at(0);
expect(row1.find('label').text()).toBe('Job type');
diff --git a/x-pack/legacy/plugins/ml/public/data_frame_analytics/pages/analytics_management/components/create_analytics_form/create_analytics_form.tsx b/x-pack/legacy/plugins/ml/public/data_frame_analytics/pages/analytics_management/components/create_analytics_form/create_analytics_form.tsx
index ab8cc5a5982bb..598f88387f410 100644
--- a/x-pack/legacy/plugins/ml/public/data_frame_analytics/pages/analytics_management/components/create_analytics_form/create_analytics_form.tsx
+++ b/x-pack/legacy/plugins/ml/public/data_frame_analytics/pages/analytics_management/components/create_analytics_form/create_analytics_form.tsx
@@ -21,15 +21,21 @@ import { FormattedMessage } from '@kbn/i18n/react';
import { metadata } from 'ui/metadata';
import { IndexPattern, INDEX_PATTERN_ILLEGAL_CHARACTERS } from 'ui/index_patterns';
+import { ml } from '../../../../../services/ml_api_service';
import { Field, EVENT_RATE_FIELD_ID } from '../../../../../../common/types/fields';
import { newJobCapsService } from '../../../../../services/new_job_capabilities_service';
import { useKibanaContext } from '../../../../../contexts/kibana';
import { CreateAnalyticsFormProps } from '../../hooks/use_create_analytics_form';
-import { JOB_TYPES } from '../../hooks/use_create_analytics_form/state';
+import {
+ JOB_TYPES,
+ DEFAULT_MODEL_MEMORY_LIMIT,
+ getJobConfigFromFormState,
+} from '../../hooks/use_create_analytics_form/state';
import { JOB_ID_MAX_LENGTH } from '../../../../../../common/constants/validation';
import { Messages } from './messages';
import { JobType } from './job_type';
+import { mmlUnitInvalidErrorMessage } from '../../hooks/use_create_analytics_form/reducer';
// based on code used by `ui/index_patterns` internally
// remove the space character from the list of illegal characters
@@ -73,6 +79,8 @@ export const CreateAnalyticsForm: FC = ({ actions, sta
jobIdInvalidMaxLength,
jobType,
loadingDepFieldOptions,
+ modelMemoryLimit,
+ modelMemoryLimitUnitValid,
sourceIndex,
sourceIndexNameEmpty,
sourceIndexNameValid,
@@ -103,6 +111,25 @@ export const CreateAnalyticsForm: FC = ({ actions, sta
}
};
+ const loadModelMemoryLimitEstimate = async () => {
+ try {
+ const jobConfig = getJobConfigFromFormState(form);
+ delete jobConfig.dest;
+ delete jobConfig.model_memory_limit;
+ const resp = await ml.dataFrameAnalytics.estimateDataFrameAnalyticsMemoryUsage(jobConfig);
+ setFormState({
+ modelMemoryLimit: resp.expected_memory_without_disk,
+ });
+ } catch (e) {
+ setFormState({
+ modelMemoryLimit:
+ jobType !== undefined
+ ? DEFAULT_MODEL_MEMORY_LIMIT[jobType]
+ : DEFAULT_MODEL_MEMORY_LIMIT.outlier_detection,
+ });
+ }
+ };
+
const loadDependentFieldOptions = async () => {
setFormState({
loadingDepFieldOptions: true,
@@ -175,6 +202,21 @@ export const CreateAnalyticsForm: FC = ({ actions, sta
}
}, [sourceIndex, jobType, sourceIndexNameEmpty]);
+ useEffect(() => {
+ const hasBasicRequiredFields =
+ jobType !== undefined && sourceIndex !== '' && sourceIndexNameValid === true;
+
+ const hasRequiredAnalysisFields =
+ (jobType === JOB_TYPES.REGRESSION &&
+ dependentVariable !== '' &&
+ trainingPercent !== undefined) ||
+ jobType === JOB_TYPES.OUTLIER_DETECTION;
+
+ if (hasBasicRequiredFields && hasRequiredAnalysisFields) {
+ loadModelMemoryLimitEstimate();
+ }
+ }, [jobType, sourceIndex, dependentVariable, trainingPercent]);
+
return (
@@ -277,7 +319,7 @@ export const CreateAnalyticsForm: FC = ({ actions, sta
placeholder={i18n.translate(
'xpack.ml.dataframe.analytics.create.sourceIndexPlaceholder',
{
- defaultMessage: 'Choose a source index pattern or saved search.',
+ defaultMessage: 'Choose a source index pattern.',
}
)}
singleSelection={{ asPlainText: true }}
@@ -437,6 +479,24 @@ export const CreateAnalyticsForm: FC = ({ actions, sta
)}
+
+ setFormState({ modelMemoryLimit: e.target.value })}
+ isInvalid={modelMemoryLimit === ''}
+ />
+
({
type SourceIndex = DataFrameAnalyticsConfig['source']['index'];
-const getMockState = (index: SourceIndex) =>
+const getMockState = ({
+ index,
+ modelMemoryLimit,
+}: {
+ index: SourceIndex;
+ modelMemoryLimit?: string;
+}) =>
merge(getInitialState(), {
form: {
jobIdEmpty: false,
@@ -30,6 +36,7 @@ const getMockState = (index: SourceIndex) =>
source: { index },
dest: { index: 'the-destination-index' },
analysis: {},
+ model_memory_limit: modelMemoryLimit,
},
});
@@ -89,27 +96,50 @@ describe('useCreateAnalyticsForm', () => {
test('validateAdvancedEditor(): check index pattern variations', () => {
// valid single index pattern
- expect(validateAdvancedEditor(getMockState('the-source-index')).isValid).toBe(true);
+ expect(validateAdvancedEditor(getMockState({ index: 'the-source-index' })).isValid).toBe(true);
// valid array with one ES index pattern
- expect(validateAdvancedEditor(getMockState(['the-source-index'])).isValid).toBe(true);
+ expect(validateAdvancedEditor(getMockState({ index: ['the-source-index'] })).isValid).toBe(
+ true
+ );
// valid array with two ES index patterns
expect(
- validateAdvancedEditor(getMockState(['the-source-index-1', 'the-source-index-2'])).isValid
+ validateAdvancedEditor(getMockState({ index: ['the-source-index-1', 'the-source-index-2'] }))
+ .isValid
).toBe(true);
// invalid comma-separated index pattern, this is only allowed in the simple form
// but not the advanced editor.
expect(
- validateAdvancedEditor(getMockState('the-source-index-1,the-source-index-2')).isValid
+ validateAdvancedEditor(getMockState({ index: 'the-source-index-1,the-source-index-2' }))
+ .isValid
).toBe(false);
expect(
validateAdvancedEditor(
- getMockState(['the-source-index-1,the-source-index-2', 'the-source-index-3'])
+ getMockState({ index: ['the-source-index-1,the-source-index-2', 'the-source-index-3'] })
).isValid
).toBe(false);
// invalid formats ("fake" TS casting to get valid TS and be able to run the tests)
- expect(validateAdvancedEditor(getMockState({} as SourceIndex)).isValid).toBe(false);
+ expect(validateAdvancedEditor(getMockState({ index: {} as SourceIndex })).isValid).toBe(false);
expect(
- validateAdvancedEditor(getMockState((undefined as unknown) as SourceIndex)).isValid
+ validateAdvancedEditor(getMockState({ index: (undefined as unknown) as SourceIndex })).isValid
+ ).toBe(false);
+ });
+
+ test('validateAdvancedEditor(): check model memory limit validation', () => {
+ // valid model_memory_limit units
+ expect(
+ validateAdvancedEditor(getMockState({ index: 'the-source-index', modelMemoryLimit: '100mb' }))
+ .isValid
+ ).toBe(true);
+ // invalid model_memory_limit units
+ expect(
+ validateAdvancedEditor(
+ getMockState({ index: 'the-source-index', modelMemoryLimit: '100bob' })
+ ).isValid
+ ).toBe(false);
+ // invalid model_memory_limit if empty
+ expect(
+ validateAdvancedEditor(getMockState({ index: 'the-source-index', modelMemoryLimit: '' }))
+ .isValid
).toBe(false);
});
});
diff --git a/x-pack/legacy/plugins/ml/public/data_frame_analytics/pages/analytics_management/hooks/use_create_analytics_form/reducer.ts b/x-pack/legacy/plugins/ml/public/data_frame_analytics/pages/analytics_management/hooks/use_create_analytics_form/reducer.ts
index 037ec95ec17cc..f0fa2ad3b66db 100644
--- a/x-pack/legacy/plugins/ml/public/data_frame_analytics/pages/analytics_management/hooks/use_create_analytics_form/reducer.ts
+++ b/x-pack/legacy/plugins/ml/public/data_frame_analytics/pages/analytics_management/hooks/use_create_analytics_form/reducer.ts
@@ -13,11 +13,29 @@ import { isValidIndexName } from '../../../../../../common/util/es_utils';
import { Action, ACTION } from './actions';
import { getInitialState, getJobConfigFromFormState, State, JOB_TYPES } from './state';
-import { isJobIdValid } from '../../../../../../common/util/job_utils';
+import {
+ isJobIdValid,
+ validateModelMemoryLimitUnits,
+} from '../../../../../../common/util/job_utils';
import { maxLengthValidator } from '../../../../../../common/util/validators';
-import { JOB_ID_MAX_LENGTH } from '../../../../../../common/constants/validation';
+import {
+ JOB_ID_MAX_LENGTH,
+ ALLOWED_DATA_UNITS,
+} from '../../../../../../common/constants/validation';
import { getDependentVar, isRegressionAnalysis } from '../../../../common/analytics';
+const mmlAllowedUnitsStr = `${ALLOWED_DATA_UNITS.slice(0, ALLOWED_DATA_UNITS.length - 1).join(
+ ', '
+)} or ${[...ALLOWED_DATA_UNITS].pop()}`;
+
+export const mmlUnitInvalidErrorMessage = i18n.translate(
+ 'xpack.ml.dataframe.analytics.create.modelMemoryUnitsInvalidError',
+ {
+ defaultMessage: 'Model memory limit data unit unrecognized. It must be {str}',
+ values: { str: mmlAllowedUnitsStr },
+ }
+);
+
const getSourceIndexString = (state: State) => {
const { jobConfig } = state;
@@ -63,6 +81,12 @@ export const validateAdvancedEditor = (state: State): State => {
const destinationIndexNameValid = isValidIndexName(destinationIndexName);
const destinationIndexPatternTitleExists =
state.indexPatternsMap[destinationIndexName] !== undefined;
+ const mml = jobConfig.model_memory_limit;
+ const modelMemoryLimitEmpty = mml === '';
+ if (!modelMemoryLimitEmpty && mml !== undefined) {
+ const { valid } = validateModelMemoryLimitUnits(mml);
+ state.form.modelMemoryLimitUnitValid = valid;
+ }
let dependentVariableEmpty = false;
if (isRegressionAnalysis(jobConfig.analysis)) {
@@ -126,7 +150,27 @@ export const validateAdvancedEditor = (state: State): State => {
});
}
+ if (modelMemoryLimitEmpty) {
+ state.advancedEditorMessages.push({
+ error: i18n.translate(
+ 'xpack.ml.dataframe.analytics.create.advancedEditorMessage.modelMemoryLimitEmpty',
+ {
+ defaultMessage: 'The model memory limit field must not be empty.',
+ }
+ ),
+ message: '',
+ });
+ }
+
+ if (!state.form.modelMemoryLimitUnitValid) {
+ state.advancedEditorMessages.push({
+ error: mmlUnitInvalidErrorMessage,
+ message: '',
+ });
+ }
+
state.isValid =
+ state.form.modelMemoryLimitUnitValid &&
!jobIdEmpty &&
jobIdValid &&
!jobIdExists &&
@@ -135,6 +179,7 @@ export const validateAdvancedEditor = (state: State): State => {
!destinationIndexNameEmpty &&
destinationIndexNameValid &&
!dependentVariableEmpty &&
+ !modelMemoryLimitEmpty &&
(!destinationIndexPatternTitleExists || !createIndexPattern);
return state;
@@ -153,11 +198,19 @@ const validateForm = (state: State): State => {
destinationIndexPatternTitleExists,
createIndexPattern,
dependentVariable,
+ modelMemoryLimit,
} = state.form;
const dependentVariableEmpty = jobType === JOB_TYPES.REGRESSION && dependentVariable === '';
+ const modelMemoryLimitEmpty = modelMemoryLimit === '';
+
+ if (!modelMemoryLimitEmpty && modelMemoryLimit !== undefined) {
+ const { valid } = validateModelMemoryLimitUnits(modelMemoryLimit);
+ state.form.modelMemoryLimitUnitValid = valid;
+ }
state.isValid =
+ state.form.modelMemoryLimitUnitValid &&
!jobIdEmpty &&
jobIdValid &&
!jobIdExists &&
@@ -166,6 +219,7 @@ const validateForm = (state: State): State => {
!destinationIndexNameEmpty &&
destinationIndexNameValid &&
!dependentVariableEmpty &&
+ !modelMemoryLimitEmpty &&
(!destinationIndexPatternTitleExists || !createIndexPattern);
return state;
diff --git a/x-pack/legacy/plugins/ml/public/data_frame_analytics/pages/analytics_management/hooks/use_create_analytics_form/state.ts b/x-pack/legacy/plugins/ml/public/data_frame_analytics/pages/analytics_management/hooks/use_create_analytics_form/state.ts
index fb97f562ea680..b90317015c8c9 100644
--- a/x-pack/legacy/plugins/ml/public/data_frame_analytics/pages/analytics_management/hooks/use_create_analytics_form/state.ts
+++ b/x-pack/legacy/plugins/ml/public/data_frame_analytics/pages/analytics_management/hooks/use_create_analytics_form/state.ts
@@ -10,8 +10,11 @@ import { mlNodesAvailable } from '../../../../../ml_nodes_check/check_ml_nodes';
import { DataFrameAnalyticsId, DataFrameAnalyticsConfig } from '../../../../common';
-const OUTLIER_DETECTION_DEFAULT_MODEL_MEMORY_LIMIT = '50mb';
-const REGRESSION_DEFAULT_MODEL_MEMORY_LIMIT = '100mb';
+export enum DEFAULT_MODEL_MEMORY_LIMIT {
+ regression = '100mb',
+ // eslint-disable-next-line @typescript-eslint/camelcase
+ outlier_detection = '50mb',
+}
export type EsIndexName = string;
export type DependentVariable = string;
@@ -53,6 +56,8 @@ export interface State {
jobIdValid: boolean;
jobType: AnalyticsJobType;
loadingDepFieldOptions: boolean;
+ modelMemoryLimit: string | undefined;
+ modelMemoryLimitUnitValid: boolean;
sourceIndex: EsIndexName;
sourceIndexNameEmpty: boolean;
sourceIndexNameValid: boolean;
@@ -94,6 +99,8 @@ export const getInitialState = (): State => ({
jobIdValid: false,
jobType: undefined,
loadingDepFieldOptions: false,
+ modelMemoryLimit: undefined,
+ modelMemoryLimitUnitValid: true,
sourceIndex: '',
sourceIndexNameEmpty: true,
sourceIndexNameValid: false,
@@ -121,11 +128,6 @@ export const getInitialState = (): State => ({
export const getJobConfigFromFormState = (
formState: State['form']
): DeepPartial => {
- const modelMemoryLimit =
- formState.jobType === JOB_TYPES.REGRESSION
- ? REGRESSION_DEFAULT_MODEL_MEMORY_LIMIT
- : OUTLIER_DETECTION_DEFAULT_MODEL_MEMORY_LIMIT;
-
const jobConfig: DeepPartial = {
source: {
// If a Kibana index patterns includes commas, we need to split
@@ -144,7 +146,7 @@ export const getJobConfigFromFormState = (
analysis: {
outlier_detection: {},
},
- model_memory_limit: modelMemoryLimit,
+ model_memory_limit: formState.modelMemoryLimit,
};
if (formState.jobType === JOB_TYPES.REGRESSION) {
diff --git a/x-pack/legacy/plugins/ml/public/jobs/jobs_list/components/validate_job.js b/x-pack/legacy/plugins/ml/public/jobs/jobs_list/components/validate_job.js
index 71e16188db948..dda48866839d5 100644
--- a/x-pack/legacy/plugins/ml/public/jobs/jobs_list/components/validate_job.js
+++ b/x-pack/legacy/plugins/ml/public/jobs/jobs_list/components/validate_job.js
@@ -23,7 +23,7 @@ export function validateModelMemoryLimit(mml) {
}
};
- let validationResults = validateModelMemoryLimitUnitsUtils(tempJob);
+ let validationResults = validateModelMemoryLimitUnitsUtils(mml);
let { valid } = validationResults;
if(valid) {
diff --git a/x-pack/legacy/plugins/ml/public/services/ml_api_service/data_frame_analytics.js b/x-pack/legacy/plugins/ml/public/services/ml_api_service/data_frame_analytics.js
index c5147fc4af7d7..3f987a1763140 100644
--- a/x-pack/legacy/plugins/ml/public/services/ml_api_service/data_frame_analytics.js
+++ b/x-pack/legacy/plugins/ml/public/services/ml_api_service/data_frame_analytics.js
@@ -47,6 +47,13 @@ export const dataFrameAnalytics = {
data: evaluateConfig
});
},
+ estimateDataFrameAnalyticsMemoryUsage(jobConfig) {
+ return http({
+ url: `${basePath}/data_frame/analytics/_estimate_memory_usage`,
+ method: 'POST',
+ data: jobConfig
+ });
+ },
deleteDataFrameAnalytics(analyticsId) {
return http({
url: `${basePath}/data_frame/analytics/${analyticsId}`,
diff --git a/x-pack/legacy/plugins/ml/public/services/ml_api_service/index.d.ts b/x-pack/legacy/plugins/ml/public/services/ml_api_service/index.d.ts
index 414229578c217..12f39bfa78dc0 100644
--- a/x-pack/legacy/plugins/ml/public/services/ml_api_service/index.d.ts
+++ b/x-pack/legacy/plugins/ml/public/services/ml_api_service/index.d.ts
@@ -13,6 +13,8 @@ import { MlServerDefaults, MlServerLimits } from '../../services/ml_server_info'
import { ES_AGGREGATION } from '../../../common/constants/aggregation_types';
import { DataFrameAnalyticsStats } from '../../data_frame_analytics/pages/analytics_management/components/analytics_list/common';
import { JobMessage } from '../../../common/types/audit_message';
+import { DataFrameAnalyticsConfig } from '../../data_frame_analytics/common/analytics';
+import { DeepPartial } from '../../../common/types/common';
// TODO This is not a complete representation of all methods of `ml.*`.
// It just satisfies needs for other parts of the code area which use
@@ -70,6 +72,9 @@ declare interface Ml {
getDataFrameAnalyticsStats(analyticsId?: string): Promise;
createDataFrameAnalytics(analyticsId: string, analyticsConfig: any): Promise;
evaluateDataFrameAnalytics(evaluateConfig: any): Promise;
+ estimateDataFrameAnalyticsMemoryUsage(
+ jobConfig: DeepPartial
+ ): Promise;
deleteDataFrameAnalytics(analyticsId: string): Promise;
startDataFrameAnalytics(analyticsId: string): Promise;
stopDataFrameAnalytics(
diff --git a/x-pack/legacy/plugins/ml/server/client/elasticsearch_ml.js b/x-pack/legacy/plugins/ml/server/client/elasticsearch_ml.js
index 7868ff31d2b1e..9c33b199a3706 100644
--- a/x-pack/legacy/plugins/ml/server/client/elasticsearch_ml.js
+++ b/x-pack/legacy/plugins/ml/server/client/elasticsearch_ml.js
@@ -170,6 +170,16 @@ export const elasticsearchJsPlugin = (Client, config, components) => { // eslint
method: 'POST'
});
+ ml.estimateDataFrameAnalyticsMemoryUsage = ca({
+ urls: [
+ {
+ fmt: '/_ml/data_frame/analytics/_estimate_memory_usage',
+ }
+ ],
+ needBody: true,
+ method: 'POST'
+ });
+
ml.deleteDataFrameAnalytics = ca({
urls: [
{
diff --git a/x-pack/legacy/plugins/ml/server/routes/data_frame_analytics.js b/x-pack/legacy/plugins/ml/server/routes/data_frame_analytics.js
index 41f08736a78e5..d467aeea31f99 100644
--- a/x-pack/legacy/plugins/ml/server/routes/data_frame_analytics.js
+++ b/x-pack/legacy/plugins/ml/server/routes/data_frame_analytics.js
@@ -91,6 +91,19 @@ export function dataFrameAnalyticsRoutes({ commonRouteConfig, elasticsearchPlugi
}
});
+ route({
+ method: 'POST',
+ path: '/api/ml/data_frame/analytics/_estimate_memory_usage',
+ handler(request) {
+ const callWithRequest = callWithRequestFactory(elasticsearchPlugin, request);
+ return callWithRequest('ml.estimateDataFrameAnalyticsMemoryUsage', { body: request.payload })
+ .catch(resp => wrapError(resp));
+ },
+ config: {
+ ...commonRouteConfig
+ }
+ });
+
route({
method: 'DELETE',
path: '/api/ml/data_frame/analytics/{analyticsId}',