Skip to content

Commit

Permalink
[ML] Add support for date_nanos time field in anomaly job wizard (#59017
Browse files Browse the repository at this point in the history
) (#59403)

* [ML] Add support for date_nanos time field in anomaly job wizard

* [ML] Edits following review

* [ML] Add functional test for creating job off date_nanos data
  • Loading branch information
peteharverson authored Mar 5, 2020
1 parent bf4614d commit 7b9880c
Show file tree
Hide file tree
Showing 7 changed files with 2,023 additions and 81 deletions.
2 changes: 1 addition & 1 deletion x-pack/plugins/ml/server/models/job_validation/messages.js
Original file line number Diff line number Diff line change
Expand Up @@ -495,7 +495,7 @@ export const getMessages = () => {
time_field_invalid: {
status: 'ERROR',
text: i18n.translate('xpack.ml.models.jobValidation.messages.timeFieldInvalidMessage', {
defaultMessage: `{timeField} cannot be used as the time-field because it's not a valid field of type 'date'.`,
defaultMessage: `{timeField} cannot be used as the time field because it is not a field of type 'date' or 'date_nanos'.`,
values: {
timeField: `'{{timeField}}'`,
},
Expand Down

This file was deleted.

104 changes: 104 additions & 0 deletions x-pack/plugins/ml/server/models/job_validation/validate_time_range.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,104 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/

import { APICaller } from 'src/core/server';
import { ES_FIELD_TYPES } from '../../../../../../src/plugins/data/server';
import { parseInterval } from '../../../../../legacy/plugins/ml/common/util/parse_interval';
import { CombinedJob } from '../../../../../legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/configs';
// @ts-ignore
import { validateJobObject } from './validate_job_object';

interface ValidateTimeRangeMessage {
id: string;
timeField?: string;
minTimeSpanReadable?: string;
bucketSpanCompareFactor?: number;
}

interface TimeRange {
start: number;
end: number;
}

const BUCKET_SPAN_COMPARE_FACTOR = 25;
const MIN_TIME_SPAN_MS = 7200000;
const MIN_TIME_SPAN_READABLE = '2 hours';

export async function isValidTimeField(callAsCurrentUser: APICaller, job: CombinedJob) {
const index = job.datafeed_config.indices.join(',');
const timeField = job.data_description.time_field;

// check if time_field is of type 'date' or 'date_nanos'
const fieldCaps = await callAsCurrentUser('fieldCaps', {
index,
fields: [timeField],
});

let fieldType = fieldCaps.fields[timeField]?.date?.type;
if (fieldType === undefined) {
fieldType = fieldCaps.fields[timeField]?.date_nanos?.type;
}
return fieldType === ES_FIELD_TYPES.DATE || fieldType === ES_FIELD_TYPES.DATE_NANOS;
}

export async function validateTimeRange(
callAsCurrentUser: APICaller,
job: CombinedJob,
timeRange: TimeRange | undefined
) {
const messages: ValidateTimeRangeMessage[] = [];

validateJobObject(job);

// check if time_field is a date type
if (!(await isValidTimeField(callAsCurrentUser, job))) {
messages.push({
id: 'time_field_invalid',
timeField: job.data_description.time_field,
});
// if the time field is invalid, skip all other checks
return messages;
}

// if there is no duration, do not run the estimate test
if (
typeof timeRange === 'undefined' ||
typeof timeRange.start === 'undefined' ||
typeof timeRange.end === 'undefined'
) {
return messages;
}

// check if time range is after the Unix epoch start
if (timeRange.start < 0 || timeRange.end < 0) {
messages.push({ id: 'time_range_before_epoch' });
}

// check for minimum time range (25 buckets or 2 hours, whichever is longer)
const interval = parseInterval(job.analysis_config.bucket_span);
if (interval === null) {
messages.push({ id: 'bucket_span_invalid' });
} else {
const bucketSpan: number = interval.asMilliseconds();
const minTimeSpanBasedOnBucketSpan = bucketSpan * BUCKET_SPAN_COMPARE_FACTOR;
const timeSpan = timeRange.end - timeRange.start;
const minRequiredTimeSpan = Math.max(MIN_TIME_SPAN_MS, minTimeSpanBasedOnBucketSpan);

if (minRequiredTimeSpan > timeSpan) {
messages.push({
id: 'time_range_short',
minTimeSpanReadable: MIN_TIME_SPAN_READABLE,
bucketSpanCompareFactor: BUCKET_SPAN_COMPARE_FACTOR,
});
}
}

if (messages.length === 0) {
messages.push({ id: 'success_time_range' });
}

return messages;
}
Loading

0 comments on commit 7b9880c

Please sign in to comment.