diff --git a/x-pack/plugins/ml/server/models/job_validation/messages.js b/x-pack/plugins/ml/server/models/job_validation/messages.js index 33931f03facc3..105d642560cc7 100644 --- a/x-pack/plugins/ml/server/models/job_validation/messages.js +++ b/x-pack/plugins/ml/server/models/job_validation/messages.js @@ -495,7 +495,7 @@ export const getMessages = () => { time_field_invalid: { status: 'ERROR', text: i18n.translate('xpack.ml.models.jobValidation.messages.timeFieldInvalidMessage', { - defaultMessage: `{timeField} cannot be used as the time-field because it's not a valid field of type 'date'.`, + defaultMessage: `{timeField} cannot be used as the time field because it is not a field of type 'date' or 'date_nanos'.`, values: { timeField: `'{{timeField}}'`, }, diff --git a/x-pack/plugins/ml/server/models/job_validation/validate_time_range.js b/x-pack/plugins/ml/server/models/job_validation/validate_time_range.js deleted file mode 100644 index e6a92b45649b0..0000000000000 --- a/x-pack/plugins/ml/server/models/job_validation/validate_time_range.js +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import _ from 'lodash'; - -import { ES_FIELD_TYPES } from '../../../../../../src/plugins/data/server'; -import { parseInterval } from '../../../../../legacy/plugins/ml/common/util/parse_interval'; -import { validateJobObject } from './validate_job_object'; - -const BUCKET_SPAN_COMPARE_FACTOR = 25; -const MIN_TIME_SPAN_MS = 7200000; -const MIN_TIME_SPAN_READABLE = '2 hours'; - -export async function isValidTimeField(callWithRequest, job) { - const index = job.datafeed_config.indices.join(','); - const timeField = job.data_description.time_field; - - // check if time_field is of type 'date' - const fieldCaps = await callWithRequest('fieldCaps', { - index, - fields: [timeField], - }); - // get the field's type with the following notation - // because a nested field could contain dots and confuse _.get - const fieldType = _.get(fieldCaps, `fields['${timeField}'].date.type`); - return fieldType === ES_FIELD_TYPES.DATE; -} - -export async function validateTimeRange(callWithRequest, job, duration) { - const messages = []; - - validateJobObject(job); - - // check if time_field is of type 'date' - if (!(await isValidTimeField(callWithRequest, job))) { - messages.push({ - id: 'time_field_invalid', - timeField: job.data_description.time_field, - }); - // if the time field is invalid, skip all other checks - return Promise.resolve(messages); - } - - // if there is no duration, do not run the estimate test - if ( - typeof duration === 'undefined' || - typeof duration.start === 'undefined' || - typeof duration.end === 'undefined' - ) { - return Promise.resolve(messages); - } - - // check if time range is after the Unix epoch start - if (duration.start < 0 || duration.end < 0) { - messages.push({ id: 'time_range_before_epoch' }); - } - - // check for minimum time range (25 buckets or 2 hours, whichever is longer) - const bucketSpan = parseInterval(job.analysis_config.bucket_span).valueOf(); - const minTimeSpanBasedOnBucketSpan = bucketSpan * BUCKET_SPAN_COMPARE_FACTOR; - const timeSpan = duration.end - duration.start; - const minRequiredTimeSpan = Math.max(MIN_TIME_SPAN_MS, minTimeSpanBasedOnBucketSpan); - - if (minRequiredTimeSpan > timeSpan) { - messages.push({ - id: 'time_range_short', - minTimeSpanReadable: MIN_TIME_SPAN_READABLE, - bucketSpanCompareFactor: BUCKET_SPAN_COMPARE_FACTOR, - }); - } - - if (messages.length === 0) { - messages.push({ id: 'success_time_range' }); - } - - return messages; -} diff --git a/x-pack/plugins/ml/server/models/job_validation/validate_time_range.ts b/x-pack/plugins/ml/server/models/job_validation/validate_time_range.ts new file mode 100644 index 0000000000000..551b5ab9173a4 --- /dev/null +++ b/x-pack/plugins/ml/server/models/job_validation/validate_time_range.ts @@ -0,0 +1,104 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { APICaller } from 'src/core/server'; +import { ES_FIELD_TYPES } from '../../../../../../src/plugins/data/server'; +import { parseInterval } from '../../../../../legacy/plugins/ml/common/util/parse_interval'; +import { CombinedJob } from '../../../../../legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/configs'; +// @ts-ignore +import { validateJobObject } from './validate_job_object'; + +interface ValidateTimeRangeMessage { + id: string; + timeField?: string; + minTimeSpanReadable?: string; + bucketSpanCompareFactor?: number; +} + +interface TimeRange { + start: number; + end: number; +} + +const BUCKET_SPAN_COMPARE_FACTOR = 25; +const MIN_TIME_SPAN_MS = 7200000; +const MIN_TIME_SPAN_READABLE = '2 hours'; + +export async function isValidTimeField(callAsCurrentUser: APICaller, job: CombinedJob) { + const index = job.datafeed_config.indices.join(','); + const timeField = job.data_description.time_field; + + // check if time_field is of type 'date' or 'date_nanos' + const fieldCaps = await callAsCurrentUser('fieldCaps', { + index, + fields: [timeField], + }); + + let fieldType = fieldCaps.fields[timeField]?.date?.type; + if (fieldType === undefined) { + fieldType = fieldCaps.fields[timeField]?.date_nanos?.type; + } + return fieldType === ES_FIELD_TYPES.DATE || fieldType === ES_FIELD_TYPES.DATE_NANOS; +} + +export async function validateTimeRange( + callAsCurrentUser: APICaller, + job: CombinedJob, + timeRange: TimeRange | undefined +) { + const messages: ValidateTimeRangeMessage[] = []; + + validateJobObject(job); + + // check if time_field is a date type + if (!(await isValidTimeField(callAsCurrentUser, job))) { + messages.push({ + id: 'time_field_invalid', + timeField: job.data_description.time_field, + }); + // if the time field is invalid, skip all other checks + return messages; + } + + // if there is no duration, do not run the estimate test + if ( + typeof timeRange === 'undefined' || + typeof timeRange.start === 'undefined' || + typeof timeRange.end === 'undefined' + ) { + return messages; + } + + // check if time range is after the Unix epoch start + if (timeRange.start < 0 || timeRange.end < 0) { + messages.push({ id: 'time_range_before_epoch' }); + } + + // check for minimum time range (25 buckets or 2 hours, whichever is longer) + const interval = parseInterval(job.analysis_config.bucket_span); + if (interval === null) { + messages.push({ id: 'bucket_span_invalid' }); + } else { + const bucketSpan: number = interval.asMilliseconds(); + const minTimeSpanBasedOnBucketSpan = bucketSpan * BUCKET_SPAN_COMPARE_FACTOR; + const timeSpan = timeRange.end - timeRange.start; + const minRequiredTimeSpan = Math.max(MIN_TIME_SPAN_MS, minTimeSpanBasedOnBucketSpan); + + if (minRequiredTimeSpan > timeSpan) { + messages.push({ + id: 'time_range_short', + minTimeSpanReadable: MIN_TIME_SPAN_READABLE, + bucketSpanCompareFactor: BUCKET_SPAN_COMPARE_FACTOR, + }); + } + } + + if (messages.length === 0) { + messages.push({ id: 'success_time_range' }); + } + + return messages; +} diff --git a/x-pack/test/functional/apps/machine_learning/anomaly_detection/date_nanos_job.ts b/x-pack/test/functional/apps/machine_learning/anomaly_detection/date_nanos_job.ts new file mode 100644 index 0000000000000..2a9824f46778d --- /dev/null +++ b/x-pack/test/functional/apps/machine_learning/anomaly_detection/date_nanos_job.ts @@ -0,0 +1,440 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import expect from '@kbn/expect'; + +import { FtrProviderContext } from '../../../ftr_provider_context'; + +interface Detector { + identifier: string; + function: string; + field?: string; + byField?: string; + overField?: string; + partitionField?: string; + excludeFrequent?: string; + description?: string; +} + +interface DatafeedConfig { + queryDelay?: string; + frequency?: string; + scrollSize?: string; +} + +interface PickFieldsConfig { + detectors: Detector[]; + influencers: string[]; + bucketSpan: string; + memoryLimit: string; + summaryCountField?: string; +} + +// type guards +// Detector +const isDetectorWithField = (arg: any): arg is Required> => { + return arg.hasOwnProperty('field'); +}; +const isDetectorWithByField = (arg: any): arg is Required> => { + return arg.hasOwnProperty('byField'); +}; +const isDetectorWithOverField = (arg: any): arg is Required> => { + return arg.hasOwnProperty('overField'); +}; +const isDetectorWithPartitionField = ( + arg: any +): arg is Required> => { + return arg.hasOwnProperty('partitionField'); +}; +const isDetectorWithExcludeFrequent = ( + arg: any +): arg is Required> => { + return arg.hasOwnProperty('excludeFrequent'); +}; +const isDetectorWithDescription = (arg: any): arg is Required> => { + return arg.hasOwnProperty('description'); +}; + +// DatafeedConfig +const isDatafeedConfigWithQueryDelay = ( + arg: any +): arg is Required> => { + return arg.hasOwnProperty('queryDelay'); +}; +const isDatafeedConfigWithFrequency = ( + arg: any +): arg is Required> => { + return arg.hasOwnProperty('frequency'); +}; +const isDatafeedConfigWithScrollSize = ( + arg: any +): arg is Required> => { + return arg.hasOwnProperty('scrollSize'); +}; + +// PickFieldsConfig +const isPickFieldsConfigWithSummaryCountField = ( + arg: any +): arg is Required> => { + return arg.hasOwnProperty('summaryCountField'); +}; + +// eslint-disable-next-line import/no-default-export +export default function({ getService }: FtrProviderContext) { + const esArchiver = getService('esArchiver'); + const ml = getService('ml'); + + const defaultValues = { + datafeedQuery: `{ + "bool": { + "must": [ + { + "match_all": {} + } + ] + } +}`, + queryDelay: '60s', + frequency: '450s', + scrollSize: '1000', + }; + + const testDataList = [ + { + suiteTitle: 'with count detector and model plot disabled', + jobSource: 'event_rate_gen_trend_nanos', + jobId: `event_rate_nanos_count_1_${Date.now()}`, + jobDescription: + 'Create advanced job based on the event rate dataset with a date_nanos time field, 30m bucketspan and count', + jobGroups: ['automated', 'event-rate', 'date-nanos'], + pickFieldsConfig: { + detectors: [ + { + identifier: 'count', + function: 'count', + description: 'event rate', + } as Detector, + ], + summaryCountField: 'count', + influencers: [], + bucketSpan: '30m', + memoryLimit: '10mb', + } as PickFieldsConfig, + datafeedConfig: {} as DatafeedConfig, + expected: { + wizard: { + timeField: '@timestamp', + }, + row: { + recordCount: '105,120', + memoryStatus: 'ok', + jobState: 'closed', + datafeedState: 'stopped', + latestTimestamp: '2016-01-01 00:00:00', + }, + counts: { + processed_record_count: '105,120', + processed_field_count: '105,120', + input_bytes: '4.2 MB', + input_field_count: '105,120', + invalid_date_count: '0', + missing_field_count: '0', + out_of_order_timestamp_count: '0', + empty_bucket_count: '0', + sparse_bucket_count: '0', + bucket_count: '17,520', + earliest_record_timestamp: '2015-01-01 00:10:00', + latest_record_timestamp: '2016-01-01 00:00:00', + input_record_count: '105,120', + latest_bucket_timestamp: '2016-01-01 00:00:00', + }, + modelSizeStats: { + result_type: 'model_size_stats', + model_bytes_exceeded: '0.0 B', + model_bytes_memory_limit: '10.0 MB', + total_by_field_count: '3', + total_over_field_count: '0', + total_partition_field_count: '2', + bucket_allocation_failures_count: '0', + memory_status: 'ok', + timestamp: '2015-12-31 23:30:00', + }, + }, + }, + ]; + + describe('job on data set with date_nanos time field', function() { + this.tags(['smoke', 'mlqa']); + before(async () => { + await esArchiver.load('ml/event_rate_nanos'); + await ml.securityUI.loginAsMlPowerUser(); + }); + + after(async () => { + await esArchiver.unload('ml/event_rate_nanos'); + await ml.api.cleanMlIndices(); + }); + + for (const testData of testDataList) { + describe(`${testData.suiteTitle}`, function() { + it('job creation loads the job management page', async () => { + await ml.navigation.navigateToMl(); + await ml.navigation.navigateToJobManagement(); + }); + + it('job creation loads the new job source selection page', async () => { + await ml.jobManagement.navigateToNewJobSourceSelection(); + }); + + it('job creation loads the job type selection page', async () => { + await ml.jobSourceSelection.selectSourceForAnomalyDetectionJob(testData.jobSource); + }); + + it('job creation loads the advanced job wizard page', async () => { + await ml.jobTypeSelection.selectAdvancedJob(); + }); + + it('job creation displays the configure datafeed step', async () => { + await ml.jobWizardCommon.assertConfigureDatafeedSectionExists(); + }); + + it('job creation pre-fills the datafeed query editor', async () => { + await ml.jobWizardAdvanced.assertDatafeedQueryEditorExists(); + await ml.jobWizardAdvanced.assertDatafeedQueryEditorValue(defaultValues.datafeedQuery); + }); + + it('job creation inputs the query delay', async () => { + await ml.jobWizardAdvanced.assertQueryDelayInputExists(); + await ml.jobWizardAdvanced.assertQueryDelayValue(defaultValues.queryDelay); + if (isDatafeedConfigWithQueryDelay(testData.datafeedConfig)) { + await ml.jobWizardAdvanced.setQueryDelay(testData.datafeedConfig.queryDelay); + } + }); + + it('job creation inputs the frequency', async () => { + await ml.jobWizardAdvanced.assertFrequencyInputExists(); + await ml.jobWizardAdvanced.assertFrequencyValue(defaultValues.frequency); + if (isDatafeedConfigWithFrequency(testData.datafeedConfig)) { + await ml.jobWizardAdvanced.setFrequency(testData.datafeedConfig.frequency); + } + }); + + it('job creation inputs the scroll size', async () => { + await ml.jobWizardAdvanced.assertScrollSizeInputExists(); + await ml.jobWizardAdvanced.assertScrollSizeValue(defaultValues.scrollSize); + if (isDatafeedConfigWithScrollSize(testData.datafeedConfig)) { + await ml.jobWizardAdvanced.setScrollSize(testData.datafeedConfig.scrollSize); + } + }); + + it('job creation pre-fills the time field', async () => { + await ml.jobWizardAdvanced.assertTimeFieldInputExists(); + await ml.jobWizardAdvanced.assertTimeFieldSelection([testData.expected.wizard.timeField]); + }); + + it('job creation displays the pick fields step', async () => { + await ml.jobWizardCommon.advanceToPickFieldsSection(); + }); + + it('job creation selects the summary count field', async () => { + await ml.jobWizardAdvanced.assertSummaryCountFieldInputExists(); + if (isPickFieldsConfigWithSummaryCountField(testData.pickFieldsConfig)) { + await ml.jobWizardAdvanced.selectSummaryCountField( + testData.pickFieldsConfig.summaryCountField + ); + } else { + await ml.jobWizardAdvanced.assertSummaryCountFieldSelection([]); + } + }); + + it('job creation adds detectors', async () => { + for (const detector of testData.pickFieldsConfig.detectors) { + await ml.jobWizardAdvanced.openCreateDetectorModal(); + await ml.jobWizardAdvanced.assertDetectorFunctionInputExists(); + await ml.jobWizardAdvanced.assertDetectorFunctionSelection([]); + await ml.jobWizardAdvanced.assertDetectorFieldInputExists(); + await ml.jobWizardAdvanced.assertDetectorFieldSelection([]); + await ml.jobWizardAdvanced.assertDetectorByFieldInputExists(); + await ml.jobWizardAdvanced.assertDetectorByFieldSelection([]); + await ml.jobWizardAdvanced.assertDetectorOverFieldInputExists(); + await ml.jobWizardAdvanced.assertDetectorOverFieldSelection([]); + await ml.jobWizardAdvanced.assertDetectorPartitionFieldInputExists(); + await ml.jobWizardAdvanced.assertDetectorPartitionFieldSelection([]); + await ml.jobWizardAdvanced.assertDetectorExcludeFrequentInputExists(); + await ml.jobWizardAdvanced.assertDetectorExcludeFrequentSelection([]); + await ml.jobWizardAdvanced.assertDetectorDescriptionInputExists(); + await ml.jobWizardAdvanced.assertDetectorDescriptionValue(''); + + await ml.jobWizardAdvanced.selectDetectorFunction(detector.function); + if (isDetectorWithField(detector)) { + await ml.jobWizardAdvanced.selectDetectorField(detector.field); + } + if (isDetectorWithByField(detector)) { + await ml.jobWizardAdvanced.selectDetectorByField(detector.byField); + } + if (isDetectorWithOverField(detector)) { + await ml.jobWizardAdvanced.selectDetectorOverField(detector.overField); + } + if (isDetectorWithPartitionField(detector)) { + await ml.jobWizardAdvanced.selectDetectorPartitionField(detector.partitionField); + } + if (isDetectorWithExcludeFrequent(detector)) { + await ml.jobWizardAdvanced.selectDetectorExcludeFrequent(detector.excludeFrequent); + } + if (isDetectorWithDescription(detector)) { + await ml.jobWizardAdvanced.setDetectorDescription(detector.description); + } + + await ml.jobWizardAdvanced.confirmAddDetectorModal(); + } + }); + + it('job creation displays detector entries', async () => { + for (const [index, detector] of testData.pickFieldsConfig.detectors.entries()) { + await ml.jobWizardAdvanced.assertDetectorEntryExists( + index, + detector.identifier, + isDetectorWithDescription(detector) ? detector.description : undefined + ); + } + }); + + it('job creation inputs the bucket span', async () => { + await ml.jobWizardCommon.assertBucketSpanInputExists(); + await ml.jobWizardCommon.setBucketSpan(testData.pickFieldsConfig.bucketSpan); + }); + + it('job creation inputs influencers', async () => { + await ml.jobWizardCommon.assertInfluencerInputExists(); + await ml.jobWizardCommon.assertInfluencerSelection([]); + for (const influencer of testData.pickFieldsConfig.influencers) { + await ml.jobWizardCommon.addInfluencer(influencer); + } + }); + + it('job creation inputs the model memory limit', async () => { + await ml.jobWizardCommon.assertModelMemoryLimitInputExists({ + withAdvancedSection: false, + }); + await ml.jobWizardCommon.setModelMemoryLimit(testData.pickFieldsConfig.memoryLimit, { + withAdvancedSection: false, + }); + }); + + it('job creation displays the job details step', async () => { + await ml.jobWizardCommon.advanceToJobDetailsSection(); + }); + + it('job creation inputs the job id', async () => { + await ml.jobWizardCommon.assertJobIdInputExists(); + await ml.jobWizardCommon.setJobId(testData.jobId); + }); + + it('job creation inputs the job description', async () => { + await ml.jobWizardCommon.assertJobDescriptionInputExists(); + await ml.jobWizardCommon.setJobDescription(testData.jobDescription); + }); + + it('job creation inputs job groups', async () => { + await ml.jobWizardCommon.assertJobGroupInputExists(); + for (const jobGroup of testData.jobGroups) { + await ml.jobWizardCommon.addJobGroup(jobGroup); + } + await ml.jobWizardCommon.assertJobGroupSelection(testData.jobGroups); + }); + + it('job creation opens the additional settings section', async () => { + await ml.jobWizardCommon.ensureAdditionalSettingsSectionOpen(); + }); + + it('job creation displays the model plot switch', async () => { + await ml.jobWizardCommon.assertModelPlotSwitchExists({ withAdvancedSection: false }); + }); + + it('job creation enables the dedicated index switch', async () => { + await ml.jobWizardCommon.assertDedicatedIndexSwitchExists({ withAdvancedSection: false }); + await ml.jobWizardCommon.activateDedicatedIndexSwitch({ withAdvancedSection: false }); + }); + + it('job creation displays the validation step', async () => { + await ml.jobWizardCommon.advanceToValidationSection(); + }); + + it('job creation displays the summary step', async () => { + await ml.jobWizardCommon.advanceToSummarySection(); + }); + + it('job creation creates the job and finishes processing', async () => { + await ml.jobWizardCommon.assertCreateJobButtonExists(); + await ml.jobWizardAdvanced.createJob(); + await ml.jobManagement.assertStartDatafeedModalExists(); + await ml.jobManagement.confirmStartDatafeedModal(); + await ml.jobManagement.waitForJobCompletion(testData.jobId); + }); + + it('job creation displays the created job in the job list', async () => { + await ml.jobTable.refreshJobList(); + await ml.jobTable.filterWithSearchString(testData.jobId); + const rows = await ml.jobTable.parseJobTable(); + expect(rows.filter(row => row.id === testData.jobId)).to.have.length(1); + }); + + it('job creation displays details for the created job in the job list', async () => { + await ml.jobTable.assertJobRowFields(testData.jobId, { + id: testData.jobId, + description: testData.jobDescription, + jobGroups: [...new Set(testData.jobGroups)].sort(), + recordCount: testData.expected.row.recordCount, + memoryStatus: testData.expected.row.memoryStatus, + jobState: testData.expected.row.jobState, + datafeedState: testData.expected.row.datafeedState, + latestTimestamp: testData.expected.row.latestTimestamp, + }); + + await ml.jobTable.assertJobRowDetailsCounts( + testData.jobId, + { + job_id: testData.jobId, + processed_record_count: testData.expected.counts.processed_record_count, + processed_field_count: testData.expected.counts.processed_field_count, + input_bytes: testData.expected.counts.input_bytes, + input_field_count: testData.expected.counts.input_field_count, + invalid_date_count: testData.expected.counts.invalid_date_count, + missing_field_count: testData.expected.counts.missing_field_count, + out_of_order_timestamp_count: testData.expected.counts.out_of_order_timestamp_count, + empty_bucket_count: testData.expected.counts.empty_bucket_count, + sparse_bucket_count: testData.expected.counts.sparse_bucket_count, + bucket_count: testData.expected.counts.bucket_count, + earliest_record_timestamp: testData.expected.counts.earliest_record_timestamp, + latest_record_timestamp: testData.expected.counts.latest_record_timestamp, + input_record_count: testData.expected.counts.input_record_count, + latest_bucket_timestamp: testData.expected.counts.latest_bucket_timestamp, + }, + { + job_id: testData.jobId, + result_type: testData.expected.modelSizeStats.result_type, + model_bytes_exceeded: testData.expected.modelSizeStats.model_bytes_exceeded, + model_bytes_memory_limit: testData.expected.modelSizeStats.model_bytes_memory_limit, + total_by_field_count: testData.expected.modelSizeStats.total_by_field_count, + total_over_field_count: testData.expected.modelSizeStats.total_over_field_count, + total_partition_field_count: + testData.expected.modelSizeStats.total_partition_field_count, + bucket_allocation_failures_count: + testData.expected.modelSizeStats.bucket_allocation_failures_count, + memory_status: testData.expected.modelSizeStats.memory_status, + timestamp: testData.expected.modelSizeStats.timestamp, + } + ); + }); + + it('job creation has detector results', async () => { + for (let i = 0; i < testData.pickFieldsConfig.detectors.length; i++) { + await ml.api.assertDetectorResultsExist(testData.jobId, i); + } + }); + }); + } + }); +} diff --git a/x-pack/test/functional/apps/machine_learning/anomaly_detection/index.ts b/x-pack/test/functional/apps/machine_learning/anomaly_detection/index.ts index 28e8b221cff4e..402c67589e285 100644 --- a/x-pack/test/functional/apps/machine_learning/anomaly_detection/index.ts +++ b/x-pack/test/functional/apps/machine_learning/anomaly_detection/index.ts @@ -17,5 +17,6 @@ export default function({ loadTestFile }: FtrProviderContext) { loadTestFile(require.resolve('./single_metric_viewer')); loadTestFile(require.resolve('./anomaly_explorer')); loadTestFile(require.resolve('./categorization_job')); + loadTestFile(require.resolve('./date_nanos_job')); }); } diff --git a/x-pack/test/functional/es_archives/ml/event_rate_nanos/data.json.gz b/x-pack/test/functional/es_archives/ml/event_rate_nanos/data.json.gz new file mode 100644 index 0000000000000..838b8d1872c0a Binary files /dev/null and b/x-pack/test/functional/es_archives/ml/event_rate_nanos/data.json.gz differ diff --git a/x-pack/test/functional/es_archives/ml/event_rate_nanos/mappings.json b/x-pack/test/functional/es_archives/ml/event_rate_nanos/mappings.json new file mode 100644 index 0000000000000..6897e05e75c2e --- /dev/null +++ b/x-pack/test/functional/es_archives/ml/event_rate_nanos/mappings.json @@ -0,0 +1,1477 @@ +{ + "type": "index", + "value": { + "aliases": { + }, + "index": "event_rate_gen_trend_nanos", + "mappings": { + "properties": { + "@timestamp": { + "format": "yyyy-MM-dd HH:mm:ss.SSSSSSSSSXX", + "type": "date_nanos" + }, + "count": { + "type": "integer" + } + } + }, + "settings": { + "index": { + "number_of_replicas": "0", + "number_of_shards": "1" + } + } + } +} + +{ + "type": "index", + "value": { + "aliases": { + ".kibana": { + } + }, + "index": ".kibana_1", + "mappings": { + "_meta": { + "migrationMappingPropertyHashes": { + "action": "c0c235fba02ebd2a2412bcda79009b58", + "action_task_params": "a9d49f184ee89641044be0ca2950fa3a", + "alert": "e588043a01d3d43477e7cad7efa0f5d8", + "apm-indices": "9bb9b2bf1fa636ed8619cbab5ce6a1dd", + "apm-services-telemetry": "07ee1939fa4302c62ddc052ec03fed90", + "canvas-element": "7390014e1091044523666d97247392fc", + "canvas-workpad": "b0a1706d356228dbdcb4a17e6b9eb231", + "config": "87aca8fdb053154f11383fce3dbf3edf", + "dashboard": "d00f614b29a80360e1190193fd333bab", + "file-upload-telemetry": "0ed4d3e1983d1217a30982630897092e", + "graph-workspace": "cd7ba1330e6682e9cc00b78850874be1", + "index-pattern": "66eccb05066c5a89924f48a9e9736499", + "infrastructure-ui-source": "ddc0ecb18383f6b26101a2fadb2dab0c", + "inventory-view": "84b320fd67209906333ffce261128462", + "kql-telemetry": "d12a98a6f19a2d273696597547e064ee", + "lens": "21c3ea0763beb1ecb0162529706b88c5", + "lens-ui-telemetry": "509bfa5978586998e05f9e303c07a327", + "map": "23d7aa4a720d4938ccde3983f87bd58d", + "maps-telemetry": "268da3a48066123fc5baf35abaa55014", + "metrics-explorer-view": "53c5365793677328df0ccb6138bf3cdd", + "migrationVersion": "4a1746014a75ade3a714e1db5763276f", + "ml-telemetry": "257fd1d4b4fdbb9cb4b8a3b27da201e9", + "namespace": "2f4316de49999235636386fe51dc06c1", + "query": "11aaeb7f5f7fa5bb43f25e18ce26e7d9", + "references": "7997cf5a56cc02bdc9c93361bde732b0", + "sample-data-telemetry": "7d3cfeb915303c9641c59681967ffeb4", + "search": "181661168bbadd1eff5902361e2a0d5c", + "server": "ec97f1c5da1a19609a60874e5af1100c", + "siem-detection-engine-rule-status": "0367e4d775814b56a4bee29384f9aafe", + "siem-ui-timeline": "ac8020190f5950dd3250b6499144e7fb", + "siem-ui-timeline-note": "8874706eedc49059d4cf0f5094559084", + "siem-ui-timeline-pinned-event": "20638091112f0e14f0e443d512301c29", + "space": "c5ca8acafa0beaa4d08d014a97b6bc6b", + "telemetry": "358ffaa88ba34a97d55af0933a117de4", + "timelion-sheet": "9a2a2748877c7a7b582fef201ab1d4cf", + "tsvb-validation-telemetry": "3a37ef6c8700ae6fc97d5c7da00e9215", + "type": "2f4316de49999235636386fe51dc06c1", + "ui-metric": "0d409297dc5ebe1e3a1da691c6ee32e3", + "updated_at": "00da57df13e94e9d98437d13ace4bfe0", + "upgrade-assistant-reindex-operation": "a53a20fe086b72c9a86da3cc12dad8a6", + "upgrade-assistant-telemetry": "56702cec857e0a9dacfb696655b4ff7b", + "url": "c7f66a0df8b1b52f17c28c4adb111105", + "visualization": "52d7a13ad68a150c4525b292d23e12cc" + } + }, + "dynamic": "strict", + "properties": { + "action": { + "properties": { + "actionTypeId": { + "type": "keyword" + }, + "config": { + "enabled": false, + "type": "object" + }, + "name": { + "type": "text" + }, + "secrets": { + "type": "binary" + } + } + }, + "action_task_params": { + "properties": { + "actionId": { + "type": "keyword" + }, + "apiKey": { + "type": "binary" + }, + "params": { + "enabled": false, + "type": "object" + } + } + }, + "alert": { + "properties": { + "actions": { + "properties": { + "actionRef": { + "type": "keyword" + }, + "actionTypeId": { + "type": "keyword" + }, + "group": { + "type": "keyword" + }, + "params": { + "enabled": false, + "type": "object" + } + }, + "type": "nested" + }, + "alertTypeId": { + "type": "keyword" + }, + "apiKey": { + "type": "binary" + }, + "apiKeyOwner": { + "type": "keyword" + }, + "consumer": { + "type": "keyword" + }, + "createdAt": { + "type": "date" + }, + "createdBy": { + "type": "keyword" + }, + "enabled": { + "type": "boolean" + }, + "muteAll": { + "type": "boolean" + }, + "mutedInstanceIds": { + "type": "keyword" + }, + "name": { + "type": "text" + }, + "params": { + "enabled": false, + "type": "object" + }, + "schedule": { + "properties": { + "interval": { + "type": "keyword" + } + } + }, + "scheduledTaskId": { + "type": "keyword" + }, + "tags": { + "type": "keyword" + }, + "throttle": { + "type": "keyword" + }, + "updatedBy": { + "type": "keyword" + } + } + }, + "apm-indices": { + "properties": { + "apm_oss": { + "properties": { + "errorIndices": { + "type": "keyword" + }, + "metricsIndices": { + "type": "keyword" + }, + "onboardingIndices": { + "type": "keyword" + }, + "sourcemapIndices": { + "type": "keyword" + }, + "spanIndices": { + "type": "keyword" + }, + "transactionIndices": { + "type": "keyword" + } + } + } + } + }, + "apm-services-telemetry": { + "properties": { + "has_any_services": { + "type": "boolean" + }, + "services_per_agent": { + "properties": { + "dotnet": { + "null_value": 0, + "type": "long" + }, + "go": { + "null_value": 0, + "type": "long" + }, + "java": { + "null_value": 0, + "type": "long" + }, + "js-base": { + "null_value": 0, + "type": "long" + }, + "nodejs": { + "null_value": 0, + "type": "long" + }, + "python": { + "null_value": 0, + "type": "long" + }, + "ruby": { + "null_value": 0, + "type": "long" + }, + "rum-js": { + "null_value": 0, + "type": "long" + } + } + } + } + }, + "canvas-element": { + "dynamic": "false", + "properties": { + "@created": { + "type": "date" + }, + "@timestamp": { + "type": "date" + }, + "content": { + "type": "text" + }, + "help": { + "type": "text" + }, + "image": { + "type": "text" + }, + "name": { + "fields": { + "keyword": { + "type": "keyword" + } + }, + "type": "text" + } + } + }, + "canvas-workpad": { + "dynamic": "false", + "properties": { + "@created": { + "type": "date" + }, + "@timestamp": { + "type": "date" + }, + "name": { + "fields": { + "keyword": { + "type": "keyword" + } + }, + "type": "text" + } + } + }, + "config": { + "dynamic": "true", + "properties": { + "buildNum": { + "type": "keyword" + }, + "dateFormat:tz": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "defaultIndex": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + } + } + }, + "dashboard": { + "properties": { + "description": { + "type": "text" + }, + "hits": { + "type": "integer" + }, + "kibanaSavedObjectMeta": { + "properties": { + "searchSourceJSON": { + "type": "text" + } + } + }, + "optionsJSON": { + "type": "text" + }, + "panelsJSON": { + "type": "text" + }, + "refreshInterval": { + "properties": { + "display": { + "type": "keyword" + }, + "pause": { + "type": "boolean" + }, + "section": { + "type": "integer" + }, + "value": { + "type": "integer" + } + } + }, + "timeFrom": { + "type": "keyword" + }, + "timeRestore": { + "type": "boolean" + }, + "timeTo": { + "type": "keyword" + }, + "title": { + "type": "text" + }, + "version": { + "type": "integer" + } + } + }, + "file-upload-telemetry": { + "properties": { + "filesUploadedTotalCount": { + "type": "long" + } + } + }, + "graph-workspace": { + "properties": { + "description": { + "type": "text" + }, + "kibanaSavedObjectMeta": { + "properties": { + "searchSourceJSON": { + "type": "text" + } + } + }, + "numLinks": { + "type": "integer" + }, + "numVertices": { + "type": "integer" + }, + "title": { + "type": "text" + }, + "version": { + "type": "integer" + }, + "wsState": { + "type": "text" + } + } + }, + "index-pattern": { + "properties": { + "fieldFormatMap": { + "type": "text" + }, + "fields": { + "type": "text" + }, + "intervalName": { + "type": "keyword" + }, + "notExpandable": { + "type": "boolean" + }, + "sourceFilters": { + "type": "text" + }, + "timeFieldName": { + "type": "keyword" + }, + "title": { + "type": "text" + }, + "type": { + "type": "keyword" + }, + "typeMeta": { + "type": "keyword" + } + } + }, + "infrastructure-ui-source": { + "properties": { + "description": { + "type": "text" + }, + "fields": { + "properties": { + "container": { + "type": "keyword" + }, + "host": { + "type": "keyword" + }, + "pod": { + "type": "keyword" + }, + "tiebreaker": { + "type": "keyword" + }, + "timestamp": { + "type": "keyword" + } + } + }, + "logAlias": { + "type": "keyword" + }, + "logColumns": { + "properties": { + "fieldColumn": { + "properties": { + "field": { + "type": "keyword" + }, + "id": { + "type": "keyword" + } + } + }, + "messageColumn": { + "properties": { + "id": { + "type": "keyword" + } + } + }, + "timestampColumn": { + "properties": { + "id": { + "type": "keyword" + } + } + } + }, + "type": "nested" + }, + "metricAlias": { + "type": "keyword" + }, + "name": { + "type": "text" + } + } + }, + "inventory-view": { + "properties": { + "autoBounds": { + "type": "boolean" + }, + "autoReload": { + "type": "boolean" + }, + "boundsOverride": { + "properties": { + "max": { + "type": "integer" + }, + "min": { + "type": "integer" + } + } + }, + "customOptions": { + "properties": { + "field": { + "type": "keyword" + }, + "text": { + "type": "keyword" + } + }, + "type": "nested" + }, + "filterQuery": { + "properties": { + "expression": { + "type": "keyword" + }, + "kind": { + "type": "keyword" + } + } + }, + "groupBy": { + "properties": { + "field": { + "type": "keyword" + }, + "label": { + "type": "keyword" + } + }, + "type": "nested" + }, + "metric": { + "properties": { + "type": { + "type": "keyword" + } + } + }, + "name": { + "type": "keyword" + }, + "nodeType": { + "type": "keyword" + }, + "time": { + "type": "integer" + }, + "view": { + "type": "keyword" + } + } + }, + "kql-telemetry": { + "properties": { + "optInCount": { + "type": "long" + }, + "optOutCount": { + "type": "long" + } + } + }, + "lens": { + "properties": { + "expression": { + "index": false, + "type": "keyword" + }, + "state": { + "type": "flattened" + }, + "title": { + "type": "text" + }, + "visualizationType": { + "type": "keyword" + } + } + }, + "lens-ui-telemetry": { + "properties": { + "count": { + "type": "integer" + }, + "date": { + "type": "date" + }, + "name": { + "type": "keyword" + }, + "type": { + "type": "keyword" + } + } + }, + "map": { + "properties": { + "bounds": { + "type": "geo_shape" + }, + "description": { + "type": "text" + }, + "layerListJSON": { + "type": "text" + }, + "mapStateJSON": { + "type": "text" + }, + "title": { + "type": "text" + }, + "uiStateJSON": { + "type": "text" + }, + "version": { + "type": "integer" + } + } + }, + "maps-telemetry": { + "properties": { + "attributesPerMap": { + "properties": { + "dataSourcesCount": { + "properties": { + "avg": { + "type": "long" + }, + "max": { + "type": "long" + }, + "min": { + "type": "long" + } + } + }, + "emsVectorLayersCount": { + "dynamic": "true", + "type": "object" + }, + "layerTypesCount": { + "dynamic": "true", + "type": "object" + }, + "layersCount": { + "properties": { + "avg": { + "type": "long" + }, + "max": { + "type": "long" + }, + "min": { + "type": "long" + } + } + } + } + }, + "indexPatternsWithGeoFieldCount": { + "type": "long" + }, + "mapsTotalCount": { + "type": "long" + }, + "settings": { + "properties": { + "showMapVisualizationTypes": { + "type": "boolean" + } + } + }, + "timeCaptured": { + "type": "date" + } + } + }, + "metrics-explorer-view": { + "properties": { + "chartOptions": { + "properties": { + "stack": { + "type": "boolean" + }, + "type": { + "type": "keyword" + }, + "yAxisMode": { + "type": "keyword" + } + } + }, + "currentTimerange": { + "properties": { + "from": { + "type": "keyword" + }, + "interval": { + "type": "keyword" + }, + "to": { + "type": "keyword" + } + } + }, + "name": { + "type": "keyword" + }, + "options": { + "properties": { + "aggregation": { + "type": "keyword" + }, + "filterQuery": { + "type": "keyword" + }, + "groupBy": { + "type": "keyword" + }, + "limit": { + "type": "integer" + }, + "metrics": { + "properties": { + "aggregation": { + "type": "keyword" + }, + "color": { + "type": "keyword" + }, + "field": { + "type": "keyword" + }, + "label": { + "type": "keyword" + } + }, + "type": "nested" + } + } + } + } + }, + "migrationVersion": { + "dynamic": "true", + "properties": { + "index-pattern": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "space": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + } + } + }, + "ml-telemetry": { + "properties": { + "file_data_visualizer": { + "properties": { + "index_creation_count": { + "type": "long" + } + } + } + } + }, + "namespace": { + "type": "keyword" + }, + "query": { + "properties": { + "description": { + "type": "text" + }, + "filters": { + "enabled": false, + "type": "object" + }, + "query": { + "properties": { + "language": { + "type": "keyword" + }, + "query": { + "index": false, + "type": "keyword" + } + } + }, + "timefilter": { + "enabled": false, + "type": "object" + }, + "title": { + "type": "text" + } + } + }, + "references": { + "properties": { + "id": { + "type": "keyword" + }, + "name": { + "type": "keyword" + }, + "type": { + "type": "keyword" + } + }, + "type": "nested" + }, + "sample-data-telemetry": { + "properties": { + "installCount": { + "type": "long" + }, + "unInstallCount": { + "type": "long" + } + } + }, + "search": { + "properties": { + "columns": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "hits": { + "type": "integer" + }, + "kibanaSavedObjectMeta": { + "properties": { + "searchSourceJSON": { + "type": "text" + } + } + }, + "sort": { + "type": "keyword" + }, + "title": { + "type": "text" + }, + "version": { + "type": "integer" + } + } + }, + "server": { + "properties": { + "uuid": { + "type": "keyword" + } + } + }, + "siem-detection-engine-rule-status": { + "properties": { + "alertId": { + "type": "keyword" + }, + "lastFailureAt": { + "type": "date" + }, + "lastFailureMessage": { + "type": "text" + }, + "lastSuccessAt": { + "type": "date" + }, + "lastSuccessMessage": { + "type": "text" + }, + "status": { + "type": "keyword" + }, + "statusDate": { + "type": "date" + } + } + }, + "siem-ui-timeline": { + "properties": { + "columns": { + "properties": { + "aggregatable": { + "type": "boolean" + }, + "category": { + "type": "keyword" + }, + "columnHeaderType": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "example": { + "type": "text" + }, + "id": { + "type": "keyword" + }, + "indexes": { + "type": "keyword" + }, + "name": { + "type": "text" + }, + "placeholder": { + "type": "text" + }, + "searchable": { + "type": "boolean" + }, + "type": { + "type": "keyword" + } + } + }, + "created": { + "type": "date" + }, + "createdBy": { + "type": "text" + }, + "dataProviders": { + "properties": { + "and": { + "properties": { + "enabled": { + "type": "boolean" + }, + "excluded": { + "type": "boolean" + }, + "id": { + "type": "keyword" + }, + "kqlQuery": { + "type": "text" + }, + "name": { + "type": "text" + }, + "queryMatch": { + "properties": { + "displayField": { + "type": "text" + }, + "displayValue": { + "type": "text" + }, + "field": { + "type": "text" + }, + "operator": { + "type": "text" + }, + "value": { + "type": "text" + } + } + } + } + }, + "enabled": { + "type": "boolean" + }, + "excluded": { + "type": "boolean" + }, + "id": { + "type": "keyword" + }, + "kqlQuery": { + "type": "text" + }, + "name": { + "type": "text" + }, + "queryMatch": { + "properties": { + "displayField": { + "type": "text" + }, + "displayValue": { + "type": "text" + }, + "field": { + "type": "text" + }, + "operator": { + "type": "text" + }, + "value": { + "type": "text" + } + } + } + } + }, + "dateRange": { + "properties": { + "end": { + "type": "date" + }, + "start": { + "type": "date" + } + } + }, + "description": { + "type": "text" + }, + "eventType": { + "type": "keyword" + }, + "favorite": { + "properties": { + "favoriteDate": { + "type": "date" + }, + "fullName": { + "type": "text" + }, + "keySearch": { + "type": "text" + }, + "userName": { + "type": "text" + } + } + }, + "filters": { + "properties": { + "exists": { + "type": "text" + }, + "match_all": { + "type": "text" + }, + "meta": { + "properties": { + "alias": { + "type": "text" + }, + "controlledBy": { + "type": "text" + }, + "disabled": { + "type": "boolean" + }, + "field": { + "type": "text" + }, + "formattedValue": { + "type": "text" + }, + "index": { + "type": "keyword" + }, + "key": { + "type": "keyword" + }, + "negate": { + "type": "boolean" + }, + "params": { + "type": "text" + }, + "type": { + "type": "keyword" + }, + "value": { + "type": "text" + } + } + }, + "missing": { + "type": "text" + }, + "query": { + "type": "text" + }, + "range": { + "type": "text" + }, + "script": { + "type": "text" + } + } + }, + "kqlMode": { + "type": "keyword" + }, + "kqlQuery": { + "properties": { + "filterQuery": { + "properties": { + "kuery": { + "properties": { + "expression": { + "type": "text" + }, + "kind": { + "type": "keyword" + } + } + }, + "serializedQuery": { + "type": "text" + } + } + } + } + }, + "savedQueryId": { + "type": "keyword" + }, + "sort": { + "properties": { + "columnId": { + "type": "keyword" + }, + "sortDirection": { + "type": "keyword" + } + } + }, + "title": { + "type": "text" + }, + "updated": { + "type": "date" + }, + "updatedBy": { + "type": "text" + } + } + }, + "siem-ui-timeline-note": { + "properties": { + "created": { + "type": "date" + }, + "createdBy": { + "type": "text" + }, + "eventId": { + "type": "keyword" + }, + "note": { + "type": "text" + }, + "timelineId": { + "type": "keyword" + }, + "updated": { + "type": "date" + }, + "updatedBy": { + "type": "text" + } + } + }, + "siem-ui-timeline-pinned-event": { + "properties": { + "created": { + "type": "date" + }, + "createdBy": { + "type": "text" + }, + "eventId": { + "type": "keyword" + }, + "timelineId": { + "type": "keyword" + }, + "updated": { + "type": "date" + }, + "updatedBy": { + "type": "text" + } + } + }, + "space": { + "properties": { + "_reserved": { + "type": "boolean" + }, + "color": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "disabledFeatures": { + "type": "keyword" + }, + "imageUrl": { + "index": false, + "type": "text" + }, + "initials": { + "type": "keyword" + }, + "name": { + "fields": { + "keyword": { + "ignore_above": 2048, + "type": "keyword" + } + }, + "type": "text" + } + } + }, + "telemetry": { + "properties": { + "enabled": { + "type": "boolean" + }, + "lastReported": { + "type": "date" + }, + "lastVersionChecked": { + "ignore_above": 256, + "type": "keyword" + }, + "sendUsageFrom": { + "ignore_above": 256, + "type": "keyword" + }, + "userHasSeenNotice": { + "type": "boolean" + } + } + }, + "timelion-sheet": { + "properties": { + "description": { + "type": "text" + }, + "hits": { + "type": "integer" + }, + "kibanaSavedObjectMeta": { + "properties": { + "searchSourceJSON": { + "type": "text" + } + } + }, + "timelion_chart_height": { + "type": "integer" + }, + "timelion_columns": { + "type": "integer" + }, + "timelion_interval": { + "type": "keyword" + }, + "timelion_other_interval": { + "type": "keyword" + }, + "timelion_rows": { + "type": "integer" + }, + "timelion_sheet": { + "type": "text" + }, + "title": { + "type": "text" + }, + "version": { + "type": "integer" + } + } + }, + "tsvb-validation-telemetry": { + "properties": { + "failedRequests": { + "type": "long" + } + } + }, + "type": { + "type": "keyword" + }, + "ui-metric": { + "properties": { + "count": { + "type": "integer" + } + } + }, + "updated_at": { + "type": "date" + }, + "upgrade-assistant-reindex-operation": { + "dynamic": "true", + "properties": { + "indexName": { + "type": "keyword" + }, + "status": { + "type": "integer" + } + } + }, + "upgrade-assistant-telemetry": { + "properties": { + "features": { + "properties": { + "deprecation_logging": { + "properties": { + "enabled": { + "null_value": true, + "type": "boolean" + } + } + } + } + }, + "ui_open": { + "properties": { + "cluster": { + "null_value": 0, + "type": "long" + }, + "indices": { + "null_value": 0, + "type": "long" + }, + "overview": { + "null_value": 0, + "type": "long" + } + } + }, + "ui_reindex": { + "properties": { + "close": { + "null_value": 0, + "type": "long" + }, + "open": { + "null_value": 0, + "type": "long" + }, + "start": { + "null_value": 0, + "type": "long" + }, + "stop": { + "null_value": 0, + "type": "long" + } + } + } + } + }, + "url": { + "properties": { + "accessCount": { + "type": "long" + }, + "accessDate": { + "type": "date" + }, + "createDate": { + "type": "date" + }, + "url": { + "fields": { + "keyword": { + "ignore_above": 2048, + "type": "keyword" + } + }, + "type": "text" + } + } + }, + "visualization": { + "properties": { + "description": { + "type": "text" + }, + "kibanaSavedObjectMeta": { + "properties": { + "searchSourceJSON": { + "type": "text" + } + } + }, + "savedSearchRefName": { + "type": "keyword" + }, + "title": { + "type": "text" + }, + "uiStateJSON": { + "type": "text" + }, + "version": { + "type": "integer" + }, + "visState": { + "type": "text" + } + } + } + } + }, + "settings": { + "index": { + "auto_expand_replicas": "0-1", + "number_of_replicas": "0", + "number_of_shards": "1" + } + } + } +} \ No newline at end of file