diff --git a/x-pack/plugins/ml/common/constants/anomalies.ts b/x-pack/plugins/ml/common/constants/anomalies.ts index d15033b738b0f..73a24bc11fe66 100644 --- a/x-pack/plugins/ml/common/constants/anomalies.ts +++ b/x-pack/plugins/ml/common/constants/anomalies.ts @@ -22,3 +22,5 @@ export enum ANOMALY_THRESHOLD { } export const PARTITION_FIELDS = ['partition_field', 'over_field', 'by_field'] as const; +export const JOB_ID = 'job_id'; +export const PARTITION_FIELD_VALUE = 'partition_field_value'; diff --git a/x-pack/plugins/ml/common/constants/messages.ts b/x-pack/plugins/ml/common/constants/messages.ts index b42eacbf7df5c..a9e4cdc4a0434 100644 --- a/x-pack/plugins/ml/common/constants/messages.ts +++ b/x-pack/plugins/ml/common/constants/messages.ts @@ -43,6 +43,34 @@ export const getMessages = once(() => { const createJobsDocsUrl = `https://www.elastic.co/guide/en/machine-learning/{{version}}/create-jobs.html`; return { + categorizer_detector_missing_per_partition_field: { + status: VALIDATION_STATUS.ERROR, + text: i18n.translate( + 'xpack.ml.models.jobValidation.messages.categorizerMissingPerPartitionFieldMessage', + { + defaultMessage: + 'Partition field must be set for detectors that reference "mlcategory" when per-partition categorization is enabled.', + } + ), + url: + 'https://www.elastic.co/guide/en/machine-learning/{{version}}/ml-configuring-categories.html', + }, + categorizer_varying_per_partition_fields: { + status: VALIDATION_STATUS.ERROR, + text: i18n.translate( + 'xpack.ml.models.jobValidation.messages.categorizerVaryingPerPartitionFieldNamesMessage', + { + defaultMessage: + 'Detectors with keyword "mlcategory" cannot have different partition_field_name when per-partition categorization is enabled. Found [{fields}].', + + values: { + fields: '"{{fields}}"', + }, + } + ), + url: + 'https://www.elastic.co/guide/en/machine-learning/{{version}}/ml-configuring-categories.html', + }, field_not_aggregatable: { status: VALIDATION_STATUS.ERROR, text: i18n.translate('xpack.ml.models.jobValidation.messages.fieldNotAggregatableMessage', { diff --git a/x-pack/plugins/ml/common/types/anomalies.ts b/x-pack/plugins/ml/common/types/anomalies.ts index a23886e8fcdc6..703d74d1bd5e5 100644 --- a/x-pack/plugins/ml/common/types/anomalies.ts +++ b/x-pack/plugins/ml/common/types/anomalies.ts @@ -57,3 +57,20 @@ export interface AnomaliesTableRecord { } export type PartitionFieldsType = typeof PARTITION_FIELDS[number]; + +export interface AnomalyCategorizerStatsDoc { + [key: string]: any; + job_id: string; + result_type: 'categorizer_stats'; + partition_field_name?: string; + partition_field_value?: string; + categorized_doc_count: number; + total_category_count: number; + frequent_category_count: number; + rare_category_count: number; + dead_category_count: number; + failed_category_count: number; + categorization_status: 'ok' | 'warn'; + log_time: number; + timestamp: number; +} diff --git a/x-pack/plugins/ml/common/types/anomaly_detection_jobs/job.ts b/x-pack/plugins/ml/common/types/anomaly_detection_jobs/job.ts index 744f9c4d759dd..48fd4ec914d07 100644 --- a/x-pack/plugins/ml/common/types/anomaly_detection_jobs/job.ts +++ b/x-pack/plugins/ml/common/types/anomaly_detection_jobs/job.ts @@ -93,6 +93,6 @@ export interface CustomRule { } export interface PerPartitionCategorization { - enabled: boolean; + enabled?: boolean; stop_on_warn?: boolean; } diff --git a/x-pack/plugins/ml/common/types/results.ts b/x-pack/plugins/ml/common/types/results.ts new file mode 100644 index 0000000000000..c761620589080 --- /dev/null +++ b/x-pack/plugins/ml/common/types/results.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export interface GetStoppedPartitionResult { + jobs: string[] | Record; +} diff --git a/x-pack/plugins/ml/common/util/job_utils.ts b/x-pack/plugins/ml/common/util/job_utils.ts index 8e6933ed5924f..9729240567c24 100644 --- a/x-pack/plugins/ml/common/util/job_utils.ts +++ b/x-pack/plugins/ml/common/util/job_utils.ts @@ -23,6 +23,7 @@ import { EntityField } from './anomaly_utils'; import { MlServerLimits } from '../types/ml_server_info'; import { JobValidationMessage, JobValidationMessageId } from '../constants/messages'; import { ES_AGGREGATION, ML_JOB_AGGREGATION } from '../constants/aggregation_types'; +import { MLCATEGORY } from '../constants/field_types'; export interface ValidationResults { valid: boolean; @@ -86,9 +87,9 @@ export function isSourceDataChartableForDetector(job: CombinedJob, detectorIndex // whereas the 'function_description' field holds an ML-built display hint for function e.g. 'count'. isSourceDataChartable = mlFunctionToESAggregation(functionName) !== null && - dtr.by_field_name !== 'mlcategory' && - dtr.partition_field_name !== 'mlcategory' && - dtr.over_field_name !== 'mlcategory'; + dtr.by_field_name !== MLCATEGORY && + dtr.partition_field_name !== MLCATEGORY && + dtr.over_field_name !== MLCATEGORY; // If the datafeed uses script fields, we can only plot the time series if // model plot is enabled. Without model plot it will be very difficult or impossible @@ -380,16 +381,25 @@ export function basicJobValidation( valid = false; } } - + let categorizerDetectorMissingPartitionField = false; if (job.analysis_config.detectors.length === 0) { messages.push({ id: 'detectors_empty' }); valid = false; } else { let v = true; + each(job.analysis_config.detectors, (d) => { if (isEmpty(d.function)) { v = false; } + // if detector has an ml category, check if the partition_field is missing + const needToHavePartitionFieldName = + job.analysis_config.per_partition_categorization?.enabled === true && + (d.by_field_name === MLCATEGORY || d.over_field_name === MLCATEGORY); + + if (needToHavePartitionFieldName && d.partition_field_name === undefined) { + categorizerDetectorMissingPartitionField = true; + } }); if (v) { messages.push({ id: 'detectors_function_not_empty' }); @@ -397,10 +407,46 @@ export function basicJobValidation( messages.push({ id: 'detectors_function_empty' }); valid = false; } + if (categorizerDetectorMissingPartitionField) { + messages.push({ id: 'categorizer_detector_missing_per_partition_field' }); + valid = false; + } } - // check for duplicate detectors if (job.analysis_config.detectors.length >= 2) { + // check if the detectors with mlcategory might have different per_partition_field values + // if per_partition_categorization is enabled + if (job.analysis_config.per_partition_categorization !== undefined) { + if ( + job.analysis_config.per_partition_categorization.enabled || + (job.analysis_config.per_partition_categorization.stop_on_warn && + Array.isArray(job.analysis_config.detectors) && + job.analysis_config.detectors.length >= 2) + ) { + const categorizationDetectors = job.analysis_config.detectors.filter( + (d) => + d.by_field_name === MLCATEGORY || + d.over_field_name === MLCATEGORY || + d.partition_field_name === MLCATEGORY + ); + const uniqPartitions = [ + ...new Set( + categorizationDetectors + .map((d) => d.partition_field_name) + .filter((name) => name !== undefined) + ), + ]; + if (uniqPartitions.length > 1) { + valid = false; + messages.push({ + id: 'categorizer_varying_per_partition_fields', + fields: uniqPartitions.join(', '), + }); + } + } + } + + // check for duplicate detectors // create an array of objects with a subset of the attributes // where we want to make sure they are not be the same across detectors const compareSubSet = job.analysis_config.detectors.map((d) => diff --git a/x-pack/plugins/ml/public/application/explorer/explorer.js b/x-pack/plugins/ml/public/application/explorer/explorer.js index 06cec14578f2a..d78df80fad94e 100644 --- a/x-pack/plugins/ml/public/application/explorer/explorer.js +++ b/x-pack/plugins/ml/public/application/explorer/explorer.js @@ -205,7 +205,7 @@ export class Explorer extends React.Component { updateLanguage = (language) => this.setState({ language }); render() { - const { showCharts, severity } = this.props; + const { showCharts, severity, stoppedPartitions } = this.props; const { annotations, @@ -298,6 +298,23 @@ export class Explorer extends React.Component {
+ + {stoppedPartitions && ( + + } + /> + )} + { + delete detector.partition_field_name; + }); + if (this._partitionFieldName !== null) this.removeInfluencer(this._partitionFieldName); + this._partitionFieldName = null; + } else { + if (this._partitionFieldName !== fieldName) { + // remove the previous field from list of influencers + // and add the new one + if (this._partitionFieldName !== null) this.removeInfluencer(this._partitionFieldName); + this.addInfluencer(fieldName); + this._partitionFieldName = fieldName; + this._detectors.forEach((detector) => { + detector.partition_field_name = fieldName; + }); + } + } + } } diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/common/job_creator/job_creator.ts b/x-pack/plugins/ml/public/application/jobs/new_job/common/job_creator/job_creator.ts index 29e8aafffef7e..4c030a22f54f7 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/common/job_creator/job_creator.ts +++ b/x-pack/plugins/ml/public/application/jobs/new_job/common/job_creator/job_creator.ts @@ -622,6 +622,36 @@ export class JobCreator { return JSON.stringify(this._datafeed_config, null, 2); } + private _initPerPartitionCategorization() { + if (this._job_config.analysis_config.per_partition_categorization === undefined) { + this._job_config.analysis_config.per_partition_categorization = {}; + } + if (this._job_config.analysis_config.per_partition_categorization?.enabled === undefined) { + this._job_config.analysis_config.per_partition_categorization!.enabled = false; + } + if (this._job_config.analysis_config.per_partition_categorization?.stop_on_warn === undefined) { + this._job_config.analysis_config.per_partition_categorization!.stop_on_warn = false; + } + } + + public get perPartitionCategorization() { + return this._job_config.analysis_config.per_partition_categorization?.enabled === true; + } + + public set perPartitionCategorization(enabled: boolean) { + this._initPerPartitionCategorization(); + this._job_config.analysis_config.per_partition_categorization!.enabled = enabled; + } + + public get perPartitionStopOnWarn() { + return this._job_config.analysis_config.per_partition_categorization?.stop_on_warn === true; + } + + public set perPartitionStopOnWarn(enabled: boolean) { + this._initPerPartitionCategorization(); + this._job_config.analysis_config.per_partition_categorization!.stop_on_warn = enabled; + } + protected _overrideConfigs(job: Job, datafeed: Datafeed) { this._job_config = job; this._datafeed_config = datafeed; diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/common/job_validator/job_validator.ts b/x-pack/plugins/ml/public/application/jobs/new_job/common/job_validator/job_validator.ts index 242a643ddd3ce..635322a6c4469 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/common/job_validator/job_validator.ts +++ b/x-pack/plugins/ml/public/application/jobs/new_job/common/job_validator/job_validator.ts @@ -51,6 +51,8 @@ export interface BasicValidations { queryDelay: Validation; frequency: Validation; scrollSize: Validation; + categorizerMissingPerPartition: Validation; + categorizerVaryingPerPartitionField: Validation; } export interface AdvancedValidations { @@ -76,6 +78,8 @@ export class JobValidator { queryDelay: { valid: true }, frequency: { valid: true }, scrollSize: { valid: true }, + categorizerMissingPerPartition: { valid: true }, + categorizerVaryingPerPartitionField: { valid: true }, }; private _advancedValidations: AdvancedValidations = { categorizationFieldValid: { valid: true }, @@ -273,6 +277,14 @@ export class JobValidator { this._advancedValidations.categorizationFieldValid.valid = valid; } + public get categorizerMissingPerPartition() { + return this._basicValidations.categorizerMissingPerPartition; + } + + public get categorizerVaryingPerPartitionField() { + return this._basicValidations.categorizerVaryingPerPartitionField; + } + /** * Indicates if the Pick Fields step has a valid input */ @@ -283,6 +295,8 @@ export class JobValidator { (this._jobCreator.type === JOB_TYPE.ADVANCED && this.modelMemoryLimit.valid)) && this.bucketSpan.valid && this.duplicateDetectors.valid && + this.categorizerMissingPerPartition.valid && + this.categorizerVaryingPerPartitionField.valid && !this.validating && (this._jobCreator.type !== JOB_TYPE.CATEGORIZATION || (this._jobCreator.type === JOB_TYPE.CATEGORIZATION && this.categorizationField)) diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/common/job_validator/util.ts b/x-pack/plugins/ml/public/application/jobs/new_job/common/job_validator/util.ts index b97841542f76a..1ce81bf0dcdf0 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/common/job_validator/util.ts +++ b/x-pack/plugins/ml/public/application/jobs/new_job/common/job_validator/util.ts @@ -130,6 +130,29 @@ export function populateValidationMessages( basicValidations.duplicateDetectors.message = msg; } + if (validationResults.contains('categorizer_detector_missing_per_partition_field')) { + basicValidations.categorizerMissingPerPartition.valid = false; + const msg = i18n.translate( + 'xpack.ml.newJob.wizard.validateJob.categorizerMissingPerPartitionFieldMessage', + { + defaultMessage: + 'Partition field must be set for detectors that reference "mlcategory" when per-partition categorization is enabled.', + } + ); + basicValidations.categorizerMissingPerPartition.message = msg; + } + if (validationResults.contains('categorizer_varying_per_partition_fields')) { + basicValidations.categorizerVaryingPerPartitionField.valid = false; + const msg = i18n.translate( + 'xpack.ml.newJob.wizard.validateJob.categorizerVaryingPerPartitionFieldNamesMessage', + { + defaultMessage: + 'Detectors with keyword "mlcategory" cannot have different partition_field_name when per-partition categorization is enabled.', + } + ); + basicValidations.categorizerVaryingPerPartitionField.message = msg; + } + if (validationResults.contains('bucket_span_empty')) { basicValidations.bucketSpan.valid = false; const msg = i18n.translate( diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/advanced_view/detector_list.tsx b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/advanced_view/detector_list.tsx index 38903dd4845a6..ee4bd73755ebe 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/advanced_view/detector_list.tsx +++ b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/advanced_view/detector_list.tsx @@ -46,7 +46,15 @@ export const DetectorList: FC = ({ isActive, onEditJob, onDeleteJob }) => }, [jobCreatorUpdated]); useEffect(() => { - setValidation(jobValidator.duplicateDetectors); + if (!jobValidator.duplicateDetectors.valid) { + setValidation(jobValidator.duplicateDetectors); + } + if (!jobValidator.categorizerVaryingPerPartitionField.valid) { + setValidation(jobValidator.categorizerVaryingPerPartitionField); + } + if (!jobValidator.categorizerMissingPerPartition.valid) { + setValidation(jobValidator.categorizerMissingPerPartition); + } }, [jobValidatorUpdated]); const Buttons: FC<{ index: number }> = ({ index }) => { @@ -129,7 +137,7 @@ export const DetectorList: FC = ({ isActive, onEditJob, onDeleteJob }) => ))} - + ); }; @@ -159,7 +167,7 @@ const NoDetectorsWarning: FC<{ show: boolean }> = ({ show }) => { ); }; -const DuplicateDetectorsWarning: FC<{ validation: Validation }> = ({ validation }) => { +const DetectorsValidationWarning: FC<{ validation: Validation }> = ({ validation }) => { if (validation.valid === true) { return null; } diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/advanced_view/extra.tsx b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/advanced_view/extra.tsx index c29ebce41593c..c619f8e4e02dd 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/advanced_view/extra.tsx +++ b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/advanced_view/extra.tsx @@ -4,13 +4,19 @@ * you may not use this file except in compliance with the Elastic License. */ -import React, { Fragment, FC } from 'react'; +import React, { Fragment, FC, useContext } from 'react'; import { EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; import { SummaryCountField } from '../summary_count_field'; import { CategorizationField } from '../categorization_field'; +import { CategorizationPerPartitionField } from '../categorization_partition_field'; +import { JobCreatorContext } from '../../../job_creator_context'; +import { isAdvancedJobCreator } from '../../../../../common/job_creator'; export const ExtraSettings: FC = () => { + const { jobCreator } = useContext(JobCreatorContext); + const showCategorizationPerPartitionField = + isAdvancedJobCreator(jobCreator) && jobCreator.categorizationFieldName !== null; return ( @@ -21,6 +27,7 @@ export const ExtraSettings: FC = () => { + {showCategorizationPerPartitionField && } ); }; diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_field/categorization_field_select.tsx b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_field/categorization_field_select.tsx index 2f3e8d43bc169..d4c96025f9e36 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_field/categorization_field_select.tsx +++ b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_field/categorization_field_select.tsx @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import React, { FC, useContext } from 'react'; +import React, { FC, useCallback, useContext, useMemo } from 'react'; import { EuiComboBox, EuiComboBoxOptionOption } from '@elastic/eui'; import { JobCreatorContext } from '../../../job_creator_context'; @@ -18,24 +18,25 @@ interface Props { } export const CategorizationFieldSelect: FC = ({ fields, changeHandler, selectedField }) => { - const { jobCreator } = useContext(JobCreatorContext); - const options: EuiComboBoxOptionOption[] = [ - ...createFieldOptions(fields, jobCreator.additionalFields), - ]; - - const selection: EuiComboBoxOptionOption[] = []; - if (selectedField !== null) { - selection.push({ label: selectedField }); - } + const { jobCreator, jobCreatorUpdated } = useContext(JobCreatorContext); + const options: EuiComboBoxOptionOption[] = useMemo( + () => [...createFieldOptions(fields, jobCreator.additionalFields)], + [fields, jobCreatorUpdated] + ); - function onChange(selectedOptions: EuiComboBoxOptionOption[]) { - const option = selectedOptions[0]; - if (typeof option !== 'undefined') { - changeHandler(option.label); - } else { - changeHandler(null); + const selection: EuiComboBoxOptionOption[] = useMemo(() => { + const selectedOptions: EuiComboBoxOptionOption[] = []; + if (selectedField !== null) { + selectedOptions.push({ label: selectedField }); } - } + return selectedOptions; + }, [selectedField]); + + const onChange = useCallback( + (selectedOptions: EuiComboBoxOptionOption[]) => + changeHandler((selectedOptions[0] && selectedOptions[0].label) ?? null), + [changeHandler] + ); return ( { + const { jobCreator: jc, jobCreatorUpdated } = useContext(JobCreatorContext); + const jobCreator = jc as AdvancedJobCreator | CategorizationJobCreator; + const [enablePerPartitionCategorization, setEnablePerPartitionCategorization] = useState(false); + useEffect(() => { + setEnablePerPartitionCategorization(jobCreator.perPartitionCategorization); + }, [jobCreatorUpdated]); + + return ( + + + } + > + + + + {enablePerPartitionCategorization && ( + <> + + } + > + + + + )} + {isCategorizationJobCreator(jobCreator) && enablePerPartitionCategorization && ( + + )} + + ); +}; diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_partition_field/categorization_per_partition_dropdown.tsx b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_partition_field/categorization_per_partition_dropdown.tsx new file mode 100644 index 0000000000000..a0eccf914af6e --- /dev/null +++ b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_partition_field/categorization_per_partition_dropdown.tsx @@ -0,0 +1,67 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { Dispatch, SetStateAction, useContext, useEffect, useState, useMemo } from 'react'; +import { EuiFormRow } from '@elastic/eui'; +import { FormattedMessage } from '@kbn/i18n/react'; +import { JobCreatorContext } from '../../../job_creator_context'; +import { CategorizationJobCreator } from '../../../../../common/job_creator'; +import { newJobCapsService } from '../../../../../../../services/new_job_capabilities_service'; +import { CategorizationPerPartitionFieldSelect } from './categorization_per_partition_input'; + +export const CategorizationPerPartitionFieldDropdown = ({ + setEnablePerPartitionCategorization, +}: { + setEnablePerPartitionCategorization: Dispatch>; +}) => { + const { jobCreator: jc, jobCreatorUpdate, jobCreatorUpdated } = useContext(JobCreatorContext); + const jobCreator = jc as CategorizationJobCreator; + + const [categorizationPartitionFieldName, setCategorizationPartitionFieldName] = useState< + string | null + >(jobCreator.categorizationPerPartitionField); + const { categoryFields } = newJobCapsService; + + const filteredCategories = useMemo( + () => categoryFields.filter((c) => c.id !== jobCreator.categorizationFieldName), + [categoryFields, jobCreatorUpdated] + ); + useEffect(() => { + jobCreator.categorizationPerPartitionField = categorizationPartitionFieldName; + jobCreatorUpdate(); + }, [categorizationPartitionFieldName]); + + useEffect(() => { + // set the first item in category as partition field by default + // because API requires partition_field to be defined in each detector with mlcategory + // if per-partition categorization is enabled + if ( + jobCreator.perPartitionCategorization && + jobCreator.categorizationPerPartitionField === null && + filteredCategories.length > 0 + ) { + jobCreator.categorizationPerPartitionField = filteredCategories[0].id; + } + setCategorizationPartitionFieldName(jobCreator.categorizationPerPartitionField); + setEnablePerPartitionCategorization(jobCreator.perPartitionCategorization); + }, [jobCreatorUpdated]); + return ( + + } + > + + + ); +}; diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_partition_field/categorization_per_partition_input.tsx b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_partition_field/categorization_per_partition_input.tsx new file mode 100644 index 0000000000000..ec398a810a848 --- /dev/null +++ b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_partition_field/categorization_per_partition_input.tsx @@ -0,0 +1,55 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { FC, useCallback, useContext, useMemo } from 'react'; +import { EuiComboBox, EuiComboBoxOptionOption } from '@elastic/eui'; + +import { JobCreatorContext } from '../../../job_creator_context'; +import { Field } from '../../../../../../../../../common/types/fields'; +import { createFieldOptions } from '../../../../../common/job_creator/util/general'; + +interface Props { + fields: Field[]; + changeHandler(i: string | null): void; + selectedField: string | null; +} + +export const CategorizationPerPartitionFieldSelect: FC = ({ + fields, + changeHandler, + selectedField, +}) => { + const { jobCreator, jobCreatorUpdated } = useContext(JobCreatorContext); + const options: EuiComboBoxOptionOption[] = useMemo( + () => [...createFieldOptions(fields, jobCreator.additionalFields)], + [fields, jobCreatorUpdated] + ); + + const selection: EuiComboBoxOptionOption[] = useMemo(() => { + const selectedOptions: EuiComboBoxOptionOption[] = []; + if (selectedField !== null) { + selectedOptions.push({ label: selectedField }); + } + return selectedOptions; + }, [selectedField]); + + const onChange = useCallback( + (selectedOptions: EuiComboBoxOptionOption[]) => + changeHandler((selectedOptions[0] && selectedOptions[0].label) ?? null), + [changeHandler] + ); + + return ( + + ); +}; diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_partition_field/categorization_per_partition_switch.tsx b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_partition_field/categorization_per_partition_switch.tsx new file mode 100644 index 0000000000000..4e2c1300b7937 --- /dev/null +++ b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_partition_field/categorization_per_partition_switch.tsx @@ -0,0 +1,54 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { FC, useContext, useEffect, useCallback, useState } from 'react'; +import { FormattedMessage } from '@kbn/i18n/react'; +import { EuiSwitch } from '@elastic/eui'; +import { JobCreatorContext } from '../../../job_creator_context'; +import { AdvancedJobCreator, CategorizationJobCreator } from '../../../../../common/job_creator'; + +export const CategorizationPerPartitionSwitch: FC = () => { + const { jobCreator: jc, jobCreatorUpdate, jobCreatorUpdated } = useContext(JobCreatorContext); + const jobCreator = jc as AdvancedJobCreator | CategorizationJobCreator; + const [enablePerPartitionCategorization, setEnablePerPartitionCategorization] = useState( + jobCreator.perPartitionCategorization + ); + + const toggleEnablePerPartitionCategorization = useCallback( + () => setEnablePerPartitionCategorization(!enablePerPartitionCategorization), + [enablePerPartitionCategorization] + ); + + useEffect(() => { + setEnablePerPartitionCategorization(jobCreator.perPartitionCategorization); + }, [jobCreatorUpdated]); + + useEffect(() => { + // also turn off stop on warn if per_partition_categorization is turned off + if (enablePerPartitionCategorization === false) { + jobCreator.perPartitionStopOnWarn = false; + } + + jobCreator.perPartitionCategorization = enablePerPartitionCategorization; + jobCreatorUpdate(); + }, [enablePerPartitionCategorization]); + + return ( + + } + /> + ); +}; diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_partition_field/categorization_stop_on_warn_switch.tsx b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_partition_field/categorization_stop_on_warn_switch.tsx new file mode 100644 index 0000000000000..8bbf03a999f88 --- /dev/null +++ b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_partition_field/categorization_stop_on_warn_switch.tsx @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { FC, useCallback, useContext, useEffect, useState } from 'react'; +import { FormattedMessage } from '@kbn/i18n/react'; +import { EuiSwitch } from '@elastic/eui'; +import { JobCreatorContext } from '../../../job_creator_context'; +import { AdvancedJobCreator, CategorizationJobCreator } from '../../../../../common/job_creator'; + +export const CategorizationPerPartitionStopOnWarnSwitch: FC = () => { + const { jobCreator: jc, jobCreatorUpdate, jobCreatorUpdated } = useContext(JobCreatorContext); + const jobCreator = jc as AdvancedJobCreator | CategorizationJobCreator; + const [stopOnWarn, setStopOnWarn] = useState(jobCreator.perPartitionStopOnWarn); + + const toggleStopOnWarn = useCallback(() => setStopOnWarn(!stopOnWarn), [stopOnWarn]); + + useEffect(() => { + jobCreator.perPartitionStopOnWarn = stopOnWarn; + jobCreatorUpdate(); + }, [stopOnWarn]); + + useEffect(() => { + setStopOnWarn(jobCreator.perPartitionStopOnWarn); + }, [jobCreatorUpdated]); + + return ( + + } + /> + ); +}; diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_partition_field/description.tsx b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_partition_field/description.tsx new file mode 100644 index 0000000000000..01369ca425391 --- /dev/null +++ b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_partition_field/description.tsx @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { memo, FC } from 'react'; +import { i18n } from '@kbn/i18n'; +import { FormattedMessage } from '@kbn/i18n/react'; +import { EuiDescribedFormGroup } from '@elastic/eui'; + +interface Props { + children: React.ReactNode; +} +export const Description: FC = memo(({ children }) => { + const title = i18n.translate('xpack.ml.newJob.wizard.perPartitionCategorization.enable.title', { + defaultMessage: 'Enable per-partition categorization', + }); + return ( + {title}} + description={ + + } + > + <>{children} + + ); +}); diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_partition_field/index.ts b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_partition_field/index.ts new file mode 100644 index 0000000000000..f3b50a19c021e --- /dev/null +++ b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_partition_field/index.ts @@ -0,0 +1,6 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +export { CategorizationPerPartitionField } from './categorization_per_partition'; diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/metric_selection.tsx b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/metric_selection.tsx index f5c3e90d63418..cbbddb5bbc5b8 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/metric_selection.tsx +++ b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/metric_selection.tsx @@ -12,6 +12,8 @@ import { JobCreatorContext } from '../../../job_creator_context'; import { CategorizationJobCreator } from '../../../../../common/job_creator'; import { CategorizationField } from '../categorization_field'; import { CategorizationDetector } from '../categorization_detector'; +import { CategorizationPerPartitionField } from '../categorization_partition_field'; + import { FieldExamples } from './field_examples'; import { ExamplesValidCallout } from './examples_valid_callout'; import { @@ -126,6 +128,8 @@ export const CategorizationDetectors: FC = ({ setIsValid }) => { )} + + ); }; diff --git a/x-pack/plugins/ml/public/application/routing/routes/explorer.tsx b/x-pack/plugins/ml/public/application/routing/routes/explorer.tsx index a2030776773a9..62d7e82a214b5 100644 --- a/x-pack/plugins/ml/public/application/routing/routes/explorer.tsx +++ b/x-pack/plugins/ml/public/application/routing/routes/explorer.tsx @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import React, { FC, useEffect, useState } from 'react'; +import React, { FC, useEffect, useState, useCallback } from 'react'; import useObservable from 'react-use/lib/useObservable'; import { i18n } from '@kbn/i18n'; @@ -32,6 +32,7 @@ import { useUrlState } from '../../util/url_state'; import { getBreadcrumbWithUrlForApp } from '../breadcrumbs'; import { useTimefilter } from '../../contexts/kibana'; import { isViewBySwimLaneData } from '../../explorer/swimlane_container'; +import { JOB_ID } from '../../../../common/constants/anomalies'; export const explorerRouteFactory = (navigateToPath: NavigateToPath): MlRoute => ({ path: '/explorer', @@ -70,6 +71,8 @@ const ExplorerUrlStateManager: FC = ({ jobsWithTim const [appState, setAppState] = useUrlState('_a'); const [globalState, setGlobalState] = useUrlState('_g'); const [lastRefresh, setLastRefresh] = useState(0); + const [stoppedPartitions, setStoppedPartitions] = useState(); + const timefilter = useTimefilter({ timeRangeSelector: true, autoRefreshSelector: true }); const { jobIds } = useJobSelection(jobsWithTimeRange); @@ -109,9 +112,31 @@ const ExplorerUrlStateManager: FC = ({ jobsWithTim } }, [globalState?.time?.from, globalState?.time?.to]); + const getJobsWithStoppedPartitions = useCallback(async (selectedJobIds: string[]) => { + try { + const fetchedStoppedPartitions = await ml.results.getCategoryStoppedPartitions( + selectedJobIds, + JOB_ID + ); + if ( + fetchedStoppedPartitions && + Array.isArray(fetchedStoppedPartitions.jobs) && + fetchedStoppedPartitions.jobs.length > 0 + ) { + setStoppedPartitions(fetchedStoppedPartitions.jobs); + } else { + setStoppedPartitions(undefined); + } + } catch (error) { + // eslint-disable-next-line no-console + console.error(error); + } + }, []); + useEffect(() => { if (jobIds.length > 0) { explorerService.updateJobSelection(jobIds); + getJobsWithStoppedPartitions(jobIds); } else { explorerService.clearJobs(); } @@ -209,6 +234,7 @@ const ExplorerUrlStateManager: FC = ({ jobsWithTim setSelectedCells, showCharts, severity: tableSeverity.val, + stoppedPartitions, }} />
diff --git a/x-pack/plugins/ml/public/application/services/ml_api_service/results.ts b/x-pack/plugins/ml/public/application/services/ml_api_service/results.ts index 08c3853ace6f8..65bd4fb1eccc2 100644 --- a/x-pack/plugins/ml/public/application/services/ml_api_service/results.ts +++ b/x-pack/plugins/ml/public/application/services/ml_api_service/results.ts @@ -5,11 +5,11 @@ */ // Service for obtaining data for the ML Results dashboards. +import { GetStoppedPartitionResult } from '../../../../common/types/results'; import { HttpService } from '../http_service'; - import { basePath } from './index'; - import { JobId } from '../../../../common/types/anomaly_detection_jobs'; +import { JOB_ID, PARTITION_FIELD_VALUE } from '../../../../common/constants/anomalies'; import { PartitionFieldsDefinition } from '../results_service/result_service_rx'; export const resultsApiProvider = (httpService: HttpService) => ({ @@ -114,4 +114,19 @@ export const resultsApiProvider = (httpService: HttpService) => ({ body, }); }, + + getCategoryStoppedPartitions( + jobIds: string[], + fieldToBucket?: typeof JOB_ID | typeof PARTITION_FIELD_VALUE + ) { + const body = JSON.stringify({ + jobIds, + fieldToBucket, + }); + return httpService.http({ + path: `${basePath()}/results/category_stopped_partitions`, + method: 'POST', + body, + }); + }, }); diff --git a/x-pack/plugins/ml/server/models/job_validation/job_validation.ts b/x-pack/plugins/ml/server/models/job_validation/job_validation.ts index 118e923283b3f..6692ecb22bd9e 100644 --- a/x-pack/plugins/ml/server/models/job_validation/job_validation.ts +++ b/x-pack/plugins/ml/server/models/job_validation/job_validation.ts @@ -7,7 +7,6 @@ import { i18n } from '@kbn/i18n'; import Boom from 'boom'; import { ILegacyScopedClusterClient } from 'kibana/server'; - import { TypeOf } from '@kbn/config-schema'; import { fieldsServiceProvider } from '../fields_service'; import { renderTemplate } from '../../../common/util/string_utils'; diff --git a/x-pack/plugins/ml/server/models/results_service/results_service.ts b/x-pack/plugins/ml/server/models/results_service/results_service.ts index 8e71384942b57..7be8bac61e69d 100644 --- a/x-pack/plugins/ml/server/models/results_service/results_service.ts +++ b/x-pack/plugins/ml/server/models/results_service/results_service.ts @@ -10,11 +10,19 @@ import get from 'lodash/get'; import moment from 'moment'; import { SearchResponse } from 'elasticsearch'; import { ILegacyScopedClusterClient } from 'kibana/server'; +import Boom from 'boom'; import { buildAnomalyTableItems } from './build_anomaly_table_items'; import { ML_RESULTS_INDEX_PATTERN } from '../../../common/constants/index_patterns'; import { ANOMALIES_TABLE_DEFAULT_QUERY_SIZE } from '../../../common/constants/search'; import { getPartitionFieldsValuesFactory } from './get_partition_fields_values'; -import { AnomaliesTableRecord, AnomalyRecordDoc } from '../../../common/types/anomalies'; +import { + AnomaliesTableRecord, + AnomalyCategorizerStatsDoc, + AnomalyRecordDoc, +} from '../../../common/types/anomalies'; +import { JOB_ID, PARTITION_FIELD_VALUE } from '../../../common/constants/anomalies'; +import { GetStoppedPartitionResult } from '../../../common/types/results'; +import { MlJobsResponse } from '../job_service/jobs'; // Service for carrying out Elasticsearch queries to obtain data for the // ML Results dashboards. @@ -432,6 +440,154 @@ export function resultsServiceProvider(mlClusterClient: ILegacyScopedClusterClie return definition; } + async function getCategorizerStats(jobId: string, partitionByValue?: string) { + const mustMatchClauses: Array>> = [ + { + match: { + result_type: 'categorizer_stats', + }, + }, + ]; + + if (typeof partitionByValue === 'string') { + mustMatchClauses.push({ + match: { + partition_by_value: partitionByValue, + }, + }); + } + const results: SearchResponse = await callAsInternalUser('search', { + index: ML_RESULTS_INDEX_PATTERN, + body: { + query: { + bool: { + must: mustMatchClauses, + filter: [ + { + term: { + job_id: jobId, + }, + }, + ], + }, + }, + }, + }); + return results ? results.hits.hits.map((r) => r._source) : []; + } + + async function getCategoryStoppedPartitions( + jobIds: string[], + fieldToBucket: typeof JOB_ID | typeof PARTITION_FIELD_VALUE = PARTITION_FIELD_VALUE + ): Promise { + let finalResults: GetStoppedPartitionResult = { + jobs: {}, + }; + // first determine from job config if stop_on_warn is true + // if false return [] + const jobConfigResponse: MlJobsResponse = await callAsInternalUser('ml.jobs', { + jobId: jobIds, + }); + + if (!jobConfigResponse || jobConfigResponse.jobs.length < 1) { + throw Boom.notFound(`Unable to find anomaly detector jobs ${jobIds.join(', ')}`); + } + + const jobIdsWithStopOnWarnSet = jobConfigResponse.jobs + .filter( + (jobConfig) => + jobConfig.analysis_config?.per_partition_categorization?.stop_on_warn === true + ) + .map((j) => j.job_id); + + let aggs: any; + if (fieldToBucket === JOB_ID) { + // if bucketing by job_id, then return list of job_ids with at least one stopped_partitions + aggs = { + unique_terms: { + terms: { + field: JOB_ID, + }, + }, + }; + } else { + // if bucketing by partition field value, then return list of unique stopped_partitions for each job + aggs = { + jobs: { + terms: { + field: JOB_ID, + }, + aggs: { + unique_stopped_partitions: { + terms: { + field: PARTITION_FIELD_VALUE, + }, + }, + }, + }, + }; + } + + if (jobIdsWithStopOnWarnSet.length > 0) { + // search for categorizer_stats documents for the current job where the categorization_status is warn + // Return all the partition_field_value values from the documents found + const mustMatchClauses: Array>> = [ + { + match: { + result_type: 'categorizer_stats', + }, + }, + { + match: { + categorization_status: 'warn', + }, + }, + ]; + const results: SearchResponse = await callAsInternalUser('search', { + index: ML_RESULTS_INDEX_PATTERN, + size: 0, + body: { + query: { + bool: { + must: mustMatchClauses, + filter: [ + { + terms: { + job_id: jobIdsWithStopOnWarnSet, + }, + }, + ], + }, + }, + aggs, + }, + }); + if (fieldToBucket === JOB_ID) { + finalResults = { + jobs: results.aggregations?.unique_terms?.buckets.map( + (b: { key: string; doc_count: number }) => b.key + ), + }; + } else if (fieldToBucket === PARTITION_FIELD_VALUE) { + const jobs: Record = jobIdsWithStopOnWarnSet.reduce( + (obj: Record, jobId: string) => { + obj[jobId] = []; + return obj; + }, + {} + ); + results.aggregations.jobs.buckets.forEach( + (bucket: { key: string | number; unique_stopped_partitions: { buckets: any[] } }) => { + jobs[bucket.key] = bucket.unique_stopped_partitions.buckets.map((b) => b.key); + } + ); + finalResults.jobs = jobs; + } + } + + return finalResults; + } + return { getAnomaliesTableData, getCategoryDefinition, @@ -439,5 +595,7 @@ export function resultsServiceProvider(mlClusterClient: ILegacyScopedClusterClie getLatestBucketTimestampByJob, getMaxAnomalyScore, getPartitionFieldsValues: getPartitionFieldsValuesFactory(mlClusterClient), + getCategorizerStats, + getCategoryStoppedPartitions, }; } diff --git a/x-pack/plugins/ml/server/routes/anomaly_detectors.ts b/x-pack/plugins/ml/server/routes/anomaly_detectors.ts index c6bdb32b262e4..0027bec910134 100644 --- a/x-pack/plugins/ml/server/routes/anomaly_detectors.ts +++ b/x-pack/plugins/ml/server/routes/anomaly_detectors.ts @@ -20,7 +20,6 @@ import { getModelSnapshotsSchema, updateModelSnapshotSchema, } from './schemas/anomaly_detectors_schema'; - /** * Routes for the anomaly detectors */ diff --git a/x-pack/plugins/ml/server/routes/apidoc.json b/x-pack/plugins/ml/server/routes/apidoc.json index f360da5df5392..86a62b28abb5e 100644 --- a/x-pack/plugins/ml/server/routes/apidoc.json +++ b/x-pack/plugins/ml/server/routes/apidoc.json @@ -49,6 +49,8 @@ "GetCategoryExamples", "GetPartitionFieldsValues", "AnomalySearch", + "GetCategorizerStats", + "GetCategoryStoppedPartitions", "Modules", "DataRecognizer", diff --git a/x-pack/plugins/ml/server/routes/results_service.ts b/x-pack/plugins/ml/server/routes/results_service.ts index 0d619bf63b8e7..2af37c17f714a 100644 --- a/x-pack/plugins/ml/server/routes/results_service.ts +++ b/x-pack/plugins/ml/server/routes/results_service.ts @@ -17,6 +17,11 @@ import { } from './schemas/results_service_schema'; import { resultsServiceProvider } from '../models/results_service'; import { ML_RESULTS_INDEX_PATTERN } from '../../common/constants/index_patterns'; +import { jobIdSchema } from './schemas/anomaly_detectors_schema'; +import { + getCategorizerStatsSchema, + getCategorizerStoppedPartitionsSchema, +} from './schemas/results_service_schema'; function getAnomaliesTableData(legacyClient: ILegacyScopedClusterClient, payload: any) { const rs = resultsServiceProvider(legacyClient); @@ -71,6 +76,19 @@ function getPartitionFieldsValues(legacyClient: ILegacyScopedClusterClient, payl return rs.getPartitionFieldsValues(jobId, searchTerm, criteriaFields, earliestMs, latestMs); } +function getCategorizerStats(legacyClient: ILegacyScopedClusterClient, params: any, query: any) { + const { jobId } = params; + const { partitionByValue } = query; + const rs = resultsServiceProvider(legacyClient); + return rs.getCategorizerStats(jobId, partitionByValue); +} + +function getCategoryStoppedPartitions(legacyClient: ILegacyScopedClusterClient, payload: any) { + const { jobIds, fieldToBucket } = payload; + const rs = resultsServiceProvider(legacyClient); + return rs.getCategoryStoppedPartitions(jobIds, fieldToBucket); +} + /** * Routes for results service */ @@ -265,4 +283,66 @@ export function resultsServiceRoutes({ router, mlLicense }: RouteInitialization) } }) ); + + /** + * @apiGroup ResultsService + * + * @api {get} /api/ml/results/:jobId/categorizer_stats + * @apiName GetCategorizerStats + * @apiDescription Returns the categorizer stats for the specified job ID + * @apiSchema (params) jobIdSchema + * @apiSchema (query) getCategorizerStatsSchema + */ + router.get( + { + path: '/api/ml/results/{jobId}/categorizer_stats', + validate: { + params: jobIdSchema, + query: getCategorizerStatsSchema, + }, + options: { + tags: ['access:ml:canGetJobs'], + }, + }, + mlLicense.fullLicenseAPIGuard(async ({ legacyClient, request, response }) => { + try { + const resp = await getCategorizerStats(legacyClient, request.params, request.query); + return response.ok({ + body: resp, + }); + } catch (e) { + return response.customError(wrapError(e)); + } + }) + ); + + /** + * @apiGroup ResultsService + * + * @api {get} /api/ml/results/category_stopped_partitions + * @apiName GetCategoryStoppedPartitions + * @apiDescription Returns information on the partitions that have stopped being categorized due to the categorization status changing from ok to warn. Can return either the list of stopped partitions for each job, or just the list of job IDs. + * @apiSchema (body) getCategorizerStoppedPartitionsSchema + */ + router.post( + { + path: '/api/ml/results/category_stopped_partitions', + validate: { + body: getCategorizerStoppedPartitionsSchema, + }, + options: { + tags: ['access:ml:canGetJobs'], + }, + }, + mlLicense.fullLicenseAPIGuard(async ({ legacyClient, request, response }) => { + try { + const resp = await getCategoryStoppedPartitions(legacyClient, request.body); + return response.ok({ + body: resp, + }); + } catch (e) { + return response.customError(wrapError(e)); + } + }) + ); } diff --git a/x-pack/plugins/ml/server/routes/schemas/results_service_schema.ts b/x-pack/plugins/ml/server/routes/schemas/results_service_schema.ts index f7317e534b33b..0bf37826b6146 100644 --- a/x-pack/plugins/ml/server/routes/schemas/results_service_schema.ts +++ b/x-pack/plugins/ml/server/routes/schemas/results_service_schema.ts @@ -52,3 +52,26 @@ export const partitionFieldValuesSchema = schema.object({ earliestMs: schema.number(), latestMs: schema.number(), }); + +export const getCategorizerStatsSchema = schema.nullable( + schema.object({ + /** + * Optional value to fetch the categorizer stats + * where results are filtered by partition_by_value = value + */ + partitionByValue: schema.maybe(schema.string()), + }) +); + +export const getCategorizerStoppedPartitionsSchema = schema.object({ + /** + * List of jobIds to fetch the categorizer partitions for + */ + jobIds: schema.arrayOf(schema.string()), + /** + * Field to aggregate results by: 'job_id' or 'partition_field_value' + * If by job_id, will return list of jobIds with at least one partition that have stopped + * If by partition_field_value, it will return a list of categorizer stopped partitions for each job_id + */ + fieldToBucket: schema.maybe(schema.string()), +}); diff --git a/x-pack/test/api_integration/apis/ml/results/get_categorizer_stats.ts b/x-pack/test/api_integration/apis/ml/results/get_categorizer_stats.ts new file mode 100644 index 0000000000000..a9d863b7526f9 --- /dev/null +++ b/x-pack/test/api_integration/apis/ml/results/get_categorizer_stats.ts @@ -0,0 +1,148 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import expect from '@kbn/expect'; +import { FtrProviderContext } from '../../../ftr_provider_context'; +import { USER } from '../../../../functional/services/ml/security_common'; +import { COMMON_REQUEST_HEADERS } from '../../../../functional/services/ml/common'; +import { Datafeed } from '../../../../../plugins/ml/common/types/anomaly_detection_jobs'; +import { AnomalyCategorizerStatsDoc } from '../../../../../plugins/ml/common/types/anomalies'; + +export default ({ getService }: FtrProviderContext) => { + const esArchiver = getService('esArchiver'); + const supertest = getService('supertestWithoutAuth'); + const ml = getService('ml'); + + const jobId = `sample_logs_${Date.now()}`; + const PARTITION_FIELD_NAME = 'event.dataset'; + const testJobConfig = { + job_id: jobId, + groups: ['sample_logs', 'bootstrap', 'categorization'], + description: "count by mlcategory (message) on 'sample logs' dataset with 15m bucket span", + analysis_config: { + bucket_span: '15m', + categorization_field_name: 'message', + per_partition_categorization: { enabled: true, stop_on_warn: true }, + detectors: [ + { + function: 'count', + by_field_name: 'mlcategory', + partition_field_name: PARTITION_FIELD_NAME, + }, + ], + influencers: ['mlcategory'], + }, + analysis_limits: { model_memory_limit: '26MB' }, + data_description: { time_field: '@timestamp', time_format: 'epoch_ms' }, + model_plot_config: { enabled: false, annotations_enabled: true }, + model_snapshot_retention_days: 10, + daily_model_snapshot_retention_after_days: 1, + allow_lazy_open: false, + }; + const testDatafeedConfig: Datafeed = { + datafeed_id: `datafeed-${jobId}`, + indices: ['ft_module_sample_logs'], + job_id: jobId, + query: { bool: { must: [{ match_all: {} }] } }, + }; + + describe('get categorizer_stats', function () { + before(async () => { + await esArchiver.loadIfNeeded('ml/module_sample_logs'); + await ml.testResources.setKibanaTimeZoneToUTC(); + await ml.api.createAndRunAnomalyDetectionLookbackJob(testJobConfig, testDatafeedConfig); + }); + + after(async () => { + await ml.testResources.deleteIndexPatternByTitle('ft_module_sample_logs'); + await ml.api.cleanMlIndices(); + }); + + it('should fetch all the categorizer stats for job id', async () => { + const { body } = await supertest + .get(`/api/ml/results/${jobId}/categorizer_stats`) + .auth(USER.ML_POWERUSER, ml.securityCommon.getPasswordForUser(USER.ML_POWERUSER)) + .set(COMMON_REQUEST_HEADERS) + .expect(200); + + body.forEach((doc: AnomalyCategorizerStatsDoc) => { + expect(doc.job_id).to.eql(jobId); + expect(doc.result_type).to.eql('categorizer_stats'); + expect(doc.partition_field_name).to.be(PARTITION_FIELD_NAME); + expect(doc.partition_field_value).to.not.be(undefined); + }); + }); + + it('should fetch categorizer stats for job id for user with view permission', async () => { + const { body } = await supertest + .get(`/api/ml/results/${jobId}/categorizer_stats`) + .auth(USER.ML_VIEWER, ml.securityCommon.getPasswordForUser(USER.ML_VIEWER)) + .set(COMMON_REQUEST_HEADERS) + .expect(200); + + body.forEach((doc: AnomalyCategorizerStatsDoc) => { + expect(doc.job_id).to.eql(jobId); + expect(doc.result_type).to.eql('categorizer_stats'); + expect(doc.partition_field_name).to.be(PARTITION_FIELD_NAME); + expect(doc.partition_field_value).to.not.be(undefined); + }); + }); + + it('should not fetch categorizer stats for job id for unauthorized user', async () => { + const { body } = await supertest + .get(`/api/ml/results/${jobId}/categorizer_stats`) + .auth(USER.ML_UNAUTHORIZED, ml.securityCommon.getPasswordForUser(USER.ML_UNAUTHORIZED)) + .set(COMMON_REQUEST_HEADERS) + .expect(404); + + expect(body.error).to.be('Not Found'); + expect(body.message).to.be('Not Found'); + }); + + it('should fetch all the categorizer stats with per-partition value for job id', async () => { + const { body } = await supertest + .get(`/api/ml/results/${jobId}/categorizer_stats`) + .query({ partitionByValue: 'sample_web_logs' }) + .auth(USER.ML_POWERUSER, ml.securityCommon.getPasswordForUser(USER.ML_POWERUSER)) + .set(COMMON_REQUEST_HEADERS) + .expect(200); + body.forEach((doc: AnomalyCategorizerStatsDoc) => { + expect(doc.job_id).to.eql(jobId); + expect(doc.result_type).to.eql('categorizer_stats'); + expect(doc.partition_field_name).to.be(PARTITION_FIELD_NAME); + expect(doc.partition_field_value).to.be('sample_web_logs'); + }); + }); + + it('should fetch categorizer stats with per-partition value for user with view permission', async () => { + const { body } = await supertest + .get(`/api/ml/results/${jobId}/categorizer_stats`) + .query({ partitionByValue: 'sample_web_logs' }) + .auth(USER.ML_VIEWER, ml.securityCommon.getPasswordForUser(USER.ML_VIEWER)) + .set(COMMON_REQUEST_HEADERS) + .expect(200); + + body.forEach((doc: AnomalyCategorizerStatsDoc) => { + expect(doc.job_id).to.eql(jobId); + expect(doc.result_type).to.eql('categorizer_stats'); + expect(doc.partition_field_name).to.be(PARTITION_FIELD_NAME); + expect(doc.partition_field_value).to.be('sample_web_logs'); + }); + }); + + it('should not fetch categorizer stats with per-partition value for unauthorized user', async () => { + const { body } = await supertest + .get(`/api/ml/results/${jobId}/categorizer_stats`) + .query({ partitionByValue: 'sample_web_logs' }) + .auth(USER.ML_UNAUTHORIZED, ml.securityCommon.getPasswordForUser(USER.ML_UNAUTHORIZED)) + .set(COMMON_REQUEST_HEADERS) + .expect(404); + + expect(body.error).to.be('Not Found'); + expect(body.message).to.be('Not Found'); + }); + }); +}; diff --git a/x-pack/test/api_integration/apis/ml/results/get_stopped_partitions.ts b/x-pack/test/api_integration/apis/ml/results/get_stopped_partitions.ts new file mode 100644 index 0000000000000..424bc8c333aab --- /dev/null +++ b/x-pack/test/api_integration/apis/ml/results/get_stopped_partitions.ts @@ -0,0 +1,184 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import expect from '@kbn/expect'; +import { Datafeed, Job } from '../../../../../plugins/ml/common/types/anomaly_detection_jobs'; +import { FtrProviderContext } from '../../../ftr_provider_context'; +import { USER } from '../../../../functional/services/ml/security_common'; +import { COMMON_REQUEST_HEADERS } from '../../../../functional/services/ml/common'; + +export default ({ getService }: FtrProviderContext) => { + const esArchiver = getService('esArchiver'); + const supertest = getService('supertestWithoutAuth'); + const ml = getService('ml'); + + const testJobId = `sample_logs_${Date.now()}`; + // non-aggregatable field to cause some partitions to change status to warn + const PARTITION_FIELD_NAME = 'agent'; + + interface TestConfig { + testDescription: string; + jobId: string; + jobConfig: Job; + datafeedConfig: Datafeed; + } + const setupTestConfigs = ( + jobId: string, + stopOnWarn: boolean, + enabledPerPartitionCat: boolean = true + ): TestConfig => { + const commonJobConfig = { + groups: ['sample_logs', 'bootstrap', 'categorization'], + description: "count by mlcategory (message) on 'sample logs' dataset with 15m bucket span", + analysis_limits: { model_memory_limit: '26MB' }, + data_description: { time_field: '@timestamp', time_format: 'epoch_ms' }, + model_snapshot_retention_days: 10, + daily_model_snapshot_retention_after_days: 1, + allow_lazy_open: false, + }; + const datafeedConfig: Datafeed = { + datafeed_id: `datafeed-${jobId}`, + indices: ['ft_module_sample_logs'], + job_id: jobId, + query: { bool: { must: [{ match_all: {} }] } }, + }; + + return { + testDescription: `stop_on_warn is ${stopOnWarn}`, + jobId, + jobConfig: { + job_id: jobId, + ...commonJobConfig, + analysis_config: { + bucket_span: '1m', + categorization_field_name: 'message', + per_partition_categorization: { + enabled: enabledPerPartitionCat, + stop_on_warn: stopOnWarn, + }, + detectors: [ + { + function: 'count', + by_field_name: 'mlcategory', + partition_field_name: PARTITION_FIELD_NAME, + }, + ], + influencers: ['mlcategory'], + }, + }, + datafeedConfig, + }; + }; + + const testSetUps: TestConfig[] = [ + setupTestConfigs(`${testJobId}_t`, true), + setupTestConfigs(`${testJobId}_f`, false), + setupTestConfigs(`${testJobId}_viewer`, true), + setupTestConfigs(`${testJobId}_unauthorized`, true), + ]; + + const testJobIds = testSetUps.map((t) => t.jobId); + + describe('get stopped_partitions', function () { + before(async () => { + await esArchiver.loadIfNeeded('ml/module_sample_logs'); + await ml.testResources.setKibanaTimeZoneToUTC(); + for (const testData of testSetUps) { + const { jobConfig, datafeedConfig } = testData; + await ml.api.createAndRunAnomalyDetectionLookbackJob(jobConfig, datafeedConfig); + } + }); + + after(async () => { + await ml.testResources.deleteIndexPatternByTitle('ft_module_sample_logs'); + await ml.api.cleanMlIndices(); + }); + + it('should fetch all the stopped partitions correctly', async () => { + const { jobId } = testSetUps[0]; + const { body } = await supertest + .post(`/api/ml/results/category_stopped_partitions`) + .auth(USER.ML_POWERUSER, ml.securityCommon.getPasswordForUser(USER.ML_POWERUSER)) + .send({ jobIds: [jobId] }) + .set(COMMON_REQUEST_HEADERS) + .expect(200); + expect(body.jobs).to.not.be(undefined); + expect(body.jobs[jobId]).to.be.an('array'); + expect(body.jobs[jobId].length).to.be.greaterThan(0); + }); + + it('should not return jobId in response if stopped_on_warn is false', async () => { + const { jobId } = testSetUps[1]; + const { body } = await supertest + .post(`/api/ml/results/category_stopped_partitions`) + .auth(USER.ML_POWERUSER, ml.securityCommon.getPasswordForUser(USER.ML_POWERUSER)) + .send({ jobIds: [jobId] }) + .set(COMMON_REQUEST_HEADERS) + .expect(200); + expect(body.jobs).to.not.be(undefined); + expect(body.jobs).to.not.have.property(jobId); + }); + + it('should fetch stopped partitions for user with view permission', async () => { + const { jobId } = testSetUps[2]; + const { body } = await supertest + .post(`/api/ml/results/category_stopped_partitions`) + .auth(USER.ML_VIEWER, ml.securityCommon.getPasswordForUser(USER.ML_VIEWER)) + .send({ jobIds: [jobId] }) + .set(COMMON_REQUEST_HEADERS) + .expect(200); + + expect(body.jobs).to.not.be(undefined); + expect(body.jobs[jobId]).to.be.an('array'); + expect(body.jobs[jobId].length).to.be.greaterThan(0); + }); + + it('should not fetch stopped partitions for unauthorized user', async () => { + const { jobId } = testSetUps[3]; + + const { body } = await supertest + .post(`/api/ml/results/category_stopped_partitions`) + .auth(USER.ML_UNAUTHORIZED, ml.securityCommon.getPasswordForUser(USER.ML_UNAUTHORIZED)) + .send({ jobIds: [jobId] }) + .set(COMMON_REQUEST_HEADERS) + .expect(404); + + expect(body.error).to.be('Not Found'); + expect(body.message).to.be('Not Found'); + }); + + it('should fetch stopped partitions for multiple job ids', async () => { + const { body } = await supertest + .post(`/api/ml/results/category_stopped_partitions`) + .auth(USER.ML_POWERUSER, ml.securityCommon.getPasswordForUser(USER.ML_POWERUSER)) + .send({ jobIds: testJobIds }) + .set(COMMON_REQUEST_HEADERS) + .expect(200); + expect(body.jobs).to.not.be(undefined); + expect(body.jobs).to.not.have.property(testSetUps[1].jobId); + + Object.keys(body.jobs).forEach((currentJobId: string) => { + expect(testJobIds).to.contain(currentJobId); + expect(body.jobs[currentJobId]).to.be.an('array'); + expect(body.jobs[currentJobId].length).to.be.greaterThan(0); + }); + }); + + it('should return array of jobIds with stopped_partitions for multiple job ids when bucketed by job_id', async () => { + const { body } = await supertest + .post(`/api/ml/results/category_stopped_partitions`) + .auth(USER.ML_POWERUSER, ml.securityCommon.getPasswordForUser(USER.ML_POWERUSER)) + .send({ jobIds: testJobIds, fieldToBucket: 'job_id' }) + .set(COMMON_REQUEST_HEADERS) + .expect(200); + + expect(body.jobs).to.not.be(undefined); + body.jobs.forEach((currentJobId: string) => { + expect(testJobIds).to.contain(currentJobId); + }); + }); + }); +}; diff --git a/x-pack/test/api_integration/apis/ml/results/index.ts b/x-pack/test/api_integration/apis/ml/results/index.ts index 7f44ebefc7b2b..f6a7c6ed81843 100644 --- a/x-pack/test/api_integration/apis/ml/results/index.ts +++ b/x-pack/test/api_integration/apis/ml/results/index.ts @@ -8,5 +8,7 @@ import { FtrProviderContext } from '../../../ftr_provider_context'; export default function ({ loadTestFile }: FtrProviderContext) { describe('ResultsService', () => { loadTestFile(require.resolve('./get_anomalies_table_data')); + loadTestFile(require.resolve('./get_categorizer_stats')); + loadTestFile(require.resolve('./get_stopped_partitions')); }); }