From c55659f53cabdacea3f97f090f7123c7f6586bbb Mon Sep 17 00:00:00 2001 From: Quynh Nguyen <43350163+qn895@users.noreply.github.com> Date: Tue, 14 Jul 2020 12:36:01 -0500 Subject: [PATCH] [ML] Anomaly Detection: Annotations enhancements (#70198) Co-authored-by: Elastic Machine --- .../ml/common/constants/annotations.ts | 3 + .../plugins/ml/common/constants/anomalies.ts | 2 + x-pack/plugins/ml/common/types/annotations.ts | 45 ++- x-pack/plugins/ml/common/types/anomalies.ts | 4 + .../annotation_description_list/index.tsx | 30 +- .../annotations/annotation_flyout/index.tsx | 89 +++++- .../annotations_table.test.js.snap | 47 +++- .../annotations_table/annotations_table.js | 260 +++++++++++++++--- .../explorer/actions/load_explorer_data.ts | 2 +- .../public/application/explorer/explorer.d.ts | 5 - .../public/application/explorer/explorer.js | 72 +++-- .../application/explorer/explorer_utils.js | 16 +- .../reducers/explorer_reducer/reducer.ts | 2 +- .../reducers/explorer_reducer/state.ts | 11 +- .../application/routing/routes/explorer.tsx | 1 - .../services/ml_api_service/annotations.ts | 14 +- .../timeseriesexplorer/timeseriesexplorer.js | 57 +++- .../get_focus_data.ts | 19 +- .../models/annotation_service/annotation.ts | 101 ++++++- .../get_partition_fields_values.ts | 6 +- .../routes/schemas/annotations_schema.ts | 27 ++ .../translations/translations/ja-JP.json | 2 - .../translations/translations/zh-CN.json | 2 - 23 files changed, 697 insertions(+), 120 deletions(-) diff --git a/x-pack/plugins/ml/common/constants/annotations.ts b/x-pack/plugins/ml/common/constants/annotations.ts index 936ff610361af..4929dfb28eb15 100644 --- a/x-pack/plugins/ml/common/constants/annotations.ts +++ b/x-pack/plugins/ml/common/constants/annotations.ts @@ -13,3 +13,6 @@ export const ANNOTATION_USER_UNKNOWN = ''; // UI enforced limit to the maximum number of characters that can be entered for an annotation. export const ANNOTATION_MAX_LENGTH_CHARS = 1000; + +export const ANNOTATION_EVENT_USER = 'user'; +export const ANNOTATION_EVENT_DELAYED_DATA = 'delayed_data'; diff --git a/x-pack/plugins/ml/common/constants/anomalies.ts b/x-pack/plugins/ml/common/constants/anomalies.ts index bbf3616c05880..d15033b738b0f 100644 --- a/x-pack/plugins/ml/common/constants/anomalies.ts +++ b/x-pack/plugins/ml/common/constants/anomalies.ts @@ -20,3 +20,5 @@ export enum ANOMALY_THRESHOLD { WARNING = 3, LOW = 0, } + +export const PARTITION_FIELDS = ['partition_field', 'over_field', 'by_field'] as const; diff --git a/x-pack/plugins/ml/common/types/annotations.ts b/x-pack/plugins/ml/common/types/annotations.ts index f2f6fe111f5cc..159a598f16bf5 100644 --- a/x-pack/plugins/ml/common/types/annotations.ts +++ b/x-pack/plugins/ml/common/types/annotations.ts @@ -58,8 +58,20 @@ // ] // } +import { PartitionFieldsType } from './anomalies'; import { ANNOTATION_TYPE } from '../constants/annotations'; +export type AnnotationFieldName = 'partition_field_name' | 'over_field_name' | 'by_field_name'; +export type AnnotationFieldValue = 'partition_field_value' | 'over_field_value' | 'by_field_value'; + +export function getAnnotationFieldName(fieldType: PartitionFieldsType): AnnotationFieldName { + return `${fieldType}_name` as AnnotationFieldName; +} + +export function getAnnotationFieldValue(fieldType: PartitionFieldsType): AnnotationFieldValue { + return `${fieldType}_value` as AnnotationFieldValue; +} + export interface Annotation { _id?: string; create_time?: number; @@ -73,8 +85,15 @@ export interface Annotation { annotation: string; job_id: string; type: ANNOTATION_TYPE.ANNOTATION | ANNOTATION_TYPE.COMMENT; + event?: string; + detector_index?: number; + partition_field_name?: string; + partition_field_value?: string; + over_field_name?: string; + over_field_value?: string; + by_field_name?: string; + by_field_value?: string; } - export function isAnnotation(arg: any): arg is Annotation { return ( arg.timestamp !== undefined && @@ -93,3 +112,27 @@ export function isAnnotations(arg: any): arg is Annotations { } return arg.every((d: Annotation) => isAnnotation(d)); } + +export interface FieldToBucket { + field: string; + missing?: string | number; +} + +export interface FieldToBucketResult { + key: string; + doc_count: number; +} + +export interface TermAggregationResult { + doc_count_error_upper_bound: number; + sum_other_doc_count: number; + buckets: FieldToBucketResult[]; +} + +export type EsAggregationResult = Record; + +export interface GetAnnotationsResponse { + aggregations?: EsAggregationResult; + annotations: Record; + success: boolean; +} diff --git a/x-pack/plugins/ml/common/types/anomalies.ts b/x-pack/plugins/ml/common/types/anomalies.ts index 639d9b3b25fae..a23886e8fcdc6 100644 --- a/x-pack/plugins/ml/common/types/anomalies.ts +++ b/x-pack/plugins/ml/common/types/anomalies.ts @@ -4,6 +4,8 @@ * you may not use this file except in compliance with the Elastic License. */ +import { PARTITION_FIELDS } from '../constants/anomalies'; + export interface Influencer { influencer_field_name: string; influencer_field_values: string[]; @@ -53,3 +55,5 @@ export interface AnomaliesTableRecord { typicalSort?: any; metricDescriptionSort?: number; } + +export type PartitionFieldsType = typeof PARTITION_FIELDS[number]; diff --git a/x-pack/plugins/ml/public/application/components/annotations/annotation_description_list/index.tsx b/x-pack/plugins/ml/public/application/components/annotations/annotation_description_list/index.tsx index cf8fd299c07d7..eee2f8dca244d 100644 --- a/x-pack/plugins/ml/public/application/components/annotations/annotation_description_list/index.tsx +++ b/x-pack/plugins/ml/public/application/components/annotations/annotation_description_list/index.tsx @@ -19,9 +19,10 @@ import { formatHumanReadableDateTimeSeconds } from '../../../util/date_utils'; interface Props { annotation: Annotation; + detectorDescription?: string; } -export const AnnotationDescriptionList = ({ annotation }: Props) => { +export const AnnotationDescriptionList = ({ annotation, detectorDescription }: Props) => { const listItems = [ { title: i18n.translate('xpack.ml.timeSeriesExplorer.annotationDescriptionList.jobIdTitle', { @@ -81,6 +82,33 @@ export const AnnotationDescriptionList = ({ annotation }: Props) => { description: annotation.modified_username, }); } + if (detectorDescription !== undefined) { + listItems.push({ + title: i18n.translate('xpack.ml.timeSeriesExplorer.annotationDescriptionList.detectorTitle', { + defaultMessage: 'Detector', + }), + description: detectorDescription, + }); + } + + if (annotation.partition_field_name !== undefined) { + listItems.push({ + title: annotation.partition_field_name, + description: annotation.partition_field_value, + }); + } + if (annotation.over_field_name !== undefined) { + listItems.push({ + title: annotation.over_field_name, + description: annotation.over_field_value, + }); + } + if (annotation.by_field_name !== undefined) { + listItems.push({ + title: annotation.by_field_name, + description: annotation.by_field_value, + }); + } return ( { public state: State = { isDeleteModalVisible: false, + applyAnnotationToSeries: true, }; public annotationSub: Rx.Subscription | null = null; @@ -150,11 +178,31 @@ class AnnotationFlyoutUI extends Component { }; public saveOrUpdateAnnotation = () => { - const { annotation } = this.props; - - if (annotation === null) { + const { annotation: originalAnnotation, chartDetails, detectorIndex } = this.props; + if (originalAnnotation === null) { return; } + const annotation = cloneDeep(originalAnnotation); + + if (this.state.applyAnnotationToSeries && chartDetails?.entityData?.entities) { + chartDetails.entityData.entities.forEach((entity: Entity) => { + const { fieldName, fieldValue } = entity; + const fieldType = entity.fieldType as PartitionFieldsType; + annotation[getAnnotationFieldName(fieldType)] = fieldName; + annotation[getAnnotationFieldValue(fieldType)] = fieldValue; + }); + annotation.detector_index = detectorIndex; + } + // if unchecked, remove all the partitions before indexing + if (!this.state.applyAnnotationToSeries) { + delete annotation.detector_index; + PARTITION_FIELDS.forEach((fieldType) => { + delete annotation[getAnnotationFieldName(fieldType)]; + delete annotation[getAnnotationFieldValue(fieldType)]; + }); + } + // Mark the annotation created by `user` if and only if annotation is being created, not updated + annotation.event = annotation.event ?? ANNOTATION_EVENT_USER; annotation$.next(null); @@ -214,7 +262,7 @@ class AnnotationFlyoutUI extends Component { }; public render(): ReactNode { - const { annotation } = this.props; + const { annotation, detectors, detectorIndex } = this.props; const { isDeleteModalVisible } = this.state; if (annotation === null) { @@ -242,10 +290,13 @@ class AnnotationFlyoutUI extends Component { } ); } + const detector = detectors ? detectors.find((d) => d.index === detectorIndex) : undefined; + const detectorDescription = + detector && 'detector_description' in detector ? detector.detector_description : ''; return ( - +

@@ -264,7 +315,10 @@ class AnnotationFlyoutUI extends Component { - + { value={annotation.annotation} /> + + + } + checked={this.state.applyAnnotationToSeries} + onChange={() => + this.setState({ + applyAnnotationToSeries: !this.state.applyAnnotationToSeries, + }) + } + /> + diff --git a/x-pack/plugins/ml/public/application/components/annotations/annotations_table/__snapshots__/annotations_table.test.js.snap b/x-pack/plugins/ml/public/application/components/annotations/annotations_table/__snapshots__/annotations_table.test.js.snap index 3b93213da4033..63ec1744b62d0 100644 --- a/x-pack/plugins/ml/public/application/components/annotations/annotations_table/__snapshots__/annotations_table.test.js.snap +++ b/x-pack/plugins/ml/public/application/components/annotations/annotations_table/__snapshots__/annotations_table.test.js.snap @@ -11,7 +11,7 @@ exports[`AnnotationsTable Initialization with annotations prop. 1`] = ` "name": "Annotation", "scope": "row", "sortable": true, - "width": "50%", + "width": "40%", }, Object { "dataType": "date", @@ -39,6 +39,27 @@ exports[`AnnotationsTable Initialization with annotations prop. 1`] = ` "name": "Last modified by", "sortable": true, }, + Object { + "field": "event", + "name": "Event", + "sortable": true, + "width": "10%", + }, + Object { + "field": "partition_field_value", + "name": "Partition", + "sortable": true, + }, + Object { + "field": "over_field_value", + "name": "Over", + "sortable": true, + }, + Object { + "field": "by_field_value", + "name": "By", + "sortable": true, + }, Object { "actions": Array [ Object { @@ -52,6 +73,12 @@ exports[`AnnotationsTable Initialization with annotations prop. 1`] = ` "name": "Actions", "width": "60px", }, + Object { + "dataType": "boolean", + "field": "current_series", + "name": "current_series", + "width": "0px", + }, ] } compressed={true} @@ -82,6 +109,24 @@ exports[`AnnotationsTable Initialization with annotations prop. 1`] = ` } responsive={true} rowProps={[Function]} + search={ + Object { + "box": Object { + "incremental": true, + "schema": true, + }, + "defaultQuery": "event:(user or delayed_data)", + "filters": Array [ + Object { + "field": "event", + "multiSelect": "or", + "name": "Event", + "options": Array [], + "type": "field_value_selection", + }, + ], + } + } sorting={ Object { "sort": Object { diff --git a/x-pack/plugins/ml/public/application/components/annotations/annotations_table/annotations_table.js b/x-pack/plugins/ml/public/application/components/annotations/annotations_table/annotations_table.js index a091da6c359d1..cf4d25f159a1a 100644 --- a/x-pack/plugins/ml/public/application/components/annotations/annotations_table/annotations_table.js +++ b/x-pack/plugins/ml/public/application/components/annotations/annotations_table/annotations_table.js @@ -9,11 +9,9 @@ * This version supports both fetching the annotations by itself (used in the jobs list) and * getting the annotations via props (used in Anomaly Explorer and Single Series Viewer). */ - import _ from 'lodash'; import PropTypes from 'prop-types'; import rison from 'rison-node'; - import React, { Component, Fragment } from 'react'; import { @@ -50,7 +48,12 @@ import { annotationsRefresh$, annotationsRefreshed, } from '../../../services/annotations_service'; +import { + ANNOTATION_EVENT_USER, + ANNOTATION_EVENT_DELAYED_DATA, +} from '../../../../../common/constants/annotations'; +const CURRENT_SERIES = 'current_series'; /** * Table component for rendering the lists of annotations for an ML job. */ @@ -66,7 +69,10 @@ export class AnnotationsTable extends Component { super(props); this.state = { annotations: [], + aggregations: null, isLoading: false, + queryText: `event:(${ANNOTATION_EVENT_USER} or ${ANNOTATION_EVENT_DELAYED_DATA})`, + searchError: undefined, jobId: Array.isArray(this.props.jobs) && this.props.jobs.length > 0 && @@ -74,6 +80,9 @@ export class AnnotationsTable extends Component { ? this.props.jobs[0].job_id : undefined, }; + this.sorting = { + sort: { field: 'timestamp', direction: 'asc' }, + }; } getAnnotations() { @@ -92,11 +101,18 @@ export class AnnotationsTable extends Component { earliestMs: null, latestMs: null, maxAnnotations: ANNOTATIONS_TABLE_DEFAULT_QUERY_SIZE, + fields: [ + { + field: 'event', + missing: ANNOTATION_EVENT_USER, + }, + ], }) .toPromise() .then((resp) => { this.setState((prevState, props) => ({ annotations: resp.annotations[props.jobs[0].job_id] || [], + aggregations: resp.aggregations, errorMessage: undefined, isLoading: false, jobId: props.jobs[0].job_id, @@ -114,6 +130,25 @@ export class AnnotationsTable extends Component { } } + getAnnotationsWithExtraInfo(annotations) { + // if there is a specific view/chart entities that the annotations can be scoped to + // add a new column called 'current_series' + if (Array.isArray(this.props.chartDetails?.entityData?.entities)) { + return annotations.map((annotation) => { + const allMatched = this.props.chartDetails?.entityData?.entities.every( + ({ fieldType, fieldValue }) => { + const field = `${fieldType}_value`; + return !(!annotation[field] || annotation[field] !== fieldValue); + } + ); + return { ...annotation, [CURRENT_SERIES]: allMatched }; + }); + } else { + // if not make it return the original annotations + return annotations; + } + } + getJob(jobId) { // check if the job was supplied via props and matches the supplied jobId if (Array.isArray(this.props.jobs) && this.props.jobs.length > 0) { @@ -134,9 +169,9 @@ export class AnnotationsTable extends Component { Array.isArray(this.props.jobs) && this.props.jobs.length > 0 ) { - this.annotationsRefreshSubscription = annotationsRefresh$.subscribe(() => - this.getAnnotations() - ); + this.annotationsRefreshSubscription = annotationsRefresh$.subscribe(() => { + this.getAnnotations(); + }); annotationsRefreshed(); } } @@ -198,9 +233,11 @@ export class AnnotationsTable extends Component { }, }, }; + let mlTimeSeriesExplorer = {}; + const entityCondition = {}; if (annotation.timestamp !== undefined && annotation.end_timestamp !== undefined) { - appState.mlTimeSeriesExplorer = { + mlTimeSeriesExplorer = { zoom: { from: new Date(annotation.timestamp).toISOString(), to: new Date(annotation.end_timestamp).toISOString(), @@ -216,6 +253,27 @@ export class AnnotationsTable extends Component { } } + // if the annotation is at the series level + // then pass the partitioning field(s) and detector index to the Single Metric Viewer + if (_.has(annotation, 'detector_index')) { + mlTimeSeriesExplorer.detector_index = annotation.detector_index; + } + if (_.has(annotation, 'partition_field_value')) { + entityCondition[annotation.partition_field_name] = annotation.partition_field_value; + } + + if (_.has(annotation, 'over_field_value')) { + entityCondition[annotation.over_field_name] = annotation.over_field_value; + } + + if (_.has(annotation, 'by_field_value')) { + // Note that analyses with by and over fields, will have a top-level by_field_name, + // but the by_field_value(s) will be in the nested causes array. + entityCondition[annotation.by_field_name] = annotation.by_field_value; + } + mlTimeSeriesExplorer.entities = entityCondition; + appState.mlTimeSeriesExplorer = mlTimeSeriesExplorer; + const _g = rison.encode(globalSettings); const _a = rison.encode(appState); @@ -251,6 +309,8 @@ export class AnnotationsTable extends Component { render() { const { isSingleMetricViewerLinkVisible = true, isNumberBadgeVisible = false } = this.props; + const { queryText, searchError } = this.state; + if (this.props.annotations === undefined) { if (this.state.isLoading === true) { return ( @@ -314,7 +374,7 @@ export class AnnotationsTable extends Component { defaultMessage: 'Annotation', }), sortable: true, - width: '50%', + width: '40%', scope: 'row', }, { @@ -351,6 +411,14 @@ export class AnnotationsTable extends Component { }), sortable: true, }, + { + field: 'event', + name: i18n.translate('xpack.ml.annotationsTable.eventColumnName', { + defaultMessage: 'Event', + }), + sortable: true, + width: '10%', + }, ]; const jobIds = _.uniq(annotations.map((a) => a.job_id)); @@ -382,22 +450,23 @@ export class AnnotationsTable extends Component { actions.push({ render: (annotation) => { + // find the original annotation because the table might not show everything + const annotationId = annotation._id; + const originalAnnotation = annotations.find((d) => d._id === annotationId); const editAnnotationsTooltipText = ( ); - const editAnnotationsTooltipAriaLabelText = ( - + const editAnnotationsTooltipAriaLabelText = i18n.translate( + 'xpack.ml.annotationsTable.editAnnotationsTooltipAriaLabel', + { defaultMessage: 'Edit annotation' } ); return ( annotation$.next(annotation)} + onClick={() => annotation$.next(originalAnnotation ?? annotation)} iconType="pencil" aria-label={editAnnotationsTooltipAriaLabelText} /> @@ -421,17 +490,14 @@ export class AnnotationsTable extends Component { defaultMessage="Job configuration not supported in Single Metric Viewer" /> ); - const openInSingleMetricViewerAriaLabelText = isDrillDownAvailable ? ( - - ) : ( - - ); + const openInSingleMetricViewerAriaLabelText = isDrillDownAvailable + ? i18n.translate('xpack.ml.annotationsTable.openInSingleMetricViewerAriaLabel', { + defaultMessage: 'Open in Single Metric Viewer', + }) + : i18n.translate( + 'xpack.ml.annotationsTable.jobConfigurationNotSupportedInSingleMetricViewerAriaLabel', + { defaultMessage: 'Job configuration not supported in Single Metric Viewer' } + ); return ( @@ -447,38 +513,152 @@ export class AnnotationsTable extends Component { }); } - columns.push({ - align: RIGHT_ALIGNMENT, - width: '60px', - name: i18n.translate('xpack.ml.annotationsTable.actionsColumnName', { - defaultMessage: 'Actions', - }), - actions, - }); - const getRowProps = (item) => { return { onMouseOver: () => this.onMouseOverRow(item), onMouseLeave: () => this.onMouseLeaveRow(), }; }; + let filterOptions = []; + const aggregations = this.props.aggregations ?? this.state.aggregations; + if (aggregations) { + const buckets = aggregations.event.buckets; + const foundUser = buckets.findIndex((d) => d.key === ANNOTATION_EVENT_USER) > -1; + filterOptions = foundUser + ? buckets + : [{ key: ANNOTATION_EVENT_USER, doc_count: 0 }, ...buckets]; + } + const filters = [ + { + type: 'field_value_selection', + field: 'event', + name: 'Event', + multiSelect: 'or', + options: filterOptions.map((field) => ({ + value: field.key, + name: field.key, + view: `${field.key} (${field.doc_count})`, + })), + }, + ]; + + if (this.props.detectors) { + columns.push({ + name: i18n.translate('xpack.ml.annotationsTable.detectorColumnName', { + defaultMessage: 'Detector', + }), + width: '10%', + render: (item) => { + if ('detector_index' in item) { + return this.props.detectors[item.detector_index].detector_description; + } + return ''; + }, + }); + } + if (Array.isArray(this.props.chartDetails?.entityData?.entities)) { + // only show the column if the field exists in that job in SMV + this.props.chartDetails?.entityData?.entities.forEach((entity) => { + if (entity.fieldType === 'partition_field') { + columns.push({ + field: 'partition_field_value', + name: i18n.translate('xpack.ml.annotationsTable.partitionSMVColumnName', { + defaultMessage: 'Partition', + }), + sortable: true, + }); + } + if (entity.fieldType === 'over_field') { + columns.push({ + field: 'over_field_value', + name: i18n.translate('xpack.ml.annotationsTable.overColumnSMVName', { + defaultMessage: 'Over', + }), + sortable: true, + }); + } + if (entity.fieldType === 'by_field') { + columns.push({ + field: 'by_field_value', + name: i18n.translate('xpack.ml.annotationsTable.byColumnSMVName', { + defaultMessage: 'By', + }), + sortable: true, + }); + } + }); + filters.push({ + type: 'is', + field: CURRENT_SERIES, + name: i18n.translate('xpack.ml.annotationsTable.seriesOnlyFilterName', { + defaultMessage: 'Filter to series', + }), + }); + } else { + // else show all the partition columns in AE because there might be multiple jobs + columns.push({ + field: 'partition_field_value', + name: i18n.translate('xpack.ml.annotationsTable.partitionAEColumnName', { + defaultMessage: 'Partition', + }), + sortable: true, + }); + columns.push({ + field: 'over_field_value', + name: i18n.translate('xpack.ml.annotationsTable.overAEColumnName', { + defaultMessage: 'Over', + }), + sortable: true, + }); + + columns.push({ + field: 'by_field_value', + name: i18n.translate('xpack.ml.annotationsTable.byAEColumnName', { + defaultMessage: 'By', + }), + sortable: true, + }); + } + const search = { + defaultQuery: queryText, + box: { + incremental: true, + schema: true, + }, + filters: filters, + }; + + columns.push( + { + align: RIGHT_ALIGNMENT, + width: '60px', + name: i18n.translate('xpack.ml.annotationsTable.actionsColumnName', { + defaultMessage: 'Actions', + }), + actions, + }, + { + // hidden column, for search only + field: CURRENT_SERIES, + name: CURRENT_SERIES, + dataType: 'boolean', + width: '0px', + } + ); return ( diff --git a/x-pack/plugins/ml/public/application/explorer/actions/load_explorer_data.ts b/x-pack/plugins/ml/public/application/explorer/actions/load_explorer_data.ts index 095b42ffac5b7..3fcb032bd3ce1 100644 --- a/x-pack/plugins/ml/public/application/explorer/actions/load_explorer_data.ts +++ b/x-pack/plugins/ml/public/application/explorer/actions/load_explorer_data.ts @@ -258,7 +258,7 @@ const loadExplorerDataProvider = (anomalyTimelineService: AnomalyTimelineService { influencers, viewBySwimlaneState } ): Partial => { return { - annotationsData, + annotations: annotationsData, influencers, loading: false, viewBySwimlaneDataLoading: false, diff --git a/x-pack/plugins/ml/public/application/explorer/explorer.d.ts b/x-pack/plugins/ml/public/application/explorer/explorer.d.ts index 90fb46d3cec4a..52181aab40328 100644 --- a/x-pack/plugins/ml/public/application/explorer/explorer.d.ts +++ b/x-pack/plugins/ml/public/application/explorer/explorer.d.ts @@ -5,11 +5,6 @@ */ import { FC } from 'react'; - -import { UrlState } from '../util/url_state'; - -import { JobSelection } from '../components/job_selector/use_job_selection'; - import { ExplorerState } from './reducers'; import { AppStateSelectedCells } from './explorer_utils'; diff --git a/x-pack/plugins/ml/public/application/explorer/explorer.js b/x-pack/plugins/ml/public/application/explorer/explorer.js index df4cea0c07987..4e27c17631506 100644 --- a/x-pack/plugins/ml/public/application/explorer/explorer.js +++ b/x-pack/plugins/ml/public/application/explorer/explorer.js @@ -14,6 +14,7 @@ import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n/react'; import { + htmlIdGenerator, EuiFlexGroup, EuiFlexItem, EuiFormRow, @@ -26,6 +27,9 @@ import { EuiSpacer, EuiTitle, EuiLoadingContent, + EuiPanel, + EuiAccordion, + EuiBadge, } from '@elastic/eui'; import { AnnotationFlyout } from '../components/annotations/annotation_flyout'; @@ -138,6 +142,7 @@ export class Explorer extends React.Component { }; state = { filterIconTriggeredQuery: undefined, language: DEFAULT_QUERY_LANG }; + htmlIdGen = htmlIdGenerator(); // Escape regular parens from fieldName as that portion of the query is not wrapped in double quotes // and will cause a syntax error when called with getKqlQueryValues @@ -202,7 +207,7 @@ export class Explorer extends React.Component { const { showCharts, severity } = this.props; const { - annotationsData, + annotations, chartsData, filterActive, filterPlaceHolder, @@ -216,6 +221,7 @@ export class Explorer extends React.Component { selectedJobs, tableData, } = this.props.explorerState; + const { annotationsData, aggregations } = annotations; const jobSelectorProps = { dateFormatTz: getDateFormatTz(), @@ -239,13 +245,12 @@ export class Explorer extends React.Component { ); } - const mainColumnWidthClassName = noInfluencersConfigured === true ? 'col-xs-12' : 'col-xs-10'; const mainColumnClasses = `column ${mainColumnWidthClassName}`; const timefilter = getTimefilter(); const bounds = timefilter.getActiveBounds(); - + const selectedJobIds = Array.isArray(selectedJobs) ? selectedJobs.map((job) => job.id) : []; return ( - {annotationsData.length > 0 && ( <> - -

- -

- - + + +

+ + + + ), + }} + /> +

+ + } + > + <> + + + + - + )} - {loading === false && ( - <> +

+ )} + +
{showCharts && }
+ - + )} diff --git a/x-pack/plugins/ml/public/application/explorer/explorer_utils.js b/x-pack/plugins/ml/public/application/explorer/explorer_utils.js index 23da9669ee9a5..6e0863f1a6e5b 100644 --- a/x-pack/plugins/ml/public/application/explorer/explorer_utils.js +++ b/x-pack/plugins/ml/public/application/explorer/explorer_utils.js @@ -34,6 +34,7 @@ import { SWIMLANE_TYPE, VIEW_BY_JOB_LABEL, } from './explorer_constants'; +import { ANNOTATION_EVENT_USER } from '../../../common/constants/annotations'; // create new job objects based on standard job config objects // new job objects just contain job id, bucket span in seconds and a selected flag. @@ -395,6 +396,12 @@ export function loadAnnotationsTableData(selectedCells, selectedJobs, interval, earliestMs: timeRange.earliestMs, latestMs: timeRange.latestMs, maxAnnotations: ANNOTATIONS_TABLE_DEFAULT_QUERY_SIZE, + fields: [ + { + field: 'event', + missing: ANNOTATION_EVENT_USER, + }, + ], }) .toPromise() .then((resp) => { @@ -410,16 +417,17 @@ export function loadAnnotationsTableData(selectedCells, selectedJobs, interval, } }); - return resolve( - annotationsData + return resolve({ + annotationsData: annotationsData .sort((a, b) => { return a.timestamp - b.timestamp; }) .map((d, i) => { d.key = String.fromCharCode(65 + i); return d; - }) - ); + }), + aggregations: resp.aggregations, + }); }) .catch((resp) => { console.log('Error loading list of annotations for jobs list:', resp); diff --git a/x-pack/plugins/ml/public/application/explorer/reducers/explorer_reducer/reducer.ts b/x-pack/plugins/ml/public/application/explorer/reducers/explorer_reducer/reducer.ts index c55c06c80ab81..a38044a8b3425 100644 --- a/x-pack/plugins/ml/public/application/explorer/reducers/explorer_reducer/reducer.ts +++ b/x-pack/plugins/ml/public/application/explorer/reducers/explorer_reducer/reducer.ts @@ -113,7 +113,7 @@ export const explorerReducer = (state: ExplorerState, nextAction: Action): Explo const { annotationsData, overallState, tableData } = payload; nextState = { ...state, - annotationsData, + annotations: annotationsData, overallSwimlaneData: overallState, tableData, viewBySwimlaneData: { diff --git a/x-pack/plugins/ml/public/application/explorer/reducers/explorer_reducer/state.ts b/x-pack/plugins/ml/public/application/explorer/reducers/explorer_reducer/state.ts index 892b46467345b..889d572f4fabc 100644 --- a/x-pack/plugins/ml/public/application/explorer/reducers/explorer_reducer/state.ts +++ b/x-pack/plugins/ml/public/application/explorer/reducers/explorer_reducer/state.ts @@ -21,10 +21,14 @@ import { SwimlaneData, ViewBySwimLaneData, } from '../../explorer_utils'; +import { Annotations, EsAggregationResult } from '../../../../../common/types/annotations'; import { SWIM_LANE_DEFAULT_PAGE_SIZE } from '../../explorer_constants'; export interface ExplorerState { - annotationsData: any[]; + annotations: { + annotationsData: Annotations; + aggregations: EsAggregationResult; + }; bounds: TimeRangeBounds | undefined; chartsData: ExplorerChartsData; fieldFormatsLoading: boolean; @@ -62,7 +66,10 @@ function getDefaultIndexPattern() { export function getExplorerDefaultState(): ExplorerState { return { - annotationsData: [], + annotations: { + annotationsData: [], + aggregations: {}, + }, bounds: undefined, chartsData: getDefaultChartsData(), fieldFormatsLoading: false, diff --git a/x-pack/plugins/ml/public/application/routing/routes/explorer.tsx b/x-pack/plugins/ml/public/application/routing/routes/explorer.tsx index 7a7865c9bd738..5336c5d57cf43 100644 --- a/x-pack/plugins/ml/public/application/routing/routes/explorer.tsx +++ b/x-pack/plugins/ml/public/application/routing/routes/explorer.tsx @@ -147,7 +147,6 @@ const ExplorerUrlStateManager: FC = ({ jobsWithTim }, [explorerAppState]); const explorerState = useObservable(explorerService.state$); - const [showCharts] = useShowCharts(); const [tableInterval] = useTableInterval(); const [tableSeverity] = useTableSeverity(); diff --git a/x-pack/plugins/ml/public/application/services/ml_api_service/annotations.ts b/x-pack/plugins/ml/public/application/services/ml_api_service/annotations.ts index 29a5732026761..f9e19ba6f757e 100644 --- a/x-pack/plugins/ml/public/application/services/ml_api_service/annotations.ts +++ b/x-pack/plugins/ml/public/application/services/ml_api_service/annotations.ts @@ -4,7 +4,11 @@ * you may not use this file except in compliance with the Elastic License. */ -import { Annotation } from '../../../../common/types/annotations'; +import { + Annotation, + FieldToBucket, + GetAnnotationsResponse, +} from '../../../../common/types/annotations'; import { http, http$ } from '../http_service'; import { basePath } from './index'; @@ -14,15 +18,19 @@ export const annotations = { earliestMs: number; latestMs: number; maxAnnotations: number; + fields: FieldToBucket[]; + detectorIndex: number; + entities: any[]; }) { const body = JSON.stringify(obj); - return http$<{ annotations: Record }>({ + return http$({ path: `${basePath()}/annotations`, method: 'POST', body, }); }, - indexAnnotation(obj: any) { + + indexAnnotation(obj: Annotation) { const body = JSON.stringify(obj); return http({ path: `${basePath()}/annotations/index`, diff --git a/x-pack/plugins/ml/public/application/timeseriesexplorer/timeseriesexplorer.js b/x-pack/plugins/ml/public/application/timeseriesexplorer/timeseriesexplorer.js index d4470e7502e0d..95dc1ed6988f6 100644 --- a/x-pack/plugins/ml/public/application/timeseriesexplorer/timeseriesexplorer.js +++ b/x-pack/plugins/ml/public/application/timeseriesexplorer/timeseriesexplorer.js @@ -28,6 +28,8 @@ import { EuiSelect, EuiSpacer, EuiTitle, + EuiAccordion, + EuiBadge, } from '@elastic/eui'; import { getToastNotifications } from '../util/dependency_cache'; @@ -125,6 +127,8 @@ function getTimeseriesexplorerDefaultState() { entitiesLoading: false, entityValues: {}, focusAnnotationData: [], + focusAggregations: {}, + focusAggregationInterval: {}, focusChartData: undefined, focusForecastData: undefined, fullRefresh: true, @@ -1025,6 +1029,7 @@ export class TimeSeriesExplorer extends React.Component { entityValues, focusAggregationInterval, focusAnnotationData, + focusAggregations, focusChartData, focusForecastData, fullRefresh, @@ -1075,8 +1080,8 @@ export class TimeSeriesExplorer extends React.Component { const entityControls = this.getControlsForDetector(); const fieldNamesWithEmptyValues = this.getFieldNamesWithEmptyValues(); const arePartitioningFieldsProvided = this.arePartitioningFieldsProvided(); - - const detectorSelectOptions = getViewableDetectors(selectedJob).map((d) => ({ + const detectors = getViewableDetectors(selectedJob); + const detectorSelectOptions = detectors.map((d) => ({ value: d.index, text: d.detector_description, })); @@ -1311,25 +1316,49 @@ export class TimeSeriesExplorer extends React.Component { )} - {showAnnotations && focusAnnotationData.length > 0 && ( -
- -

- -

- + {focusAnnotationData && focusAnnotationData.length > 0 && ( + +

+ + + + ), + }} + /> +

+ + } + > -
+ )} - +

number; @@ -37,6 +38,7 @@ export interface FocusData { showForecastCheckbox?: any; focusAnnotationData?: any; focusForecastData?: any; + focusAggregations?: any; } export function getFocusData( @@ -84,11 +86,23 @@ export function getFocusData( earliestMs: searchBounds.min.valueOf(), latestMs: searchBounds.max.valueOf(), maxAnnotations: ANNOTATIONS_TABLE_DEFAULT_QUERY_SIZE, + fields: [ + { + field: 'event', + missing: ANNOTATION_EVENT_USER, + }, + ], + detectorIndex, + entities: nonBlankEntities, }) .pipe( catchError(() => { // silent fail - return of({ annotations: {} as Record }); + return of({ + annotations: {} as Record, + aggregations: {}, + success: false, + }); }) ), // Plus query for forecast data if there is a forecastId stored in the appState. @@ -146,13 +160,14 @@ export function getFocusData( d.key = String.fromCharCode(65 + i); return d; }); + + refreshFocusData.focusAggregations = annotations.aggregations; } if (forecastData) { refreshFocusData.focusForecastData = processForecastResults(forecastData.results); refreshFocusData.showForecastCheckbox = refreshFocusData.focusForecastData.length > 0; } - return refreshFocusData; }) ); diff --git a/x-pack/plugins/ml/server/models/annotation_service/annotation.ts b/x-pack/plugins/ml/server/models/annotation_service/annotation.ts index c2582107062bb..f7353034b7453 100644 --- a/x-pack/plugins/ml/server/models/annotation_service/annotation.ts +++ b/x-pack/plugins/ml/server/models/annotation_service/annotation.ts @@ -8,7 +8,8 @@ import Boom from 'boom'; import _ from 'lodash'; import { ILegacyScopedClusterClient } from 'kibana/server'; -import { ANNOTATION_TYPE } from '../../../common/constants/annotations'; +import { ANNOTATION_EVENT_USER, ANNOTATION_TYPE } from '../../../common/constants/annotations'; +import { PARTITION_FIELDS } from '../../../common/constants/anomalies'; import { ML_ANNOTATIONS_INDEX_ALIAS_READ, ML_ANNOTATIONS_INDEX_ALIAS_WRITE, @@ -19,20 +20,35 @@ import { Annotations, isAnnotation, isAnnotations, + getAnnotationFieldName, + getAnnotationFieldValue, + EsAggregationResult, } from '../../../common/types/annotations'; // TODO All of the following interface/type definitions should // eventually be replaced by the proper upstream definitions interface EsResult { - _source: object; + _source: Annotation; _id: string; } +export interface FieldToBucket { + field: string; + missing?: string | number; +} + export interface IndexAnnotationArgs { jobIds: string[]; earliestMs: number; latestMs: number; maxAnnotations: number; + fields?: FieldToBucket[]; + detectorIndex?: number; + entities?: any[]; +} + +export interface AggTerm { + terms: FieldToBucket; } export interface GetParams { @@ -43,9 +59,8 @@ export interface GetParams { export interface GetResponse { success: true; - annotations: { - [key: string]: Annotations; - }; + annotations: Record; + aggregations: EsAggregationResult; } export interface IndexParams { @@ -96,10 +111,14 @@ export function annotationProvider({ callAsCurrentUser }: ILegacyScopedClusterCl earliestMs, latestMs, maxAnnotations, + fields, + detectorIndex, + entities, }: IndexAnnotationArgs) { const obj: GetResponse = { success: true, annotations: {}, + aggregations: {}, }; const boolCriteria: object[] = []; @@ -182,6 +201,64 @@ export function annotationProvider({ callAsCurrentUser }: ILegacyScopedClusterCl }); } + // Find unique buckets (e.g. events) from the queried annotations to show in dropdowns + const aggs: Record = {}; + if (fields) { + fields.forEach((fieldToBucket) => { + aggs[fieldToBucket.field] = { + terms: { + ...fieldToBucket, + }, + }; + }); + } + + // Build should clause to further query for annotations in SMV + // we want to show either the exact match with detector index and by/over/partition fields + // OR annotations without any partition fields defined + let shouldClauses; + if (detectorIndex !== undefined && Array.isArray(entities)) { + // build clause to get exact match of detector index and by/over/partition fields + const beExactMatch = []; + beExactMatch.push({ + term: { + detector_index: detectorIndex, + }, + }); + + entities.forEach(({ fieldName, fieldType, fieldValue }) => { + beExactMatch.push({ + term: { + [getAnnotationFieldName(fieldType)]: fieldName, + }, + }); + beExactMatch.push({ + term: { + [getAnnotationFieldValue(fieldType)]: fieldValue, + }, + }); + }); + + // clause to get annotations that have no partition fields + const haveAnyPartitionFields: object[] = []; + PARTITION_FIELDS.forEach((field) => { + haveAnyPartitionFields.push({ + exists: { + field: getAnnotationFieldName(field), + }, + }); + haveAnyPartitionFields.push({ + exists: { + field: getAnnotationFieldValue(field), + }, + }); + }); + shouldClauses = [ + { bool: { must_not: haveAnyPartitionFields } }, + { bool: { must: beExactMatch } }, + ]; + } + const params: GetParams = { index: ML_ANNOTATIONS_INDEX_ALIAS_READ, size: maxAnnotations, @@ -201,8 +278,10 @@ export function annotationProvider({ callAsCurrentUser }: ILegacyScopedClusterCl }, }, ], + ...(shouldClauses ? { should: shouldClauses, minimum_should_match: 1 } : {}), }, }, + ...(fields ? { aggs } : {}), }, }; @@ -217,9 +296,19 @@ export function annotationProvider({ callAsCurrentUser }: ILegacyScopedClusterCl const docs: Annotations = _.get(resp, ['hits', 'hits'], []).map((d: EsResult) => { // get the original source document and the document id, we need it // to identify the annotation when editing/deleting it. - return { ...d._source, _id: d._id } as Annotation; + // if original `event` is undefined then substitute with 'user` by default + // since annotation was probably generated by user on the UI + return { + ...d._source, + event: d._source?.event ?? ANNOTATION_EVENT_USER, + _id: d._id, + } as Annotation; }); + const aggregations = _.get(resp, ['aggregations'], {}) as EsAggregationResult; + if (fields) { + obj.aggregations = aggregations; + } if (isAnnotations(docs) === false) { // No need to translate, this will not be exposed in the UI. throw new Error(`Annotations didn't pass integrity check.`); diff --git a/x-pack/plugins/ml/server/models/results_service/get_partition_fields_values.ts b/x-pack/plugins/ml/server/models/results_service/get_partition_fields_values.ts index d7403c45f1be2..663ee846571e7 100644 --- a/x-pack/plugins/ml/server/models/results_service/get_partition_fields_values.ts +++ b/x-pack/plugins/ml/server/models/results_service/get_partition_fields_values.ts @@ -6,13 +6,11 @@ import Boom from 'boom'; import { ILegacyScopedClusterClient } from 'kibana/server'; +import { PARTITION_FIELDS } from '../../../common/constants/anomalies'; +import { PartitionFieldsType } from '../../../common/types/anomalies'; import { ML_RESULTS_INDEX_PATTERN } from '../../../common/constants/index_patterns'; import { CriteriaField } from './results_service'; -const PARTITION_FIELDS = ['partition_field', 'over_field', 'by_field'] as const; - -type PartitionFieldsType = typeof PARTITION_FIELDS[number]; - type SearchTerm = | { [key in PartitionFieldsType]?: string; diff --git a/x-pack/plugins/ml/server/routes/schemas/annotations_schema.ts b/x-pack/plugins/ml/server/routes/schemas/annotations_schema.ts index fade2093ac842..14a2f632419bc 100644 --- a/x-pack/plugins/ml/server/routes/schemas/annotations_schema.ts +++ b/x-pack/plugins/ml/server/routes/schemas/annotations_schema.ts @@ -16,6 +16,14 @@ export const indexAnnotationSchema = schema.object({ create_username: schema.maybe(schema.string()), modified_time: schema.maybe(schema.number()), modified_username: schema.maybe(schema.string()), + event: schema.maybe(schema.string()), + detector_index: schema.maybe(schema.number()), + partition_field_name: schema.maybe(schema.string()), + partition_field_value: schema.maybe(schema.string()), + over_field_name: schema.maybe(schema.string()), + over_field_value: schema.maybe(schema.string()), + by_field_name: schema.maybe(schema.string()), + by_field_value: schema.maybe(schema.string()), /** Document id */ _id: schema.maybe(schema.string()), key: schema.maybe(schema.string()), @@ -26,6 +34,25 @@ export const getAnnotationsSchema = schema.object({ earliestMs: schema.oneOf([schema.nullable(schema.number()), schema.maybe(schema.number())]), latestMs: schema.oneOf([schema.nullable(schema.number()), schema.maybe(schema.number())]), maxAnnotations: schema.number(), + /** Fields to find unique values for (e.g. events or created_by) */ + fields: schema.maybe( + schema.arrayOf( + schema.object({ + field: schema.string(), + missing: schema.maybe(schema.string()), + }) + ) + ), + detectorIndex: schema.maybe(schema.number()), + entities: schema.maybe( + schema.arrayOf( + schema.object({ + fieldType: schema.maybe(schema.string()), + fieldName: schema.maybe(schema.string()), + fieldValue: schema.maybe(schema.string()), + }) + ) + ), }); export const deleteAnnotationSchema = schema.object({ annotationId: schema.string() }); diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index 2c7ce81d4de72..78878ea1638c1 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -9698,7 +9698,6 @@ "xpack.ml.datavisualizerBreadcrumbLabel": "データビジュアライザー", "xpack.ml.dataVisualizerPageLabel": "データビジュアライザー", "xpack.ml.dfAnalyticsList.analyticsDetails.messagesPane.errorMessage": "メッセージを読み込めませんでした", - "xpack.ml.explorer.annotationsTitle": "注釈", "xpack.ml.explorer.anomaliesTitle": "異常", "xpack.ml.explorer.anomalyTimelineTitle": "異常のタイムライン", "xpack.ml.explorer.charts.detectorLabel": "「{fieldName}」で分割された {detectorLabel}{br} Y 軸イベントの分布", @@ -10814,7 +10813,6 @@ "xpack.ml.timeSeriesExplorer.annotationFlyout.noAnnotationTextError": "注釈テキストを入力してください", "xpack.ml.timeSeriesExplorer.annotationFlyout.updateButtonLabel": "更新", "xpack.ml.timeSeriesExplorer.annotationsLabel": "注釈", - "xpack.ml.timeSeriesExplorer.annotationsTitle": "注釈", "xpack.ml.timeSeriesExplorer.anomaliesTitle": "異常", "xpack.ml.timeSeriesExplorer.autoSelectingFirstJobText": "、初めのジョブを自動選択します", "xpack.ml.timeSeriesExplorer.canNotViewRequestedJobsWarningMessage": "リクエストされた‘{invalidIdsCount, plural, one {ジョブ} other {件のジョブ}} {invalidIds} をこのダッシュボードで表示できません", diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index 76bf22f302f25..c9070a5b7cfa5 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -9702,7 +9702,6 @@ "xpack.ml.datavisualizerBreadcrumbLabel": "数据可视化工具", "xpack.ml.dataVisualizerPageLabel": "数据可视化工具", "xpack.ml.dfAnalyticsList.analyticsDetails.messagesPane.errorMessage": "无法加载消息", - "xpack.ml.explorer.annotationsTitle": "注释", "xpack.ml.explorer.anomaliesTitle": "异常", "xpack.ml.explorer.anomalyTimelineTitle": "异常时间线", "xpack.ml.explorer.charts.detectorLabel": "{detectorLabel}{br}y 轴事件分布按 “{fieldName}” 分割", @@ -10818,7 +10817,6 @@ "xpack.ml.timeSeriesExplorer.annotationFlyout.noAnnotationTextError": "输入注释文本", "xpack.ml.timeSeriesExplorer.annotationFlyout.updateButtonLabel": "更新", "xpack.ml.timeSeriesExplorer.annotationsLabel": "注释", - "xpack.ml.timeSeriesExplorer.annotationsTitle": "注释", "xpack.ml.timeSeriesExplorer.anomaliesTitle": "异常", "xpack.ml.timeSeriesExplorer.autoSelectingFirstJobText": "，自动选择第一个作业", "xpack.ml.timeSeriesExplorer.canNotViewRequestedJobsWarningMessage": "您无法在此仪表板中查看请求的 {invalidIdsCount, plural, one {作业} other {作业}} {invalidIds}",