diff --git a/packages/kbn-alerts-as-data-utils/src/field_maps/types.ts b/packages/kbn-alerts-as-data-utils/src/field_maps/types.ts index 04f9d045f6e28..0a0b68a2f26e6 100644 --- a/packages/kbn-alerts-as-data-utils/src/field_maps/types.ts +++ b/packages/kbn-alerts-as-data-utils/src/field_maps/types.ts @@ -34,6 +34,7 @@ export interface EcsMetadata { scaling_factor?: number; short: string; type: string; + properties?: Record; } export interface FieldMap { @@ -50,5 +51,6 @@ export interface FieldMap { path?: string; scaling_factor?: number; dynamic?: boolean | 'strict'; + properties?: Record; }; } diff --git a/packages/kbn-alerts-as-data-utils/src/schemas/create_schema_from_field_map.ts b/packages/kbn-alerts-as-data-utils/src/schemas/create_schema_from_field_map.ts index 99bbb502e1011..a0599d85fab33 100644 --- a/packages/kbn-alerts-as-data-utils/src/schemas/create_schema_from_field_map.ts +++ b/packages/kbn-alerts-as-data-utils/src/schemas/create_schema_from_field_map.ts @@ -198,6 +198,7 @@ const generateSchemaLines = ({ break; case 'float': case 'integer': + case 'double': lineWriter.addLine(`${keyToWrite}: ${getSchemaDefinition('schemaNumber', isArray)},`); break; case 'boolean': diff --git a/packages/kbn-alerts-as-data-utils/src/schemas/generated/ml_anomaly_detection_schema.ts b/packages/kbn-alerts-as-data-utils/src/schemas/generated/ml_anomaly_detection_schema.ts new file mode 100644 index 0000000000000..2e5912bca84c2 --- /dev/null +++ b/packages/kbn-alerts-as-data-utils/src/schemas/generated/ml_anomaly_detection_schema.ts @@ -0,0 +1,120 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +// ---------------------------------- WARNING ---------------------------------- +// this file was generated, and should not be edited by hand +// ---------------------------------- WARNING ---------------------------------- +import * as rt from 'io-ts'; +import { Either } from 'fp-ts/lib/Either'; +import { AlertSchema } from './alert_schema'; +const ISO_DATE_PATTERN = /^d{4}-d{2}-d{2}Td{2}:d{2}:d{2}.d{3}Z$/; +export const IsoDateString = new rt.Type( + 'IsoDateString', + rt.string.is, + (input, context): Either => { + if (typeof input === 'string' && ISO_DATE_PATTERN.test(input)) { + return rt.success(input); + } else { + return rt.failure(input, context); + } + }, + rt.identity +); +export type IsoDateStringC = typeof IsoDateString; +export const schemaDate = IsoDateString; +export const schemaDateArray = rt.array(IsoDateString); +export const schemaDateRange = rt.partial({ + gte: schemaDate, + lte: schemaDate, +}); +export const schemaDateRangeArray = rt.array(schemaDateRange); +export const schemaUnknown = rt.unknown; +export const schemaUnknownArray = rt.array(rt.unknown); +export const schemaString = rt.string; +export const schemaStringArray = rt.array(schemaString); +export const schemaNumber = rt.number; +export const schemaNumberArray = rt.array(schemaNumber); +export const schemaStringOrNumber = rt.union([schemaString, schemaNumber]); +export const schemaStringOrNumberArray = rt.array(schemaStringOrNumber); +export const schemaBoolean = rt.boolean; +export const schemaBooleanArray = rt.array(schemaBoolean); +const schemaGeoPointCoords = rt.type({ + type: schemaString, + coordinates: schemaNumberArray, +}); +const schemaGeoPointString = schemaString; +const schemaGeoPointLatLon = rt.type({ + lat: schemaNumber, + lon: schemaNumber, +}); +const schemaGeoPointLocation = rt.type({ + location: schemaNumberArray, +}); +const schemaGeoPointLocationString = rt.type({ + location: schemaString, +}); +export const schemaGeoPoint = rt.union([ + schemaGeoPointCoords, + schemaGeoPointString, + schemaGeoPointLatLon, + schemaGeoPointLocation, + schemaGeoPointLocationString, +]); +export const schemaGeoPointArray = rt.array(schemaGeoPoint); +// prettier-ignore +const MlAnomalyDetectionAlertRequired = rt.type({ + kibana: rt.type({ + alert: rt.type({ + job_id: schemaString, + }), + }), +}); +const MlAnomalyDetectionAlertOptional = rt.partial({ + kibana: rt.partial({ + alert: rt.partial({ + anomaly_score: schemaNumber, + anomaly_timestamp: schemaDate, + is_interim: schemaBoolean, + top_influencers: rt.array( + rt.partial({ + influencer_field_name: schemaString, + influencer_field_value: schemaString, + influencer_score: schemaNumber, + initial_influencer_score: schemaNumber, + is_interim: schemaBoolean, + job_id: schemaString, + timestamp: schemaDate, + }) + ), + top_records: rt.array( + rt.partial({ + actual: schemaNumber, + by_field_name: schemaString, + by_field_value: schemaString, + detector_index: schemaNumber, + field_name: schemaString, + function: schemaString, + initial_record_score: schemaNumber, + is_interim: schemaBoolean, + job_id: schemaString, + over_field_name: schemaString, + over_field_value: schemaString, + partition_field_name: schemaString, + partition_field_value: schemaString, + record_score: schemaNumber, + timestamp: schemaDate, + typical: schemaNumber, + }) + ), + }), + }), +}); + +// prettier-ignore +export const MlAnomalyDetectionAlertSchema = rt.intersection([MlAnomalyDetectionAlertRequired, MlAnomalyDetectionAlertOptional, AlertSchema]); +// prettier-ignore +export type MlAnomalyDetectionAlert = rt.TypeOf; diff --git a/packages/kbn-alerts-as-data-utils/src/schemas/index.ts b/packages/kbn-alerts-as-data-utils/src/schemas/index.ts index 77d9476d2034b..28da937087cf1 100644 --- a/packages/kbn-alerts-as-data-utils/src/schemas/index.ts +++ b/packages/kbn-alerts-as-data-utils/src/schemas/index.ts @@ -13,6 +13,7 @@ import type { ObservabilityMetricsAlert } from './generated/observability_metric import type { ObservabilitySloAlert } from './generated/observability_slo_schema'; import type { ObservabilityUptimeAlert } from './generated/observability_uptime_schema'; import type { SecurityAlert } from './generated/security_schema'; +import type { MlAnomalyDetectionAlert } from './generated/ml_anomaly_detection_schema'; export * from './create_schema_from_field_map'; @@ -24,6 +25,7 @@ export type { ObservabilitySloAlert } from './generated/observability_slo_schema export type { ObservabilityUptimeAlert } from './generated/observability_uptime_schema'; export type { SecurityAlert } from './generated/security_schema'; export type { StackAlert } from './generated/stack_schema'; +export type { MlAnomalyDetectionAlert } from './generated/ml_anomaly_detection_schema'; export type AADAlert = | Alert @@ -32,4 +34,5 @@ export type AADAlert = | ObservabilityMetricsAlert | ObservabilitySloAlert | ObservabilityUptimeAlert - | SecurityAlert; + | SecurityAlert + | MlAnomalyDetectionAlert; diff --git a/x-pack/plugins/alerting/common/alert_schema/context_to_schema_name.ts b/x-pack/plugins/alerting/common/alert_schema/context_to_schema_name.ts index 52435b05dbaff..e2268f5e1429f 100644 --- a/x-pack/plugins/alerting/common/alert_schema/context_to_schema_name.ts +++ b/x-pack/plugins/alerting/common/alert_schema/context_to_schema_name.ts @@ -9,7 +9,7 @@ import { capitalize } from 'lodash'; export const contextToSchemaName = (context: string) => { return `${context - .split('.') + .split(/[.\-]/) .map((part: string) => capitalize(part)) .join('')}Alert`; }; diff --git a/x-pack/plugins/ml/common/types/alerts.ts b/x-pack/plugins/ml/common/types/alerts.ts index 267096e105ef6..12bae1a9d3d16 100644 --- a/x-pack/plugins/ml/common/types/alerts.ts +++ b/x-pack/plugins/ml/common/types/alerts.ts @@ -44,6 +44,35 @@ interface BaseAnomalyAlertDoc { unique_key: string; } +export interface TopRecordAADDoc { + job_id: string; + record_score: number; + initial_record_score: number; + timestamp: number; + is_interim: boolean; + function: string; + field_name?: string; + by_field_name?: string; + by_field_value?: string | number; + over_field_name?: string; + over_field_value?: string | number; + partition_field_name?: string; + partition_field_value?: string | number; + typical: number[]; + actual: number[]; + detector_index: number; +} + +export interface TopInfluencerAADDoc { + job_id: string; + influencer_score: number; + initial_influencer_score: number; + is_interim: boolean; + timestamp: number; + influencer_field_name: string; + influencer_field_value: string | number; +} + export interface RecordAnomalyAlertDoc extends BaseAnomalyAlertDoc { result_type: typeof ML_ANOMALY_RESULT_TYPE.RECORD; function: string; diff --git a/x-pack/plugins/ml/server/lib/alerts/alerting_service.ts b/x-pack/plugins/ml/server/lib/alerts/alerting_service.ts index 18be37a187c44..dc85428a63386 100644 --- a/x-pack/plugins/ml/server/lib/alerts/alerting_service.ts +++ b/x-pack/plugins/ml/server/lib/alerts/alerting_service.ts @@ -9,7 +9,7 @@ import Boom from '@hapi/boom'; import { i18n } from '@kbn/i18n'; import rison from '@kbn/rison'; import type { Duration } from 'moment/moment'; -import { memoize } from 'lodash'; +import { memoize, pick } from 'lodash'; import { FIELD_FORMAT_IDS, type IFieldFormat, @@ -24,6 +24,7 @@ import { ML_ANOMALY_RESULT_TYPE, } from '@kbn/ml-anomaly-utils'; import { DEFAULT_SPACE_ID } from '@kbn/spaces-plugin/common'; +import { ALERT_REASON, ALERT_URL } from '@kbn/rule-data-utils'; import type { MlClient } from '../ml_client'; import type { MlAnomalyDetectionAlertParams, @@ -36,8 +37,13 @@ import type { PreviewResultsKeys, RecordAnomalyAlertDoc, TopHitsResultsKeys, + TopInfluencerAADDoc, + TopRecordAADDoc, } from '../../../common/types/alerts'; -import type { AnomalyDetectionAlertContext } from './register_anomaly_detection_alert_type'; +import type { + AnomalyDetectionAlertContext, + AnomalyDetectionAlertPayload, +} from './register_anomaly_detection_alert_type'; import { resolveMaxTimeInterval } from '../../../common/util/job_utils'; import { getTopNBuckets, resolveLookbackInterval } from '../../../common/util/alerts'; import type { DatafeedsService } from '../../models/job_service/datafeeds'; @@ -391,12 +397,89 @@ export function alertingServiceProvider( return alertInstanceKey; }; + /** + * Returns a callback for formatting elasticsearch aggregation response + * to the alert-as-data document. + * @param resultType + */ + const getResultsToPayloadFormatter = ( + resultType: MlAnomalyResultType, + useInitialScore: boolean = false + ) => { + const resultsLabel = getAggResultsLabel(resultType); + + return ( + v: AggResultsResponse + ): Omit | undefined => { + const aggTypeResults = v[resultsLabel.aggGroupLabel]; + if (aggTypeResults.doc_count === 0) { + return; + } + const requestedAnomalies = aggTypeResults[resultsLabel.topHitsLabel].hits.hits; + const topAnomaly = requestedAnomalies[0]; + const timestamp = topAnomaly._source.timestamp; + + return { + [ALERT_REASON]: i18n.translate( + 'xpack.ml.alertTypes.anomalyDetectionAlertingRule.alertMessage', + { + defaultMessage: + 'Alerts are raised based on real-time scores. Remember that scores may be adjusted over time as data continues to be analyzed.', + } + ), + job_id: [...new Set(requestedAnomalies.map((h) => h._source.job_id))][0], + is_interim: requestedAnomalies.some((h) => h._source.is_interim), + anomaly_timestamp: timestamp, + anomaly_score: topAnomaly._source[getScoreFields(resultType, useInitialScore)], + top_records: v.record_results.top_record_hits.hits.hits.map((h) => { + const { actual, typical } = getTypicalAndActualValues(h._source); + return pick( + { + ...h._source, + typical, + actual, + }, + [ + 'job_id', + 'record_score', + 'initial_record_score', + 'detector_index', + 'is_interim', + 'timestamp', + 'partition_field_name', + 'partition_field_value', + 'function', + 'actual', + 'typical', + ] + ) as TopRecordAADDoc; + }) as TopRecordAADDoc[], + top_influencers: v.influencer_results.top_influencer_hits.hits.hits.map((influencerDoc) => { + return pick( + { + ...influencerDoc._source, + }, + [ + 'job_id', + 'influencer_field_name', + 'influencer_field_value', + 'influencer_score', + 'initial_influencer_score', + 'is_interim', + 'timestamp', + ] + ) as TopInfluencerAADDoc; + }) as TopInfluencerAADDoc[], + }; + }; + }; + /** * Returns a callback for formatting elasticsearch aggregation response * to the alert context. * @param resultType */ - const getResultsFormatter = ( + const getResultsToContextFormatter = ( resultType: MlAnomalyResultType, useInitialScore: boolean = false, formatters: FieldFormatters @@ -468,7 +551,7 @@ export function alertingServiceProvider( * @param previewTimeInterval - Relative time interval to test the alert condition * @param checkIntervalGap - Interval between alert executions */ - const fetchAnomalies = async ( + const fetchPreviewResults = async ( params: MlAnomalyDetectionAlertParams, previewTimeInterval?: string, checkIntervalGap?: Duration @@ -570,7 +653,7 @@ export function alertingServiceProvider( const fieldsFormatters = await getFormatters(datafeeds![0]!.indices[0]); - const formatter = getResultsFormatter( + const formatter = getResultsToContextFormatter( params.resultType, !!previewTimeInterval, fieldsFormatters @@ -660,7 +743,7 @@ export function alertingServiceProvider( */ const fetchResult = async ( params: AnomalyESQueryParams - ): Promise => { + ): Promise => { const { resultType, jobIds, @@ -670,7 +753,6 @@ export function alertingServiceProvider( anomalyScoreField, includeInterimResults, anomalyScoreThreshold, - indexPattern, } = params; const requestBody = { @@ -761,9 +843,44 @@ export function alertingServiceProvider( prev.max_score.value > current.max_score.value ? prev : current ); + return topResult; + }; + + const getFormatted = async ( + indexPattern: string, + resultType: MlAnomalyDetectionAlertParams['resultType'], + spaceId: string, + value: AggResultsResponse + ): Promise< + | { payload: AnomalyDetectionAlertPayload; context: AnomalyDetectionAlertContext; name: string } + | undefined + > => { const formatters = await getFormatters(indexPattern); - return getResultsFormatter(params.resultType, false, formatters)(topResult); + const context = getResultsToContextFormatter(resultType, false, formatters)(value); + const payload = getResultsToPayloadFormatter(resultType, false)(value); + + if (!context || !payload) return; + + const anomalyExplorerUrl = buildExplorerUrl( + context.jobIds, + { from: context.bucketRange.start, to: context.bucketRange.end }, + resultType, + spaceId, + context + ); + + return { + payload: { + ...payload, + [ALERT_URL]: anomalyExplorerUrl, + }, + context: { + ...context, + anomalyExplorerUrl, + }, + name: context.alertInstanceKey, + }; }; return { @@ -777,7 +894,13 @@ export function alertingServiceProvider( params: MlAnomalyDetectionAlertParams, spaceId: string ): Promise< - { context: AnomalyDetectionAlertContext; name: string; isHealthy: boolean } | undefined + | { + payload: AnomalyDetectionAlertPayload; + context: AnomalyDetectionAlertContext; + name: string; + isHealthy: boolean; + } + | undefined > => { const queryParams = await getQueryParams(params); @@ -787,50 +910,57 @@ export function alertingServiceProvider( const result = await fetchResult(queryParams); - if (result) { - const anomalyExplorerUrl = buildExplorerUrl( - result.jobIds, - { from: result.bucketRange.start, to: result.bucketRange.end }, - params.resultType, - spaceId, - result + const formattedResult = result + ? await getFormatted(queryParams.indexPattern, queryParams.resultType, spaceId, result) + : undefined; + + if (!formattedResult) { + // If no anomalies found, report as recovered. + + const url = buildExplorerUrl( + queryParams.jobIds, + { + from: `now-${queryParams.lookBackTimeInterval}`, + to: 'now', + mode: 'relative', + }, + queryParams.resultType, + spaceId ); - const executionResult = { - ...result, - anomalyExplorerUrl, - }; + const message = i18n.translate( + 'xpack.ml.alertTypes.anomalyDetectionAlertingRule.recoveredMessage', + { + defaultMessage: + 'No anomalies have been found in the past {lookbackInterval} that exceed the severity threshold of {severity}.', + values: { + severity: queryParams.anomalyScoreThreshold, + lookbackInterval: queryParams.lookBackTimeInterval, + }, + } + ); - return { context: executionResult, name: result.alertInstanceKey, isHealthy: false }; + return { + name: '', + isHealthy: true, + payload: { + [ALERT_URL]: url, + [ALERT_REASON]: message, + job_id: queryParams.jobIds[0], + }, + context: { + anomalyExplorerUrl: url, + jobIds: queryParams.jobIds, + message, + } as AnomalyDetectionAlertContext, + }; } return { - name: '', - isHealthy: true, - context: { - anomalyExplorerUrl: buildExplorerUrl( - queryParams.jobIds, - { - from: `now-${queryParams.lookBackTimeInterval}`, - to: 'now', - mode: 'relative', - }, - queryParams.resultType, - spaceId - ), - jobIds: queryParams.jobIds, - message: i18n.translate( - 'xpack.ml.alertTypes.anomalyDetectionAlertingRule.recoveredMessage', - { - defaultMessage: - 'No anomalies have been found in the past {lookbackInterval} that exceed the severity threshold of {severity}.', - values: { - severity: queryParams.anomalyScoreThreshold, - lookbackInterval: queryParams.lookBackTimeInterval, - }, - } - ), - } as AnomalyDetectionAlertContext, + context: formattedResult.context, + payload: formattedResult.payload, + name: formattedResult.name, + isHealthy: false, }; }, /** @@ -844,16 +974,16 @@ export function alertingServiceProvider( timeRange, sampleSize, }: MlAnomalyDetectionAlertPreviewRequest): Promise => { - const res = await fetchAnomalies(alertParams, timeRange); + const previewResults = await fetchPreviewResults(alertParams, timeRange); - if (!res) { + if (!previewResults) { throw Boom.notFound(`No results found`); } return { // sum of all alert responses within the time range - count: res.length, - results: res.slice(0, sampleSize), + count: previewResults.length, + results: previewResults.slice(0, sampleSize), }; }, }; diff --git a/x-pack/plugins/ml/server/lib/alerts/register_anomaly_detection_alert_type.ts b/x-pack/plugins/ml/server/lib/alerts/register_anomaly_detection_alert_type.ts index 2a1f19b48802e..2935643348f76 100644 --- a/x-pack/plugins/ml/server/lib/alerts/register_anomaly_detection_alert_type.ts +++ b/x-pack/plugins/ml/server/lib/alerts/register_anomaly_detection_alert_type.ts @@ -11,8 +11,15 @@ import type { ActionGroup, AlertInstanceContext, AlertInstanceState, + RuleTypeParams, RuleTypeState, + RecoveredActionGroupId, } from '@kbn/alerting-plugin/common'; +import { IRuleTypeAlerts, RuleExecutorOptions } from '@kbn/alerting-plugin/server'; +import { ALERT_NAMESPACE, ALERT_REASON, ALERT_URL } from '@kbn/rule-data-utils'; +import { MlAnomalyDetectionAlert } from '@kbn/alerts-as-data-utils'; +import { ES_FIELD_TYPES } from '@kbn/field-types'; +import { expandFlattenedAlert } from '@kbn/alerting-plugin/server/alerts_client/lib'; import { ML_ALERT_TYPES } from '../../../common/constants/alerts'; import { PLUGIN_ID } from '../../../common/constants/app'; import { MINIMUM_FULL_LICENSE } from '../../../common/license'; @@ -36,6 +43,19 @@ export type AnomalyDetectionAlertBaseContext = AlertInstanceContext & { message: string; }; +// Flattened alert payload for alert-as-data +export type AnomalyDetectionAlertPayload = { + job_id: string; + anomaly_score?: number; + is_interim?: boolean; + anomaly_timestamp?: number; + top_records?: any; + top_influencers?: any; +} & { + [ALERT_URL]: string; + [ALERT_REASON]: string; +}; + export type AnomalyDetectionAlertContext = AnomalyDetectionAlertBaseContext & { timestampIso8601: string; timestamp: number; @@ -45,10 +65,88 @@ export type AnomalyDetectionAlertContext = AnomalyDetectionAlertBaseContext & { topInfluencers?: InfluencerAnomalyAlertDoc[]; }; +export type ExecutorOptions

= RuleExecutorOptions< + P, + RuleTypeState, + {}, + AnomalyDetectionAlertContext, + typeof ANOMALY_SCORE_MATCH_GROUP_ID, + MlAnomalyDetectionAlert +>; + export const ANOMALY_SCORE_MATCH_GROUP_ID = 'anomaly_score_match'; export type AnomalyScoreMatchGroupId = typeof ANOMALY_SCORE_MATCH_GROUP_ID; +export const ANOMALY_DETECTION_AAD_INDEX_NAME = 'ml.anomaly-detection'; + +const ML_ALERT_NAMESPACE = ALERT_NAMESPACE; + +export const ALERT_ANOMALY_DETECTION_JOB_ID = `${ML_ALERT_NAMESPACE}.job_id` as const; + +export const ALERT_ANOMALY_SCORE = `${ML_ALERT_NAMESPACE}.anomaly_score` as const; +export const ALERT_ANOMALY_IS_INTERIM = `${ML_ALERT_NAMESPACE}.is_interim` as const; +export const ALERT_ANOMALY_TIMESTAMP = `${ML_ALERT_NAMESPACE}.anomaly_timestamp` as const; + +export const ALERT_TOP_RECORDS = `${ML_ALERT_NAMESPACE}.top_records` as const; +export const ALERT_TOP_INFLUENCERS = `${ML_ALERT_NAMESPACE}.top_influencers` as const; + +export const ANOMALY_DETECTION_AAD_CONFIG: IRuleTypeAlerts = { + context: ANOMALY_DETECTION_AAD_INDEX_NAME, + mappings: { + fieldMap: { + [ALERT_ANOMALY_DETECTION_JOB_ID]: { + type: ES_FIELD_TYPES.KEYWORD, + array: false, + required: true, + }, + [ALERT_ANOMALY_SCORE]: { type: ES_FIELD_TYPES.DOUBLE, array: false, required: false }, + [ALERT_ANOMALY_IS_INTERIM]: { type: ES_FIELD_TYPES.BOOLEAN, array: false, required: false }, + [ALERT_ANOMALY_TIMESTAMP]: { type: ES_FIELD_TYPES.DATE, array: false, required: false }, + [ALERT_TOP_RECORDS]: { + type: ES_FIELD_TYPES.OBJECT, + array: true, + required: false, + dynamic: false, + properties: { + job_id: { type: ES_FIELD_TYPES.KEYWORD }, + record_score: { type: ES_FIELD_TYPES.DOUBLE }, + initial_record_score: { type: ES_FIELD_TYPES.DOUBLE }, + detector_index: { type: ES_FIELD_TYPES.INTEGER }, + is_interim: { type: ES_FIELD_TYPES.BOOLEAN }, + timestamp: { type: ES_FIELD_TYPES.DATE }, + partition_field_name: { type: ES_FIELD_TYPES.KEYWORD }, + partition_field_value: { type: ES_FIELD_TYPES.KEYWORD }, + over_field_name: { type: ES_FIELD_TYPES.KEYWORD }, + over_field_value: { type: ES_FIELD_TYPES.KEYWORD }, + by_field_name: { type: ES_FIELD_TYPES.KEYWORD }, + by_field_value: { type: ES_FIELD_TYPES.KEYWORD }, + function: { type: ES_FIELD_TYPES.KEYWORD }, + typical: { type: ES_FIELD_TYPES.DOUBLE }, + actual: { type: ES_FIELD_TYPES.DOUBLE }, + field_name: { type: ES_FIELD_TYPES.KEYWORD }, + }, + }, + [ALERT_TOP_INFLUENCERS]: { + type: ES_FIELD_TYPES.OBJECT, + array: true, + required: false, + dynamic: false, + properties: { + job_id: { type: ES_FIELD_TYPES.KEYWORD }, + influencer_field_name: { type: ES_FIELD_TYPES.KEYWORD }, + influencer_field_value: { type: ES_FIELD_TYPES.KEYWORD }, + influencer_score: { type: ES_FIELD_TYPES.DOUBLE }, + initial_influencer_score: { type: ES_FIELD_TYPES.DOUBLE }, + is_interim: { type: ES_FIELD_TYPES.BOOLEAN }, + timestamp: { type: ES_FIELD_TYPES.DATE }, + }, + }, + }, + }, + shouldWrite: true, +}; + export const THRESHOLD_MET_GROUP: ActionGroup = { id: ANOMALY_SCORE_MATCH_GROUP_ID, name: i18n.translate('xpack.ml.anomalyDetectionAlert.actionGroupName', { @@ -66,7 +164,9 @@ export function registerAnomalyDetectionAlertType({ RuleTypeState, AlertInstanceState, AnomalyDetectionAlertContext, - AnomalyScoreMatchGroupId + AnomalyScoreMatchGroupId, + RecoveredActionGroupId, + MlAnomalyDetectionAlert >({ id: ML_ALERT_TYPES.ANOMALY_DETECTION, name: i18n.translate('xpack.ml.anomalyDetectionAlert.name', { @@ -140,29 +240,62 @@ export function registerAnomalyDetectionAlertType({ minimumLicenseRequired: MINIMUM_FULL_LICENSE, isExportable: true, doesSetRecoveryContext: true, - async executor({ services, params, spaceId }) { + executor: async ({ + services, + params, + spaceId, + }: ExecutorOptions) => { const fakeRequest = {} as KibanaRequest; const { execute } = mlSharedServices.alertingServiceProvider( services.savedObjectsClient, fakeRequest ); + + const { alertsClient } = services; + if (!alertsClient) return { state: {} }; + const executionResult = await execute(params, spaceId); - if (executionResult && !executionResult.isHealthy) { - const alertInstanceName = executionResult.name; - const alertInstance = services.alertFactory.create(alertInstanceName); - alertInstance.scheduleActions(ANOMALY_SCORE_MATCH_GROUP_ID, executionResult.context); + if (!executionResult) return { state: {} }; + + const { isHealthy, name, context, payload } = executionResult; + + if (!isHealthy) { + alertsClient.report({ + id: name, + actionGroup: ANOMALY_SCORE_MATCH_GROUP_ID, + context, + payload: expandFlattenedAlert({ + [ALERT_URL]: payload[ALERT_URL], + [ALERT_REASON]: payload[ALERT_REASON], + [ALERT_ANOMALY_DETECTION_JOB_ID]: payload.job_id, + [ALERT_ANOMALY_SCORE]: payload.anomaly_score, + [ALERT_ANOMALY_IS_INTERIM]: payload.is_interim, + [ALERT_ANOMALY_TIMESTAMP]: payload.anomaly_timestamp, + [ALERT_TOP_RECORDS]: payload.top_records, + [ALERT_TOP_INFLUENCERS]: payload.top_influencers, + }), + }); } // Set context for recovered alerts - const { getRecoveredAlerts } = services.alertFactory.done(); - for (const recoveredAlert of getRecoveredAlerts()) { - if (!!executionResult?.isHealthy) { - recoveredAlert.setContext(executionResult.context); + for (const recoveredAlert of alertsClient.getRecoveredAlerts()) { + if (isHealthy) { + const alertId = recoveredAlert.alert.getId(); + alertsClient.setAlertData({ + id: alertId, + context, + payload: expandFlattenedAlert({ + [ALERT_URL]: payload[ALERT_URL], + [ALERT_REASON]: payload[ALERT_REASON], + [ALERT_ANOMALY_DETECTION_JOB_ID]: payload.job_id, + }), + }); } } return { state: {} }; }, + alerts: ANOMALY_DETECTION_AAD_CONFIG, }); } diff --git a/x-pack/plugins/ml/tsconfig.json b/x-pack/plugins/ml/tsconfig.json index 7962e2dd27296..4a0077770808a 100644 --- a/x-pack/plugins/ml/tsconfig.json +++ b/x-pack/plugins/ml/tsconfig.json @@ -106,5 +106,7 @@ "@kbn/react-kibana-mount", "@kbn/core-http-browser", "@kbn/data-view-editor-plugin", + "@kbn/rule-data-utils", + "@kbn/alerts-as-data-utils", ], } diff --git a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group2/ml_rule_types/anomaly_detection/alert.ts b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group2/ml_rule_types/anomaly_detection/alert.ts index 1c970174ad3a2..941f663bb6284 100644 --- a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group2/ml_rule_types/anomaly_detection/alert.ts +++ b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group2/ml_rule_types/anomaly_detection/alert.ts @@ -23,6 +23,8 @@ const ES_TEST_INDEX_SOURCE = 'ml-alert:anomaly-detection'; const ES_TEST_INDEX_REFERENCE = '-na-'; const ES_TEST_OUTPUT_INDEX_NAME = `${ES_TEST_INDEX_NAME}-ad-alert-output`; +const AAD_INDEX = '.alerts-ml.anomaly-detection.alerts-default'; + const ALERT_INTERVAL_SECONDS = 3; const AD_JOB_ID = 'rt-anomaly-mean-value'; @@ -144,6 +146,18 @@ export default function alertTests({ getService }: FtrProviderContext) { '/s/space1/app/ml/explorer/?_g=(ml%3A(jobIds%3A!(rt-anomaly-mean-value))' ); } + + log.debug('Checking docs in the alerts-as-data index...'); + + const aadDocs = await waitForAAD(1); + + for (const doc of aadDocs) { + const { job_id: jobId, url } = doc._source.kibana.alert; + expect(jobId).to.be(AD_JOB_ID); + expect(url).to.contain( + '/s/space1/app/ml/explorer/?_g=(ml%3A(jobIds%3A!(rt-anomaly-mean-value))' + ); + } }); async function waitForDocs(count: number): Promise { @@ -154,6 +168,20 @@ export default function alertTests({ getService }: FtrProviderContext) { ); } + async function waitForAAD(numDocs: number): Promise { + return await retry.try(async () => { + const searchResult = await es.search({ index: AAD_INDEX, size: 1000 }); + + // @ts-expect-error doesn't handle total: number + const value = searchResult.hits.total.value?.value || searchResult.hits.total.value; + if (value < numDocs) { + // @ts-expect-error doesn't handle total: number + throw new Error(`Expected ${numDocs} but received ${searchResult.hits.total.value}.`); + } + return searchResult.hits.hits; + }); + } + async function createAlert({ name, ...params diff --git a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group4/generate_alert_schemas.ts b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group4/generate_alert_schemas.ts index 478a9b17a21f5..51b1adc7526a9 100644 --- a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group4/generate_alert_schemas.ts +++ b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group4/generate_alert_schemas.ts @@ -71,7 +71,7 @@ export default function checkAlertSchemasTest({ getService }: FtrProviderContext createSchemaFromFieldMap({ outputFile: `schemas/generated/${alertsDefinition.context.replaceAll( - '.', + /[.\-]/g, '_' )}_schema.ts`, fieldMap: alertsDefinition.mappings.fieldMap,