From a28c0a36aee2d596dc057cf0424a3afa2ef727fb Mon Sep 17 00:00:00 2001 From: Phillip Burch Date: Tue, 29 Sep 2020 18:06:38 -0500 Subject: [PATCH] [Metrics UI] Add anomalies to timeline (#78602) * Add ability to fetch anomalies by metric * Add ability to fetch anomalies to timeline * Show anomaly annotation on timeline * Fix type check * Fix typos * Add influencers to tooltip, add legend * Remove unused variable * Only show anomalies with a score greater than 50 Co-authored-by: Elastic Machine --- .../http_api/infra_ml/results/common.ts | 8 + .../results/metrics_hosts_anomalies.ts | 6 +- .../infra_ml/results/metrics_k8s_anomalies.ts | 4 +- .../ml/anomaly_detection/flyout_home.tsx | 2 +- .../ml/anomaly_detection/job_setup_screen.tsx | 2 +- .../components/timeline/timeline.tsx | 150 ++++++++++++++++-- .../hooks/use_metrics_hosts_anomalies.ts | 10 +- .../hooks/use_metrics_k8s_anomalies.ts | 10 +- .../inventory_view/hooks/use_timeline.ts | 8 +- .../lib/infra_ml/metrics_hosts_anomalies.ts | 98 ++++-------- .../lib/infra_ml/metrics_k8s_anomalies.ts | 88 ++++------ .../server/lib/infra_ml/queries/common.ts | 10 ++ .../queries/metrics_hosts_anomalies.ts | 8 + .../infra_ml/queries/metrics_k8s_anomalies.ts | 11 +- .../results/metrics_hosts_anomalies.ts | 4 +- .../infra_ml/results/metrics_k8s_anomalies.ts | 2 + 16 files changed, 262 insertions(+), 159 deletions(-) diff --git a/x-pack/plugins/infra/common/http_api/infra_ml/results/common.ts b/x-pack/plugins/infra/common/http_api/infra_ml/results/common.ts index 0474fbd1cfc2f..deb8e6401008d 100644 --- a/x-pack/plugins/infra/common/http_api/infra_ml/results/common.ts +++ b/x-pack/plugins/infra/common/http_api/infra_ml/results/common.ts @@ -57,3 +57,11 @@ export const sortRT = rt.type({ }); export type Sort = rt.TypeOf; + +export const metricRT = rt.keyof({ + memory_usage: null, + network_in: null, + network_out: null, +}); + +export type Metric = rt.TypeOf; diff --git a/x-pack/plugins/infra/common/http_api/infra_ml/results/metrics_hosts_anomalies.ts b/x-pack/plugins/infra/common/http_api/infra_ml/results/metrics_hosts_anomalies.ts index 9fdac09fec20e..a08dd438a32c8 100644 --- a/x-pack/plugins/infra/common/http_api/infra_ml/results/metrics_hosts_anomalies.ts +++ b/x-pack/plugins/infra/common/http_api/infra_ml/results/metrics_hosts_anomalies.ts @@ -7,7 +7,7 @@ import * as rt from 'io-ts'; import { timeRangeRT, routeTimingMetadataRT } from '../../shared'; -import { anomalyTypeRT, paginationCursorRT, sortRT, paginationRT } from './common'; +import { anomalyTypeRT, paginationCursorRT, sortRT, paginationRT, metricRT } from './common'; export const INFA_ML_GET_METRICS_HOSTS_ANOMALIES_PATH = '/api/infra/infra_ml/results/metrics_hosts_anomalies'; @@ -18,6 +18,7 @@ const metricsHostAnomalyCommonFieldsRT = rt.type({ typical: rt.number, actual: rt.number, type: anomalyTypeRT, + influencers: rt.array(rt.string), duration: rt.number, startTime: rt.number, jobId: rt.string, @@ -64,12 +65,11 @@ export const getMetricsHostsAnomaliesRequestPayloadRT = rt.type({ timeRange: timeRangeRT, }), rt.partial({ + metric: metricRT, // Pagination properties pagination: paginationRT, // Sort properties sort: sortRT, - // // Dataset filters - // datasets: rt.array(rt.string), }), ]), }); diff --git a/x-pack/plugins/infra/common/http_api/infra_ml/results/metrics_k8s_anomalies.ts b/x-pack/plugins/infra/common/http_api/infra_ml/results/metrics_k8s_anomalies.ts index ab1f245a74c0c..7450bb39276ac 100644 --- a/x-pack/plugins/infra/common/http_api/infra_ml/results/metrics_k8s_anomalies.ts +++ b/x-pack/plugins/infra/common/http_api/infra_ml/results/metrics_k8s_anomalies.ts @@ -7,7 +7,7 @@ import * as rt from 'io-ts'; import { timeRangeRT, routeTimingMetadataRT } from '../../shared'; -import { paginationCursorRT, anomalyTypeRT, sortRT, paginationRT } from './common'; +import { paginationCursorRT, anomalyTypeRT, sortRT, paginationRT, metricRT } from './common'; export const INFA_ML_GET_METRICS_K8S_ANOMALIES_PATH = '/api/infra/infra_ml/results/metrics_k8s_anomalies'; @@ -18,6 +18,7 @@ const metricsK8sAnomalyCommonFieldsRT = rt.type({ typical: rt.number, actual: rt.number, type: anomalyTypeRT, + influencers: rt.array(rt.string), duration: rt.number, startTime: rt.number, jobId: rt.string, @@ -64,6 +65,7 @@ export const getMetricsK8sAnomaliesRequestPayloadRT = rt.type({ timeRange: timeRangeRT, }), rt.partial({ + metric: metricRT, // Pagination properties pagination: paginationRT, // Sort properties diff --git a/x-pack/plugins/infra/public/pages/metrics/inventory_view/components/ml/anomaly_detection/flyout_home.tsx b/x-pack/plugins/infra/public/pages/metrics/inventory_view/components/ml/anomaly_detection/flyout_home.tsx index 9cf898b684336..801dff9c4a17a 100644 --- a/x-pack/plugins/infra/public/pages/metrics/inventory_view/components/ml/anomaly_detection/flyout_home.tsx +++ b/x-pack/plugins/infra/public/pages/metrics/inventory_view/components/ml/anomaly_detection/flyout_home.tsx @@ -84,7 +84,7 @@ export const FlyoutHome = (props: Props) => { return ( ); diff --git a/x-pack/plugins/infra/public/pages/metrics/inventory_view/components/ml/anomaly_detection/job_setup_screen.tsx b/x-pack/plugins/infra/public/pages/metrics/inventory_view/components/ml/anomaly_detection/job_setup_screen.tsx index 730cd7b6e9ef5..428c002da6383 100644 --- a/x-pack/plugins/infra/public/pages/metrics/inventory_view/components/ml/anomaly_detection/job_setup_screen.tsx +++ b/x-pack/plugins/infra/public/pages/metrics/inventory_view/components/ml/anomaly_detection/job_setup_screen.tsx @@ -223,7 +223,7 @@ export const JobSetupScreen = (props: Props) => { label={ } compressed diff --git a/x-pack/plugins/infra/public/pages/metrics/inventory_view/components/timeline/timeline.tsx b/x-pack/plugins/infra/public/pages/metrics/inventory_view/components/timeline/timeline.tsx index 2792b6eb18b00..a3b02b858385e 100644 --- a/x-pack/plugins/infra/public/pages/metrics/inventory_view/components/timeline/timeline.tsx +++ b/x-pack/plugins/infra/public/pages/metrics/inventory_view/components/timeline/timeline.tsx @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import React, { useMemo, useCallback } from 'react'; +import React, { useMemo, useCallback, useEffect } from 'react'; import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n/react'; import moment from 'moment'; @@ -18,7 +18,12 @@ import { TooltipValue, niceTimeFormatter, ElementClickListener, + RectAnnotation, + RectAnnotationDatum, } from '@elastic/charts'; +import { EuiFlexItem } from '@elastic/eui'; +import { EuiFlexGroup } from '@elastic/eui'; +import { EuiIcon } from '@elastic/eui'; import { useUiSetting } from '../../../../../../../../../src/plugins/kibana_react/public'; import { toMetricOpt } from '../../../../../../common/snapshot_metric_i18n'; import { MetricsExplorerAggregation } from '../../../../../../common/http_api'; @@ -35,6 +40,8 @@ import { calculateDomain } from '../../../metrics_explorer/components/helpers/ca import { euiStyled } from '../../../../../../../observability/public'; import { InfraFormatter } from '../../../../../lib/lib'; +import { useMetricsHostsAnomaliesResults } from '../../hooks/use_metrics_hosts_anomalies'; +import { useMetricsK8sAnomaliesResults } from '../../hooks/use_metrics_k8s_anomalies'; interface Props { interval: string; @@ -47,7 +54,8 @@ export const Timeline: React.FC = ({ interval, yAxisFormatter, isVisible const { metric, nodeType, accountId, region } = useWaffleOptionsContext(); const { currentTime, jumpToTime, stopAutoReload } = useWaffleTimeContext(); const { filterQueryAsJson } = useWaffleFiltersContext(); - const { loading, error, timeseries, reload } = useTimeline( + + const { loading, error, startTime, endTime, timeseries, reload } = useTimeline( filterQueryAsJson, [metric], nodeType, @@ -59,6 +67,40 @@ export const Timeline: React.FC = ({ interval, yAxisFormatter, isVisible isVisible ); + const anomalyParams = { + sourceId: 'default', + startTime, + endTime, + defaultSortOptions: { + direction: 'desc' as const, + field: 'anomalyScore' as const, + }, + defaultPaginationOptions: { pageSize: 100 }, + }; + + const { metricsHostsAnomalies, getMetricsHostsAnomalies } = useMetricsHostsAnomaliesResults( + anomalyParams + ); + const { metricsK8sAnomalies, getMetricsK8sAnomalies } = useMetricsK8sAnomaliesResults( + anomalyParams + ); + + const getAnomalies = useMemo(() => { + if (nodeType === 'host') { + return getMetricsHostsAnomalies; + } else if (nodeType === 'pod') { + return getMetricsK8sAnomalies; + } + }, [nodeType, getMetricsK8sAnomalies, getMetricsHostsAnomalies]); + + const anomalies = useMemo(() => { + if (nodeType === 'host') { + return metricsHostsAnomalies; + } else if (nodeType === 'pod') { + return metricsK8sAnomalies; + } + }, [nodeType, metricsHostsAnomalies, metricsK8sAnomalies]); + const metricLabel = toMetricOpt(metric.type)?.textLC; const chartMetric = { @@ -104,6 +146,25 @@ export const Timeline: React.FC = ({ interval, yAxisFormatter, isVisible [jumpToTime, stopAutoReload] ); + const anomalyMetricName = useMemo(() => { + const metricType = metric.type; + if (metricType === 'memory') { + return 'memory_usage'; + } + if (metricType === 'rx') { + return 'network_in'; + } + if (metricType === 'tx') { + return 'network_out'; + } + }, [metric]); + + useEffect(() => { + if (getAnomalies && anomalyMetricName) { + getAnomalies(anomalyMetricName); + } + }, [getAnomalies, anomalyMetricName]); + if (loading) { return ( @@ -130,21 +191,86 @@ export const Timeline: React.FC = ({ interval, yAxisFormatter, isVisible ); } + function generateAnnotationData(results: Array<[number, string[]]>): RectAnnotationDatum[] { + return results.map((anomaly) => { + const [val, influencers] = anomaly; + return { + coordinates: { + x0: val, + x1: moment(val).add(15, 'minutes').valueOf(), + y0: dataDomain?.min, + y1: dataDomain?.max, + }, + details: influencers.join(','), + }; + }); + } + return ( - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + {anomalies && ( + [a.startTime, a.influencers]) + )} + style={{ fill: '#D36086' }} + /> + )} props.theme.eui.paddingSizes.xs}; + padding-left: ${(props) => props.theme.eui.paddingSizes.xs}; width: 100%; height: 100%; `; diff --git a/x-pack/plugins/infra/public/pages/metrics/inventory_view/hooks/use_metrics_hosts_anomalies.ts b/x-pack/plugins/infra/public/pages/metrics/inventory_view/hooks/use_metrics_hosts_anomalies.ts index f755057d0b76d..f33e3ea16b389 100644 --- a/x-pack/plugins/infra/public/pages/metrics/inventory_view/hooks/use_metrics_hosts_anomalies.ts +++ b/x-pack/plugins/infra/public/pages/metrics/inventory_view/hooks/use_metrics_hosts_anomalies.ts @@ -7,6 +7,7 @@ import { useMemo, useState, useCallback, useEffect, useReducer } from 'react'; import { INFA_ML_GET_METRICS_HOSTS_ANOMALIES_PATH, + Metric, Sort, Pagination, PaginationCursor, @@ -168,7 +169,7 @@ export const useMetricsHostsAnomaliesResults = ({ const [getMetricsHostsAnomaliesRequest, getMetricsHostsAnomalies] = useTrackedPromise( { cancelPreviousOn: 'creation', - createPromise: async () => { + createPromise: async (metric: Metric) => { const { timeRange: { start: queryStartTime, end: queryEndTime }, sortOptions, @@ -179,6 +180,7 @@ export const useMetricsHostsAnomaliesResults = ({ sourceId, queryStartTime, queryEndTime, + metric, sortOptions, { ...paginationOptions, @@ -249,10 +251,6 @@ export const useMetricsHostsAnomaliesResults = ({ }); }, [filteredDatasets]); - useEffect(() => { - getMetricsHostsAnomalies(); - }, [getMetricsHostsAnomalies]); // TODO: FIgure out the deps here. - const handleFetchNextPage = useCallback(() => { if (reducerState.lastReceivedCursors) { dispatch({ type: 'fetchNextPage' }); @@ -294,6 +292,7 @@ export const callGetMetricHostsAnomaliesAPI = async ( sourceId: string, startTime: number, endTime: number, + metric: Metric, sort: Sort, pagination: Pagination ) => { @@ -307,6 +306,7 @@ export const callGetMetricHostsAnomaliesAPI = async ( startTime, endTime, }, + metric, sort, pagination, }, diff --git a/x-pack/plugins/infra/public/pages/metrics/inventory_view/hooks/use_metrics_k8s_anomalies.ts b/x-pack/plugins/infra/public/pages/metrics/inventory_view/hooks/use_metrics_k8s_anomalies.ts index 4a7b78e1fdf92..89e70c4c5c4c7 100644 --- a/x-pack/plugins/infra/public/pages/metrics/inventory_view/hooks/use_metrics_k8s_anomalies.ts +++ b/x-pack/plugins/infra/public/pages/metrics/inventory_view/hooks/use_metrics_k8s_anomalies.ts @@ -13,6 +13,7 @@ import { getMetricsK8sAnomaliesSuccessReponsePayloadRT, getMetricsK8sAnomaliesRequestPayloadRT, MetricsK8sAnomaly, + Metric, } from '../../../../../common/http_api/infra_ml'; import { useTrackedPromise } from '../../../../utils/use_tracked_promise'; import { npStart } from '../../../../legacy_singletons'; @@ -168,7 +169,7 @@ export const useMetricsK8sAnomaliesResults = ({ const [getMetricsK8sAnomaliesRequest, getMetricsK8sAnomalies] = useTrackedPromise( { cancelPreviousOn: 'creation', - createPromise: async () => { + createPromise: async (metric: Metric) => { const { timeRange: { start: queryStartTime, end: queryEndTime }, sortOptions, @@ -180,6 +181,7 @@ export const useMetricsK8sAnomaliesResults = ({ sourceId, queryStartTime, queryEndTime, + metric, sortOptions, { ...paginationOptions, @@ -251,10 +253,6 @@ export const useMetricsK8sAnomaliesResults = ({ }); }, [filteredDatasets]); - useEffect(() => { - getMetricsK8sAnomalies(); - }, [getMetricsK8sAnomalies]); - const handleFetchNextPage = useCallback(() => { if (reducerState.lastReceivedCursors) { dispatch({ type: 'fetchNextPage' }); @@ -296,6 +294,7 @@ export const callGetMetricsK8sAnomaliesAPI = async ( sourceId: string, startTime: number, endTime: number, + metric: Metric, sort: Sort, pagination: Pagination, datasets?: string[] @@ -310,6 +309,7 @@ export const callGetMetricsK8sAnomaliesAPI = async ( startTime, endTime, }, + metric, sort, pagination, datasets, diff --git a/x-pack/plugins/infra/public/pages/metrics/inventory_view/hooks/use_timeline.ts b/x-pack/plugins/infra/public/pages/metrics/inventory_view/hooks/use_timeline.ts index acf9011ac7ddd..597c268180819 100644 --- a/x-pack/plugins/infra/public/pages/metrics/inventory_view/hooks/use_timeline.ts +++ b/x-pack/plugins/infra/public/pages/metrics/inventory_view/hooks/use_timeline.ts @@ -81,10 +81,12 @@ export function useTimeline( ]); const { timeLength, intervalInSeconds } = timeLengthResult; + const endTime = currentTime + intervalInSeconds * 1000; + const startTime = currentTime - timeLength * 1000; const timerange: InfraTimerangeInput = { interval: displayInterval ?? '', - to: currentTime + intervalInSeconds * 1000, - from: currentTime - timeLength * 1000, + to: endTime, + from: startTime, ignoreLookback: true, forceInterval: true, }; @@ -127,6 +129,8 @@ export function useTimeline( error: (error && error.message) || null, loading: !interval ? true : loading, timeseries, + startTime, + endTime, reload: makeRequest, }; } diff --git a/x-pack/plugins/infra/server/lib/infra_ml/metrics_hosts_anomalies.ts b/x-pack/plugins/infra/server/lib/infra_ml/metrics_hosts_anomalies.ts index e0afa458aac88..a3a0f91afaab8 100644 --- a/x-pack/plugins/infra/server/lib/infra_ml/metrics_hosts_anomalies.ts +++ b/x-pack/plugins/infra/server/lib/infra_ml/metrics_hosts_anomalies.ts @@ -7,7 +7,7 @@ import { RequestHandlerContext } from 'src/core/server'; import { InfraRequestHandlerContext } from '../../types'; import { TracingSpan, startTracingSpan } from '../../../common/performance_tracing'; -import { fetchMlJob, getLogEntryDatasets } from './common'; +import { fetchMlJob } from './common'; import { getJobId, metricsHostsJobTypes } from '../../../common/infra_ml'; import { Sort, Pagination } from '../../../common/http_api/infra_ml'; import type { MlSystem, MlAnomalyDetectors } from '../../types'; @@ -21,37 +21,43 @@ import { interface MappedAnomalyHit { id: string; anomalyScore: number; - dataset: string; typical: number; actual: number; jobId: string; startTime: number; duration: number; - hostName: string[]; + influencers: string[]; categoryId?: string; } async function getCompatibleAnomaliesJobIds( spaceId: string, sourceId: string, + metric: 'memory_usage' | 'network_in' | 'network_out' | undefined, mlAnomalyDetectors: MlAnomalyDetectors ) { - const metricsHostsJobIds = metricsHostsJobTypes.map((jt) => getJobId(spaceId, sourceId, jt)); + let metricsHostsJobIds = metricsHostsJobTypes; + + if (metric) { + metricsHostsJobIds = metricsHostsJobIds.filter((jt) => jt === `hosts_${metric}`); + } const jobIds: string[] = []; let jobSpans: TracingSpan[] = []; try { await Promise.all( - metricsHostsJobIds.map((id) => { - return (async () => { - const { - timing: { spans }, - } = await fetchMlJob(mlAnomalyDetectors, id); - jobIds.push(id); - jobSpans = [...jobSpans, ...spans]; - })(); - }) + metricsHostsJobIds + .map((jt) => getJobId(spaceId, sourceId, jt)) + .map((id) => { + return (async () => { + const { + timing: { spans }, + } = await fetchMlJob(mlAnomalyDetectors, id); + jobIds.push(id); + jobSpans = [...jobSpans, ...spans]; + })(); + }) ); } catch (e) { if (isMlPrivilegesError(e)) { @@ -71,6 +77,7 @@ export async function getMetricsHostsAnomalies( sourceId: string, startTime: number, endTime: number, + metric: 'memory_usage' | 'network_in' | 'network_out' | undefined, sort: Sort, pagination: Pagination ) { @@ -82,6 +89,7 @@ export async function getMetricsHostsAnomalies( } = await getCompatibleAnomaliesJobIds( context.infra.spaceId, sourceId, + metric, context.infra.mlAnomalyDetectors ); @@ -131,22 +139,20 @@ const parseAnomalyResult = (anomaly: MappedAnomalyHit, jobId: string) => { const { id, anomalyScore, - dataset, typical, actual, duration, - hostName, + influencers, startTime: anomalyStartTime, } = anomaly; return { id, anomalyScore, - dataset, typical, actual, duration, - hostName, + influencers, startTime: anomalyStartTime, type: 'metrics_hosts' as const, jobId, @@ -169,16 +175,6 @@ async function fetchMetricsHostsAnomalies( const finalizeFetchLogEntryAnomaliesSpan = startTracingSpan('fetch metrics hosts anomalies'); - // console.log( - // 'data', - // JSON.stringify( - // await mlSystem.mlAnomalySearch( - // createMetricsHostsAnomaliesQuery(jobIds, startTime, endTime, sort, expandedPagination) - // ), - // null, - // 2 - // ) - // ); const results = decodeOrThrow(metricsHostsAnomaliesResponseRT)( await mlSystem.mlAnomalySearch( createMetricsHostsAnomaliesQuery(jobIds, startTime, endTime, sort, expandedPagination) @@ -216,11 +212,13 @@ async function fetchMetricsHostsAnomalies( record_score: anomalyScore, typical, actual, + influencers, bucket_span: duration, timestamp: anomalyStartTime, by_field_value: categoryId, } = result._source; + const hostInfluencers = influencers.filter((i) => i.influencer_field_name === 'host.name'); return { id: result._id, anomalyScore, @@ -228,7 +226,10 @@ async function fetchMetricsHostsAnomalies( typical: typical[0], actual: actual[0], jobId: job_id, - hostName: result._source['host.name'], + influencers: hostInfluencers.reduce( + (acc: string[], i) => [...acc, ...i.influencer_field_values], + [] + ), startTime: anomalyStartTime, duration: duration * 1000, categoryId, @@ -246,44 +247,3 @@ async function fetchMetricsHostsAnomalies( }, }; } - -// TODO: FIgure out why we need datasets -export async function getMetricsHostsAnomaliesDatasets( - context: { - infra: { - mlSystem: MlSystem; - mlAnomalyDetectors: MlAnomalyDetectors; - spaceId: string; - }; - }, - sourceId: string, - startTime: number, - endTime: number -) { - const { - jobIds, - timing: { spans: jobSpans }, - } = await getCompatibleAnomaliesJobIds( - context.infra.spaceId, - sourceId, - context.infra.mlAnomalyDetectors - ); - - if (jobIds.length === 0) { - throw new InsufficientAnomalyMlJobsConfigured( - 'Log rate or categorisation ML jobs need to be configured to search for anomaly datasets' - ); - } - - const { - data: datasets, - timing: { spans: datasetsSpans }, - } = await getLogEntryDatasets(context.infra.mlSystem, startTime, endTime, jobIds); - - return { - datasets, - timing: { - spans: [...jobSpans, ...datasetsSpans], - }, - }; -} diff --git a/x-pack/plugins/infra/server/lib/infra_ml/metrics_k8s_anomalies.ts b/x-pack/plugins/infra/server/lib/infra_ml/metrics_k8s_anomalies.ts index 29507900e1847..1a9b48ade83ed 100644 --- a/x-pack/plugins/infra/server/lib/infra_ml/metrics_k8s_anomalies.ts +++ b/x-pack/plugins/infra/server/lib/infra_ml/metrics_k8s_anomalies.ts @@ -7,7 +7,7 @@ import { RequestHandlerContext } from 'src/core/server'; import { InfraRequestHandlerContext } from '../../types'; import { TracingSpan, startTracingSpan } from '../../../common/performance_tracing'; -import { fetchMlJob, getLogEntryDatasets } from './common'; +import { fetchMlJob } from './common'; import { getJobId, metricsK8SJobTypes } from '../../../common/infra_ml'; import { Sort, Pagination } from '../../../common/http_api/infra_ml'; import type { MlSystem, MlAnomalyDetectors } from '../../types'; @@ -21,11 +21,11 @@ import { interface MappedAnomalyHit { id: string; anomalyScore: number; - // dataset: string; typical: number; actual: number; jobId: string; startTime: number; + influencers: string[]; duration: number; categoryId?: string; } @@ -33,24 +33,31 @@ interface MappedAnomalyHit { async function getCompatibleAnomaliesJobIds( spaceId: string, sourceId: string, + metric: 'memory_usage' | 'network_in' | 'network_out' | undefined, mlAnomalyDetectors: MlAnomalyDetectors ) { - const metricsK8sJobIds = metricsK8SJobTypes.map((jt) => getJobId(spaceId, sourceId, jt)); + let metricsK8sJobIds = metricsK8SJobTypes; + + if (metric) { + metricsK8sJobIds = metricsK8sJobIds.filter((jt) => jt === `k8s_${metric}`); + } const jobIds: string[] = []; let jobSpans: TracingSpan[] = []; try { await Promise.all( - metricsK8sJobIds.map((id) => { - return (async () => { - const { - timing: { spans }, - } = await fetchMlJob(mlAnomalyDetectors, id); - jobIds.push(id); - jobSpans = [...jobSpans, ...spans]; - })(); - }) + metricsK8sJobIds + .map((jt) => getJobId(spaceId, sourceId, jt)) + .map((id) => { + return (async () => { + const { + timing: { spans }, + } = await fetchMlJob(mlAnomalyDetectors, id); + jobIds.push(id); + jobSpans = [...jobSpans, ...spans]; + })(); + }) ); } catch (e) { if (isMlPrivilegesError(e)) { @@ -70,6 +77,7 @@ export async function getMetricK8sAnomalies( sourceId: string, startTime: number, endTime: number, + metric: 'memory_usage' | 'network_in' | 'network_out' | undefined, sort: Sort, pagination: Pagination ) { @@ -81,6 +89,7 @@ export async function getMetricK8sAnomalies( } = await getCompatibleAnomaliesJobIds( context.infra.spaceId, sourceId, + metric, context.infra.mlAnomalyDetectors ); @@ -126,21 +135,21 @@ const parseAnomalyResult = (anomaly: MappedAnomalyHit, jobId: string) => { const { id, anomalyScore, - // dataset, typical, actual, duration, + influencers, startTime: anomalyStartTime, } = anomaly; return { id, anomalyScore, - // dataset, typical, actual, duration, startTime: anomalyStartTime, + influencers, type: 'metrics_k8s' as const, jobId, }; @@ -199,19 +208,25 @@ async function fetchMetricK8sAnomalies( record_score: anomalyScore, typical, actual, - // partition_field_value: dataset, bucket_span: duration, timestamp: anomalyStartTime, by_field_value: categoryId, + influencers, } = result._source; + const podInfluencers = influencers.filter( + (i) => i.influencer_field_name === 'kubernetes.pod.uid' + ); return { id: result._id, anomalyScore, - // dataset, typical: typical[0], actual: actual[0], jobId: job_id, + influencers: podInfluencers.reduce( + (acc: string[], i) => [...acc, ...i.influencer_field_values], + [] + ), startTime: anomalyStartTime, duration: duration * 1000, categoryId, @@ -229,44 +244,3 @@ async function fetchMetricK8sAnomalies( }, }; } - -// TODO: FIgure out why we need datasets -export async function getMetricK8sAnomaliesDatasets( - context: { - infra: { - mlSystem: MlSystem; - mlAnomalyDetectors: MlAnomalyDetectors; - spaceId: string; - }; - }, - sourceId: string, - startTime: number, - endTime: number -) { - const { - jobIds, - timing: { spans: jobSpans }, - } = await getCompatibleAnomaliesJobIds( - context.infra.spaceId, - sourceId, - context.infra.mlAnomalyDetectors - ); - - if (jobIds.length === 0) { - throw new InsufficientAnomalyMlJobsConfigured( - 'Log rate or categorisation ML jobs need to be configured to search for anomaly datasets' - ); - } - - const { - data: datasets, - timing: { spans: datasetsSpans }, - } = await getLogEntryDatasets(context.infra.mlSystem, startTime, endTime, jobIds); - - return { - datasets, - timing: { - spans: [...jobSpans, ...datasetsSpans], - }, - }; -} diff --git a/x-pack/plugins/infra/server/lib/infra_ml/queries/common.ts b/x-pack/plugins/infra/server/lib/infra_ml/queries/common.ts index 63e39ef022392..eb08b6d692336 100644 --- a/x-pack/plugins/infra/server/lib/infra_ml/queries/common.ts +++ b/x-pack/plugins/infra/server/lib/infra_ml/queries/common.ts @@ -40,6 +40,16 @@ export const createTimeRangeFilters = (startTime: number, endTime: number) => [ }, ]; +export const createAnomalyScoreFilter = (minScore: number) => [ + { + range: { + record_score: { + gte: minScore, + }, + }, + }, +]; + export const createResultTypeFilters = (resultTypes: Array<'model_plot' | 'record'>) => [ { terms: { diff --git a/x-pack/plugins/infra/server/lib/infra_ml/queries/metrics_hosts_anomalies.ts b/x-pack/plugins/infra/server/lib/infra_ml/queries/metrics_hosts_anomalies.ts index b61119b60bc18..bbdc77af1fbe6 100644 --- a/x-pack/plugins/infra/server/lib/infra_ml/queries/metrics_hosts_anomalies.ts +++ b/x-pack/plugins/infra/server/lib/infra_ml/queries/metrics_hosts_anomalies.ts @@ -11,6 +11,7 @@ import { createTimeRangeFilters, createResultTypeFilters, defaultRequestParameters, + createAnomalyScoreFilter, } from './common'; import { Sort, Pagination } from '../../../../common/http_api/infra_ml'; @@ -35,6 +36,7 @@ export const createMetricsHostsAnomaliesQuery = ( const filters = [ ...createJobIdsFilters(jobIds), + ...createAnomalyScoreFilter(50), ...createTimeRangeFilters(startTime, endTime), ...createResultTypeFilters(['record']), ]; @@ -86,6 +88,12 @@ export const metricsHostsAnomalyHitRT = rt.type({ record_score: rt.number, typical: rt.array(rt.number), actual: rt.array(rt.number), + influencers: rt.array( + rt.type({ + influencer_field_name: rt.string, + influencer_field_values: rt.array(rt.string), + }) + ), 'host.name': rt.array(rt.string), bucket_span: rt.number, timestamp: rt.number, diff --git a/x-pack/plugins/infra/server/lib/infra_ml/queries/metrics_k8s_anomalies.ts b/x-pack/plugins/infra/server/lib/infra_ml/queries/metrics_k8s_anomalies.ts index 84ed8b064c5ca..79bfdc91dc5a4 100644 --- a/x-pack/plugins/infra/server/lib/infra_ml/queries/metrics_k8s_anomalies.ts +++ b/x-pack/plugins/infra/server/lib/infra_ml/queries/metrics_k8s_anomalies.ts @@ -11,6 +11,7 @@ import { createTimeRangeFilters, createResultTypeFilters, defaultRequestParameters, + createAnomalyScoreFilter, } from './common'; import { Sort, Pagination } from '../../../../common/http_api/infra_ml'; @@ -35,6 +36,7 @@ export const createMetricsK8sAnomaliesQuery = ( const filters = [ ...createJobIdsFilters(jobIds), + ...createAnomalyScoreFilter(50), ...createTimeRangeFilters(startTime, endTime), ...createResultTypeFilters(['record']), ]; @@ -48,6 +50,8 @@ export const createMetricsK8sAnomaliesQuery = ( 'timestamp', 'bucket_span', 'by_field_value', + 'influencers.influencer_field_name', + 'influencers.influencer_field_values', ]; const { querySortDirection, queryCursor } = parsePaginationCursor(sort, pagination); @@ -83,7 +87,12 @@ export const metricsK8sAnomalyHitRT = rt.type({ record_score: rt.number, typical: rt.array(rt.number), actual: rt.array(rt.number), - // partition_field_value: rt.string, + influencers: rt.array( + rt.type({ + influencer_field_name: rt.string, + influencer_field_values: rt.array(rt.string), + }) + ), bucket_span: rt.number, timestamp: rt.number, }), diff --git a/x-pack/plugins/infra/server/routes/infra_ml/results/metrics_hosts_anomalies.ts b/x-pack/plugins/infra/server/routes/infra_ml/results/metrics_hosts_anomalies.ts index 29122ae159cdc..9dc309c605206 100644 --- a/x-pack/plugins/infra/server/routes/infra_ml/results/metrics_hosts_anomalies.ts +++ b/x-pack/plugins/infra/server/routes/infra_ml/results/metrics_hosts_anomalies.ts @@ -36,6 +36,7 @@ export const initGetHostsAnomaliesRoute = ({ framework }: InfraBackendLibs) => { timeRange: { startTime, endTime }, sort: sortParam, pagination: paginationParam, + metric, }, } = request.body; @@ -54,12 +55,11 @@ export const initGetHostsAnomaliesRoute = ({ framework }: InfraBackendLibs) => { sourceId, startTime, endTime, + metric, sort, pagination ); - // console.log('---- anomalies', anomalies); - return response.ok({ body: getMetricsHostsAnomaliesSuccessReponsePayloadRT.encode({ data: { diff --git a/x-pack/plugins/infra/server/routes/infra_ml/results/metrics_k8s_anomalies.ts b/x-pack/plugins/infra/server/routes/infra_ml/results/metrics_k8s_anomalies.ts index 5260c55836c59..1618018b85fcf 100644 --- a/x-pack/plugins/infra/server/routes/infra_ml/results/metrics_k8s_anomalies.ts +++ b/x-pack/plugins/infra/server/routes/infra_ml/results/metrics_k8s_anomalies.ts @@ -35,6 +35,7 @@ export const initGetK8sAnomaliesRoute = ({ framework }: InfraBackendLibs) => { timeRange: { startTime, endTime }, sort: sortParam, pagination: paginationParam, + metric, }, } = request.body; @@ -53,6 +54,7 @@ export const initGetK8sAnomaliesRoute = ({ framework }: InfraBackendLibs) => { sourceId, startTime, endTime, + metric, sort, pagination );