From 4d8e3ddaa97c47012fbfea65435ab2dacc912683 Mon Sep 17 00:00:00 2001 From: Melissa Alvarez Date: Fri, 20 Nov 2020 10:39:30 -0500 Subject: [PATCH] [ML] Data frame analytics: Adds functionality to map view (#83710) * get all jobs from index node * create map from modelId and enable url share * highlight source node * add map endpoint to api doc * use variables in css.fix types.ensure map tab is shown * fix translations --- .../common/constants/data_frame_analytics.ts | 2 +- .../ml/common/types/ml_url_generator.ts | 3 + .../analytics_navigation_bar.tsx | 11 +- .../models_management/models_list.tsx | 2 +- .../pages/analytics_management/page.tsx | 11 +- .../pages/job_map/components/_legend.scss | 25 +- .../pages/job_map/components/controls.tsx | 13 +- .../job_map/components/cytoscape_options.tsx | 3 +- .../pages/job_map/components/legend.tsx | 33 +- .../pages/job_map/job_map.tsx | 71 ++- .../ml_api_service/data_frame_analytics.ts | 8 +- .../data_frame_analytics_urls_generator.ts | 3 +- .../data_frame_analytics/analytics_manager.ts | 570 +++++++++++------- .../models/data_frame_analytics/types.ts | 53 +- x-pack/plugins/ml/server/routes/apidoc.json | 1 + .../ml/server/routes/data_frame_analytics.ts | 34 +- .../routes/schemas/data_analytics_schema.ts | 2 +- 17 files changed, 564 insertions(+), 281 deletions(-) diff --git a/x-pack/plugins/ml/common/constants/data_frame_analytics.ts b/x-pack/plugins/ml/common/constants/data_frame_analytics.ts index 958d5ae25018..7eef86869b9e 100644 --- a/x-pack/plugins/ml/common/constants/data_frame_analytics.ts +++ b/x-pack/plugins/ml/common/constants/data_frame_analytics.ts @@ -16,7 +16,7 @@ export const JOB_MAP_NODE_TYPES = { ANALYTICS: 'analytics', TRANSFORM: 'transform', INDEX: 'index', - INFERENCE_MODEL: 'inferenceModel', + TRAINED_MODEL: 'trainedModel', } as const; export type JobMapNodeTypes = typeof JOB_MAP_NODE_TYPES[keyof typeof JOB_MAP_NODE_TYPES]; diff --git a/x-pack/plugins/ml/common/types/ml_url_generator.ts b/x-pack/plugins/ml/common/types/ml_url_generator.ts index 9a3d8fc4a4f0..b5a78ee746ef 100644 --- a/x-pack/plugins/ml/common/types/ml_url_generator.ts +++ b/x-pack/plugins/ml/common/types/ml_url_generator.ts @@ -156,6 +156,7 @@ export type TimeSeriesExplorerUrlState = MLPageState< export interface DataFrameAnalyticsQueryState { jobId?: JobId | JobId[]; + modelId?: string; groupIds?: string[]; globalState?: MlCommonGlobalState; } @@ -170,6 +171,7 @@ export interface DataFrameAnalyticsExplorationQueryState { jobId: JobId; analysisType: DataFrameAnalysisConfigType; defaultIsTraining?: boolean; + modelId?: string; }; } @@ -180,6 +182,7 @@ export type DataFrameAnalyticsExplorationUrlState = MLPageState< analysisType: DataFrameAnalysisConfigType; globalState?: MlCommonGlobalState; defaultIsTraining?: boolean; + modelId?: string; } >; diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/analytics_navigation_bar/analytics_navigation_bar.tsx b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/analytics_navigation_bar/analytics_navigation_bar.tsx index a5d3555fcc27..bf90ce58fb85 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/analytics_navigation_bar/analytics_navigation_bar.tsx +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/analytics_navigation_bar/analytics_navigation_bar.tsx @@ -15,10 +15,11 @@ interface Tab { path: string; } -export const AnalyticsNavigationBar: FC<{ selectedTabId?: string; jobId?: string }> = ({ - jobId, - selectedTabId, -}) => { +export const AnalyticsNavigationBar: FC<{ + selectedTabId?: string; + jobId?: string; + modelId?: string; +}> = ({ jobId, modelId, selectedTabId }) => { const navigateToPath = useNavigateToPath(); const tabs = useMemo(() => { @@ -38,7 +39,7 @@ export const AnalyticsNavigationBar: FC<{ selectedTabId?: string; jobId?: string path: '/data_frame_analytics/models', }, ]; - if (jobId !== undefined) { + if (jobId !== undefined || modelId !== undefined) { navTabs.push({ id: 'map', name: i18n.translate('xpack.ml.dataframe.mapTabLabel', { diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/models_management/models_list.tsx b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/models_management/models_list.tsx index 2d74d08c4550..cde29d357b1c 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/models_management/models_list.tsx +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/models_management/models_list.tsx @@ -342,7 +342,7 @@ export const ModelsList: FC = () => { onClick: async (item) => { const path = await mlUrlGenerator.createUrl({ page: ML_PAGES.DATA_FRAME_ANALYTICS_MAP, - pageState: { jobId: item.metadata?.analytics_config.id }, + pageState: { modelId: item.model_id }, }); await navigateToPath(path, false); diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/page.tsx b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/page.tsx index 5a17b91818a1..38b7088690e1 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/page.tsx +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/page.tsx @@ -59,6 +59,7 @@ export const Page: FC = () => { const location = useLocation(); const selectedTabId = useMemo(() => location.pathname.split('/').pop(), [location]); const mapJobId = globalState?.ml?.jobId; + const mapModelId = globalState?.ml?.modelId; return ( @@ -106,8 +107,14 @@ export const Page: FC = () => { - - {selectedTabId === 'map' && mapJobId && } + + {selectedTabId === 'map' && (mapJobId || mapModelId) && ( + + )} {selectedTabId === 'data_frame_analytics' && ( = ({ analyticsId, details, getNodeData }) => { +export const Controls: FC = ({ analyticsId, modelId, details, getNodeData }) => { const [showFlyout, setShowFlyout] = useState(false); const [selectedNode, setSelectedNode] = useState(); @@ -98,10 +99,12 @@ export const Controls: FC = ({ analyticsId, details, getNodeData }) => { } const nodeDataButton = - analyticsId !== nodeLabel && nodeType === JOB_MAP_NODE_TYPES.ANALYTICS ? ( + analyticsId !== nodeLabel && + modelId !== nodeLabel && + (nodeType === JOB_MAP_NODE_TYPES.ANALYTICS || nodeType === JOB_MAP_NODE_TYPES.INDEX) ? ( { - getNodeData(nodeLabel); + getNodeData({ id: nodeLabel, type: nodeType }); setShowFlyout(false); }} iconType="branch" diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/job_map/components/cytoscape_options.tsx b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/job_map/components/cytoscape_options.tsx index 85d10aa89741..18be614afb5c 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/job_map/components/cytoscape_options.tsx +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/job_map/components/cytoscape_options.tsx @@ -80,7 +80,8 @@ export const cytoscapeOptions: cytoscape.CytoscapeOptions = { { selector: 'node', style: { - 'background-color': theme.euiColorGhost, + 'background-color': (el: cytoscape.NodeSingular) => + el.data('isRoot') ? theme.euiColorLightShade : theme.euiColorGhost, 'background-height': '60%', 'background-width': '60%', 'border-color': (el: cytoscape.NodeSingular) => borderColorForNode(el), diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/job_map/components/legend.tsx b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/job_map/components/legend.tsx index c29b6aca804d..04e415eca169 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/job_map/components/legend.tsx +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/job_map/components/legend.tsx @@ -6,6 +6,7 @@ import React, { FC } from 'react'; import { EuiFlexGroup, EuiFlexItem, EuiText } from '@elastic/eui'; +import { FormattedMessage } from '@kbn/i18n/react'; import { JOB_MAP_NODE_TYPES } from '../../../../../../common/constants/data_frame_analytics'; export const JobMapLegend: FC = () => ( @@ -17,7 +18,10 @@ export const JobMapLegend: FC = () => ( - {JOB_MAP_NODE_TYPES.INDEX} + @@ -41,7 +45,10 @@ export const JobMapLegend: FC = () => ( - {JOB_MAP_NODE_TYPES.ANALYTICS} + @@ -49,11 +56,29 @@ export const JobMapLegend: FC = () => ( - + - {'inference model'} + + + + + + + + + + + + + diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/job_map/job_map.tsx b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/job_map/job_map.tsx index 53d47937409d..6395d491d5e6 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/job_map/job_map.tsx +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/job_map/job_map.tsx @@ -15,6 +15,7 @@ import { Cytoscape, Controls, JobMapLegend } from './components'; import { ml } from '../../../services/ml_api_service'; import { useMlKibana } from '../../../contexts/kibana'; import { useRefDimensions } from './components/use_ref_dimensions'; +import { JOB_MAP_NODE_TYPES } from '../../../../../common/constants/data_frame_analytics'; const cytoscapeDivStyle = { background: `linear-gradient( @@ -36,22 +37,36 @@ ${theme.euiColorLightShade}`, marginTop: 0, }; -export const JobMapTitle: React.FC<{ analyticsId: string }> = ({ analyticsId }) => ( +export const JobMapTitle: React.FC<{ analyticsId?: string; modelId?: string }> = ({ + analyticsId, + modelId, +}) => ( - {i18n.translate('xpack.ml.dataframe.analyticsMap.analyticsIdTitle', { - defaultMessage: 'Map for analytics ID {analyticsId}', - values: { analyticsId }, - })} + {analyticsId + ? i18n.translate('xpack.ml.dataframe.analyticsMap.analyticsIdTitle', { + defaultMessage: 'Map for analytics ID {analyticsId}', + values: { analyticsId }, + }) + : i18n.translate('xpack.ml.dataframe.analyticsMap.modelIdTitle', { + defaultMessage: 'Map for trained model ID {modelId}', + values: { modelId }, + })} ); +interface GetDataObjectParameter { + id: string; + type: string; +} + interface Props { - analyticsId: string; + analyticsId?: string; + modelId?: string; } -export const JobMap: FC = ({ analyticsId }) => { +export const JobMap: FC = ({ analyticsId, modelId }) => { const [elements, setElements] = useState([]); const [nodeDetails, setNodeDetails] = useState({}); const [error, setError] = useState(undefined); @@ -60,14 +75,33 @@ export const JobMap: FC = ({ analyticsId }) => { services: { notifications }, } = useMlKibana(); - const getData = async (id?: string) => { + const getDataWrapper = async (params?: GetDataObjectParameter) => { + const { id, type } = params ?? {}; const treatAsRoot = id !== undefined; - const idToUse = treatAsRoot ? id : analyticsId; - // Pass in treatAsRoot flag - endpoint will take job destIndex to grab jobs created from it + let idToUse: string; + + if (id !== undefined) { + idToUse = id; + } else if (modelId !== undefined) { + idToUse = modelId; + } else { + idToUse = analyticsId as string; + } + + await getData( + idToUse, + treatAsRoot, + modelId !== undefined && treatAsRoot === false ? JOB_MAP_NODE_TYPES.TRAINED_MODEL : type + ); + }; + + const getData = async (idToUse: string, treatAsRoot: boolean, type?: string) => { + // Pass in treatAsRoot flag - endpoint will take job or index to grab jobs created from it // TODO: update analyticsMap return type here const analyticsMap: any = await ml.dataFrameAnalytics.getDataFrameAnalyticsMap( idToUse, - treatAsRoot + treatAsRoot, + type ); const { elements: nodeElements, details, error: fetchError } = analyticsMap; @@ -86,7 +120,7 @@ export const JobMap: FC = ({ analyticsId }) => { } if (nodeElements && nodeElements.length > 0) { - if (id === undefined) { + if (treatAsRoot === false) { setElements(nodeElements); setNodeDetails(details); } else { @@ -98,8 +132,8 @@ export const JobMap: FC = ({ analyticsId }) => { }; useEffect(() => { - getData(); - }, [analyticsId]); + getDataWrapper(); + }, [analyticsId, modelId]); if (error !== undefined) { notifications.toasts.addDanger( @@ -119,14 +153,19 @@ export const JobMap: FC = ({ analyticsId }) => {
- + - +
diff --git a/x-pack/plugins/ml/public/application/services/ml_api_service/data_frame_analytics.ts b/x-pack/plugins/ml/public/application/services/ml_api_service/data_frame_analytics.ts index 21556a4702b4..8e541443c34a 100644 --- a/x-pack/plugins/ml/public/application/services/ml_api_service/data_frame_analytics.ts +++ b/x-pack/plugins/ml/public/application/services/ml_api_service/data_frame_analytics.ts @@ -83,12 +83,12 @@ export const dataFrameAnalytics = { body, }); }, - getDataFrameAnalyticsMap(analyticsId?: string, treatAsRoot?: boolean) { - const analyticsIdString = analyticsId !== undefined ? `/${analyticsId}` : ''; + getDataFrameAnalyticsMap(id: string, treatAsRoot: boolean, type?: string) { + const idString = id !== undefined ? `/${id}` : ''; return http({ - path: `${basePath()}/data_frame/analytics/map${analyticsIdString}`, + path: `${basePath()}/data_frame/analytics/map${idString}`, method: 'GET', - query: { treatAsRoot }, + query: { treatAsRoot, type }, }); }, evaluateDataFrameAnalytics(evaluateConfig: any) { diff --git a/x-pack/plugins/ml/public/ml_url_generator/data_frame_analytics_urls_generator.ts b/x-pack/plugins/ml/public/ml_url_generator/data_frame_analytics_urls_generator.ts index dc9c3bd86cc6..10764022a3ce 100644 --- a/x-pack/plugins/ml/public/ml_url_generator/data_frame_analytics_urls_generator.ts +++ b/x-pack/plugins/ml/public/ml_url_generator/data_frame_analytics_urls_generator.ts @@ -104,11 +104,12 @@ export function createDataFrameAnalyticsMapUrl( let url = `${appBasePath}/${ML_PAGES.DATA_FRAME_ANALYTICS_MAP}`; if (mlUrlGeneratorState) { - const { jobId, analysisType, defaultIsTraining, globalState } = mlUrlGeneratorState; + const { jobId, modelId, analysisType, defaultIsTraining, globalState } = mlUrlGeneratorState; const queryState: DataFrameAnalyticsExplorationQueryState = { ml: { jobId, + modelId, analysisType, defaultIsTraining, }, diff --git a/x-pack/plugins/ml/server/models/data_frame_analytics/analytics_manager.ts b/x-pack/plugins/ml/server/models/data_frame_analytics/analytics_manager.ts index f1f0b352ca92..769ec09a6b91 100644 --- a/x-pack/plugins/ml/server/models/data_frame_analytics/analytics_manager.ts +++ b/x-pack/plugins/ml/server/models/data_frame_analytics/analytics_manager.ts @@ -10,12 +10,17 @@ import { JOB_MAP_NODE_TYPES, JobMapNodeTypes, } from '../../../common/constants/data_frame_analytics'; +import { TrainedModelConfigResponse } from '../../../common/types/trained_models'; import { INDEX_META_DATA_CREATED_BY } from '../../../common/constants/file_datavisualizer'; import { getAnalysisType } from '../../../common/util/analytics_utils'; import { AnalyticsMapEdgeElement, AnalyticsMapReturnType, AnalyticsMapNodeElement, + ExtendAnalyticsMapArgs, + GetAnalyticsMapArgs, + InitialElementsReturnType, + isCompleteInitialReturnType, isAnalyticsMapEdgeElement, isAnalyticsMapNodeElement, isIndexPatternLinkReturnType, @@ -29,7 +34,7 @@ import type { MlClient } from '../../lib/ml_client'; export class AnalyticsManager { private _client: IScopedClusterClient['asInternalUser']; private _mlClient: MlClient; - public _inferenceModels: any; // TODO: update types + public _inferenceModels: TrainedModelConfigResponse[]; constructor(mlClient: MlClient, client: IScopedClusterClient['asInternalUser']) { this._client = client; @@ -37,11 +42,11 @@ export class AnalyticsManager { this._inferenceModels = []; } - public set inferenceModels(models: any) { + public set inferenceModels(models) { this._inferenceModels = models; } - public get inferenceModels(): any { + public get inferenceModels() { return this._inferenceModels; } @@ -56,16 +61,20 @@ export class AnalyticsManager { } } - private isDuplicateElement(analyticsId: string, elements: any[]): boolean { + private isDuplicateElement(analyticsId: string, elements: MapElements[]): boolean { let isDuplicate = false; - elements.forEach((elem: any) => { - if (elem.data.label === analyticsId && elem.data.type === JOB_MAP_NODE_TYPES.ANALYTICS) { + elements.forEach((elem) => { + if ( + isAnalyticsMapNodeElement(elem) && + elem.data.label === analyticsId && + elem.data.type === JOB_MAP_NODE_TYPES.ANALYTICS + ) { isDuplicate = true; } }); return isDuplicate; } - // @ts-ignore // TODO: is this needed? + private async getAnalyticsModelData(modelId: string) { const resp = await this._mlClient.getTrainedModels({ model_id: modelId, @@ -80,11 +89,17 @@ export class AnalyticsManager { return models; } - private async getAnalyticsJobData(analyticsId: string) { - const resp = await this._mlClient.getDataFrameAnalytics({ - id: analyticsId, - }); - const jobData = resp?.body?.data_frame_analytics[0]; + private async getAnalyticsData(analyticsId?: string) { + const options = analyticsId + ? { + id: analyticsId, + } + : undefined; + const resp = await this._mlClient.getDataFrameAnalytics(options); + const jobData = analyticsId + ? resp?.body?.data_frame_analytics[0] + : resp?.body?.data_frame_analytics; + return jobData; } @@ -130,7 +145,7 @@ export class AnalyticsManager { return { isWildcardIndexPattern, isIndexPattern: true, indexData, meta }; } else if (type.includes(JOB_MAP_NODE_TYPES.ANALYTICS)) { // fetch job associated with this index - const jobData = await this.getAnalyticsJobData(id); + const jobData = await this.getAnalyticsData(id); return { jobData, isJob: true }; } else if (type === JOB_MAP_NODE_TYPES.TRANSFORM) { // fetch transform so we can get original index pattern @@ -155,12 +170,12 @@ export class AnalyticsManager { let edgeElement; if (analyticsModel !== undefined) { - const modelId = `${analyticsModel.model_id}-${JOB_MAP_NODE_TYPES.INFERENCE_MODEL}`; + const modelId = `${analyticsModel.model_id}-${JOB_MAP_NODE_TYPES.TRAINED_MODEL}`; modelElement = { data: { id: modelId, label: analyticsModel.model_id, - type: JOB_MAP_NODE_TYPES.INFERENCE_MODEL, + type: JOB_MAP_NODE_TYPES.TRAINED_MODEL, }, }; // Create edge for job and corresponding model @@ -201,29 +216,41 @@ export class AnalyticsManager { } /** - * Works backward from jobId to return related jobs from source indices - * @param jobId + * Prepares the initial elements for incoming modelId + * @param modelId */ - async getAnalyticsMap(analyticsId: string): Promise { - const result: any = { elements: [], details: {}, error: null }; - const modelElements: MapElements[] = []; - const indexPatternElements: MapElements[] = []; + async getInitialElementsModelRoot(modelId: string): Promise { + const resultElements = []; + const modelElements = []; + const details: any = {}; + // fetch model data and create model elements + let data = await this.getAnalyticsModelData(modelId); + const modelNodeId = `${data.model_id}-${JOB_MAP_NODE_TYPES.TRAINED_MODEL}`; + const sourceJobId = data?.metadata?.analytics_config?.id; + let nextLinkId: string | undefined; + let nextType: JobMapNodeTypes | undefined; + let previousNodeId: string | undefined; + + modelElements.push({ + data: { + id: modelNodeId, + label: data.model_id, + type: JOB_MAP_NODE_TYPES.TRAINED_MODEL, + isRoot: true, + }, + }); - try { - await this.setInferenceModels(); - // Create first node for incoming analyticsId - let data = await this.getAnalyticsJobData(analyticsId); - let nextLinkId = data?.source?.index[0]; - let nextType: JobMapNodeTypes = JOB_MAP_NODE_TYPES.INDEX; - let complete = false; - let link: NextLinkReturnType; - let count = 0; - let rootTransform; - let rootIndexPattern; - - let previousNodeId = `${data.id}-${JOB_MAP_NODE_TYPES.ANALYTICS}`; + details[modelNodeId] = data; + // fetch source job data and create elements + if (sourceJobId !== undefined) { + data = await this.getAnalyticsData(sourceJobId); - result.elements.push({ + nextLinkId = data?.source?.index[0]; + nextType = JOB_MAP_NODE_TYPES.INDEX; + + previousNodeId = `${data.id}-${JOB_MAP_NODE_TYPES.ANALYTICS}`; + + resultElements.push({ data: { id: previousNodeId, label: data.id, @@ -231,167 +258,178 @@ export class AnalyticsManager { analysisType: getAnalysisType(data?.analysis), }, }); - result.details[previousNodeId] = data; + // Create edge between job and model + modelElements.push({ + data: { + id: `${previousNodeId}~${modelNodeId}`, + source: previousNodeId, + target: modelNodeId, + }, + }); - let { modelElement, modelDetails, edgeElement } = this.getAnalyticsModelElements(analyticsId); - if (isAnalyticsMapNodeElement(modelElement)) { - modelElements.push(modelElement); - result.details[modelElement.data.id] = modelDetails; - } - if (isAnalyticsMapEdgeElement(edgeElement)) { - modelElements.push(edgeElement); - } - // Add a safeguard against infinite loops. - while (complete === false) { - count++; - if (count >= 100) { - break; - } + details[previousNodeId] = data; + } - try { - link = await this.getNextLink({ - id: nextLinkId, - type: nextType, - }); - } catch (error) { - result.error = error.message || 'Something went wrong'; - break; - } - // If it's index pattern, check meta data to see what to fetch next - if (isIndexPatternLinkReturnType(link) && link.isIndexPattern === true) { - if (link.isWildcardIndexPattern === true) { - // Create index nodes for each of the indices included in the index pattern then break - const { details, elements } = this.getIndexPatternElements( - link.indexData, - previousNodeId - ); - - indexPatternElements.push(...elements); - result.details = { ...result.details, ...details }; - complete = true; - } else { - const nodeId = `${nextLinkId}-${JOB_MAP_NODE_TYPES.INDEX}`; - result.elements.unshift({ - data: { id: nodeId, label: nextLinkId, type: JOB_MAP_NODE_TYPES.INDEX }, - }); - result.details[nodeId] = link.indexData; - } + return { data, details, resultElements, modelElements, nextLinkId, nextType, previousNodeId }; + } - // Check meta data - if ( - link.isWildcardIndexPattern === false && - (link.meta === undefined || link.meta?.created_by === INDEX_META_DATA_CREATED_BY) - ) { - rootIndexPattern = nextLinkId; - complete = true; - break; - } + /** + * Prepares the initial elements for incoming jobId + * @param jobId + */ + async getInitialElementsJobRoot(jobId: string): Promise { + const resultElements = []; + const modelElements = []; + const details: any = {}; + const data = await this.getAnalyticsData(jobId); + const nextLinkId = data?.source?.index[0]; + const nextType: JobMapNodeTypes = JOB_MAP_NODE_TYPES.INDEX; + + const previousNodeId = `${data.id}-${JOB_MAP_NODE_TYPES.ANALYTICS}`; + + resultElements.push({ + data: { + id: previousNodeId, + label: data.id, + type: JOB_MAP_NODE_TYPES.ANALYTICS, + analysisType: getAnalysisType(data?.analysis), + isRoot: true, + }, + }); - if (link.meta?.created_by === 'data-frame-analytics') { - nextLinkId = link.meta.analytics; - nextType = JOB_MAP_NODE_TYPES.ANALYTICS; - } + details[previousNodeId] = data; - if (link.meta?.created_by === JOB_MAP_NODE_TYPES.TRANSFORM) { - nextLinkId = link.meta._transform?.transform; - nextType = JOB_MAP_NODE_TYPES.TRANSFORM; - } - } else if (isJobDataLinkReturnType(link) && link.isJob === true) { - data = link.jobData; - const nodeId = `${data.id}-${JOB_MAP_NODE_TYPES.ANALYTICS}`; - previousNodeId = nodeId; + const { modelElement, modelDetails, edgeElement } = this.getAnalyticsModelElements(jobId); + if (isAnalyticsMapNodeElement(modelElement)) { + modelElements.push(modelElement); + details[modelElement.data.id] = modelDetails; + } + if (isAnalyticsMapEdgeElement(edgeElement)) { + modelElements.push(edgeElement); + } - result.elements.unshift({ - data: { - id: nodeId, - label: data.id, - type: JOB_MAP_NODE_TYPES.ANALYTICS, - analysisType: getAnalysisType(data?.analysis), - }, - }); - result.details[nodeId] = data; - nextLinkId = data?.source?.index[0]; - nextType = JOB_MAP_NODE_TYPES.INDEX; - - // Get inference model for analytics job and create model node - ({ modelElement, modelDetails, edgeElement } = this.getAnalyticsModelElements(data.id)); - if (isAnalyticsMapNodeElement(modelElement)) { - modelElements.push(modelElement); - result.details[modelElement.data.id] = modelDetails; - } - if (isAnalyticsMapEdgeElement(edgeElement)) { - modelElements.push(edgeElement); + return { data, details, resultElements, modelElements, nextLinkId, nextType, previousNodeId }; + } + + /** + * Works backward from jobId or modelId to return related jobs, indices, models, and transforms + * @param jobId (optional) + * @param modelId (optional) + */ + async getAnalyticsMap({ + analyticsId, + modelId, + }: GetAnalyticsMapArgs): Promise { + const result: AnalyticsMapReturnType = { elements: [], details: {}, error: null }; + const modelElements: MapElements[] = []; + const indexPatternElements: MapElements[] = []; + + try { + await this.setInferenceModels(); + // Create first node for incoming analyticsId or modelId + let initialData: InitialElementsReturnType = {} as InitialElementsReturnType; + if (analyticsId !== undefined) { + initialData = await this.getInitialElementsJobRoot(analyticsId); + } else if (modelId !== undefined) { + initialData = await this.getInitialElementsModelRoot(modelId); + } + + const { + resultElements, + details: initialDetails, + modelElements: initialModelElements, + } = initialData; + + result.elements.push(...resultElements); + result.details = initialDetails; + modelElements.push(...initialModelElements); + + if (isCompleteInitialReturnType(initialData)) { + let { data, nextLinkId, nextType, previousNodeId } = initialData; + + let complete = false; + let link: NextLinkReturnType; + let count = 0; + let rootTransform; + let rootIndexPattern; + let modelElement; + let modelDetails; + let edgeElement; + + // Add a safeguard against infinite loops. + while (complete === false) { + count++; + if (count >= 100) { + break; } - } else if (isTransformLinkReturnType(link) && link.isTransform === true) { - data = link.transformData; - const nodeId = `${data.id}-${JOB_MAP_NODE_TYPES.TRANSFORM}`; - previousNodeId = nodeId; - rootTransform = data.dest.index; + try { + link = await this.getNextLink({ + id: nextLinkId, + type: nextType, + }); + } catch (error) { + result.error = error.message || 'Something went wrong'; + break; + } + // If it's index pattern, check meta data to see what to fetch next + if (isIndexPatternLinkReturnType(link) && link.isIndexPattern === true) { + if (link.isWildcardIndexPattern === true) { + // Create index nodes for each of the indices included in the index pattern then break + const { details, elements } = this.getIndexPatternElements( + link.indexData, + previousNodeId + ); + + indexPatternElements.push(...elements); + result.details = { ...result.details, ...details }; + complete = true; + } else { + const nodeId = `${nextLinkId}-${JOB_MAP_NODE_TYPES.INDEX}`; + result.elements.unshift({ + data: { id: nodeId, label: nextLinkId, type: JOB_MAP_NODE_TYPES.INDEX }, + }); + result.details[nodeId] = link.indexData; + } - result.elements.unshift({ - data: { id: nodeId, label: data.id, type: JOB_MAP_NODE_TYPES.TRANSFORM }, - }); - result.details[nodeId] = data; - nextLinkId = data?.source?.index[0]; - nextType = JOB_MAP_NODE_TYPES.INDEX; - } - } // end while + // Check meta data + if ( + link.isWildcardIndexPattern === false && + (link.meta === undefined || link.meta?.created_by === INDEX_META_DATA_CREATED_BY) + ) { + rootIndexPattern = nextLinkId; + complete = true; + break; + } - // create edge elements - const elemLength = result.elements.length - 1; - for (let i = 0; i < elemLength; i++) { - const currentElem = result.elements[i]; - const nextElem = result.elements[i + 1]; - if ( - currentElem !== undefined && - nextElem !== undefined && - currentElem?.data?.id.includes('*') === false && - nextElem?.data?.id.includes('*') === false - ) { - result.elements.push({ - data: { - id: `${currentElem.data.id}~${nextElem.data.id}`, - source: currentElem.data.id, - target: nextElem.data.id, - }, - }); - } - } + if (link.meta?.created_by === 'data-frame-analytics') { + nextLinkId = link.meta.analytics; + nextType = JOB_MAP_NODE_TYPES.ANALYTICS; + } - // fetch all jobs associated with root transform if defined, otherwise check root index - if (rootTransform !== undefined || rootIndexPattern !== undefined) { - const analyticsJobs = await this._mlClient.getDataFrameAnalytics(); - const jobs = analyticsJobs?.body?.data_frame_analytics || []; - const comparator = rootTransform !== undefined ? rootTransform : rootIndexPattern; + if (link.meta?.created_by === JOB_MAP_NODE_TYPES.TRANSFORM) { + nextLinkId = link.meta._transform?.transform; + nextType = JOB_MAP_NODE_TYPES.TRANSFORM; + } + } else if (isJobDataLinkReturnType(link) && link.isJob === true) { + data = link.jobData; + const nodeId = `${data.id}-${JOB_MAP_NODE_TYPES.ANALYTICS}`; + previousNodeId = nodeId; - for (let i = 0; i < jobs.length; i++) { - if ( - jobs[i]?.source?.index[0] === comparator && - this.isDuplicateElement(jobs[i].id, result.elements) === false - ) { - const nodeId = `${jobs[i].id}-${JOB_MAP_NODE_TYPES.ANALYTICS}`; - result.elements.push({ + result.elements.unshift({ data: { id: nodeId, - label: jobs[i].id, + label: data.id, type: JOB_MAP_NODE_TYPES.ANALYTICS, - analysisType: getAnalysisType(jobs[i]?.analysis), - }, - }); - result.details[nodeId] = jobs[i]; - const source = `${comparator}-${JOB_MAP_NODE_TYPES.INDEX}`; - result.elements.push({ - data: { - id: `${source}~${nodeId}`, - source, - target: nodeId, + analysisType: getAnalysisType(data?.analysis), }, }); + result.details[nodeId] = data; + nextLinkId = data?.source?.index[0]; + nextType = JOB_MAP_NODE_TYPES.INDEX; + // Get inference model for analytics job and create model node - ({ modelElement, modelDetails, edgeElement } = this.getAnalyticsModelElements( - jobs[i].id - )); + ({ modelElement, modelDetails, edgeElement } = this.getAnalyticsModelElements(data.id)); if (isAnalyticsMapNodeElement(modelElement)) { modelElements.push(modelElement); result.details[modelElement.data.id] = modelDetails; @@ -399,12 +437,88 @@ export class AnalyticsManager { if (isAnalyticsMapEdgeElement(edgeElement)) { modelElements.push(edgeElement); } + } else if (isTransformLinkReturnType(link) && link.isTransform === true) { + data = link.transformData; + + const nodeId = `${data.id}-${JOB_MAP_NODE_TYPES.TRANSFORM}`; + previousNodeId = nodeId; + rootTransform = data.dest.index; + + result.elements.unshift({ + data: { id: nodeId, label: data.id, type: JOB_MAP_NODE_TYPES.TRANSFORM }, + }); + result.details[nodeId] = data; + nextLinkId = data?.source?.index[0]; + nextType = JOB_MAP_NODE_TYPES.INDEX; + } + } // end while + + // create edge elements + const elemLength = result.elements.length - 1; + for (let i = 0; i < elemLength; i++) { + const currentElem = result.elements[i]; + const nextElem = result.elements[i + 1]; + if ( + currentElem !== undefined && + nextElem !== undefined && + currentElem?.data?.id.includes('*') === false && + nextElem?.data?.id.includes('*') === false + ) { + result.elements.push({ + data: { + id: `${currentElem.data.id}~${nextElem.data.id}`, + source: currentElem.data.id, + target: nextElem.data.id, + }, + }); + } + } + + // fetch all jobs associated with root transform if defined, otherwise check root index + if (rootTransform !== undefined || rootIndexPattern !== undefined) { + const jobs = await this.getAnalyticsData(); + const comparator = rootTransform !== undefined ? rootTransform : rootIndexPattern; + + for (let i = 0; i < jobs.length; i++) { + if ( + jobs[i]?.source?.index[0] === comparator && + this.isDuplicateElement(jobs[i].id, result.elements) === false + ) { + const nodeId = `${jobs[i].id}-${JOB_MAP_NODE_TYPES.ANALYTICS}`; + result.elements.push({ + data: { + id: nodeId, + label: jobs[i].id, + type: JOB_MAP_NODE_TYPES.ANALYTICS, + analysisType: getAnalysisType(jobs[i]?.analysis), + }, + }); + result.details[nodeId] = jobs[i]; + const source = `${comparator}-${JOB_MAP_NODE_TYPES.INDEX}`; + result.elements.push({ + data: { + id: `${source}~${nodeId}`, + source, + target: nodeId, + }, + }); + // Get inference model for analytics job and create model node + ({ modelElement, modelDetails, edgeElement } = this.getAnalyticsModelElements( + jobs[i].id + )); + if (isAnalyticsMapNodeElement(modelElement)) { + modelElements.push(modelElement); + result.details[modelElement.data.id] = modelDetails; + } + if (isAnalyticsMapEdgeElement(edgeElement)) { + modelElements.push(edgeElement); + } + } } } } // Include model and index pattern nodes in result elements now that all other nodes have been created result.elements.push(...modelElements, ...indexPatternElements); - return result; } catch (error) { result.error = error.message || 'An error occurred fetching map'; @@ -412,56 +526,64 @@ export class AnalyticsManager { } } - async extendAnalyticsMapForAnalyticsJob(analyticsId: string): Promise { - const result: any = { elements: [], details: {}, error: null }; - + async extendAnalyticsMapForAnalyticsJob({ + analyticsId, + index, + }: ExtendAnalyticsMapArgs): Promise { + const result: AnalyticsMapReturnType = { elements: [], details: {}, error: null }; try { await this.setInferenceModels(); + const jobs = await this.getAnalyticsData(); + let rootIndex; + let rootIndexNodeId; + + if (analyticsId !== undefined) { + const jobData = await this.getAnalyticsData(analyticsId); + const currentJobNodeId = `${jobData.id}-${JOB_MAP_NODE_TYPES.ANALYTICS}`; + rootIndex = Array.isArray(jobData?.dest?.index) + ? jobData?.dest?.index[0] + : jobData?.dest?.index; + rootIndexNodeId = `${rootIndex}-${JOB_MAP_NODE_TYPES.INDEX}`; + + // Fetch inference model for incoming job id and add node and edge + const { modelElement, modelDetails, edgeElement } = this.getAnalyticsModelElements( + analyticsId + ); + if (isAnalyticsMapNodeElement(modelElement)) { + result.elements.push(modelElement); + result.details[modelElement.data.id] = modelDetails; + } + if (isAnalyticsMapEdgeElement(edgeElement)) { + result.elements.push(edgeElement); + } - const jobData = await this.getAnalyticsJobData(analyticsId); - const currentJobNodeId = `${jobData.id}-${JOB_MAP_NODE_TYPES.ANALYTICS}`; - const destIndex = Array.isArray(jobData?.dest?.index) - ? jobData?.dest?.index[0] - : jobData?.dest?.index; - const destIndexNodeId = `${destIndex}-${JOB_MAP_NODE_TYPES.INDEX}`; - const analyticsJobs = await this._mlClient.getDataFrameAnalytics(); - const jobs = analyticsJobs?.body?.data_frame_analytics || []; - - // Fetch inference model for incoming job id and add node and edge - const { modelElement, modelDetails, edgeElement } = this.getAnalyticsModelElements( - analyticsId - ); - if (isAnalyticsMapNodeElement(modelElement)) { - result.elements.push(modelElement); - result.details[modelElement.data.id] = modelDetails; - } - if (isAnalyticsMapEdgeElement(edgeElement)) { - result.elements.push(edgeElement); + // If rootIndex node has not been created, create it + const rootIndexDetails = await this.getIndexData(rootIndex); + result.elements.push({ + data: { + id: rootIndexNodeId, + label: rootIndex, + type: JOB_MAP_NODE_TYPES.INDEX, + }, + }); + result.details[rootIndexNodeId] = rootIndexDetails; + + // Connect incoming job to rootIndex + result.elements.push({ + data: { + id: `${currentJobNodeId}~${rootIndexNodeId}`, + source: currentJobNodeId, + target: rootIndexNodeId, + }, + }); + } else { + rootIndex = index; + rootIndexNodeId = `${rootIndex}-${JOB_MAP_NODE_TYPES.INDEX}`; } - // If destIndex node has not been created, create it - const destIndexDetails = await this.getIndexData(destIndex); - result.elements.push({ - data: { - id: destIndexNodeId, - label: destIndex, - type: JOB_MAP_NODE_TYPES.INDEX, - }, - }); - result.details[destIndexNodeId] = destIndexDetails; - - // Connect incoming job to destIndex - result.elements.push({ - data: { - id: `${currentJobNodeId}~${destIndexNodeId}`, - source: currentJobNodeId, - target: destIndexNodeId, - }, - }); - for (let i = 0; i < jobs.length; i++) { if ( - jobs[i]?.source?.index[0] === destIndex && + jobs[i]?.source?.index[0] === rootIndex && this.isDuplicateElement(jobs[i].id, result.elements) === false ) { // Create node for associated job @@ -478,8 +600,8 @@ export class AnalyticsManager { result.elements.push({ data: { - id: `${destIndexNodeId}~${nodeId}`, - source: destIndexNodeId, + id: `${rootIndexNodeId}~${nodeId}`, + source: rootIndexNodeId, target: nodeId, }, }); diff --git a/x-pack/plugins/ml/server/models/data_frame_analytics/types.ts b/x-pack/plugins/ml/server/models/data_frame_analytics/types.ts index 5d6cec8cdfa6..e34d68ec7840 100644 --- a/x-pack/plugins/ml/server/models/data_frame_analytics/types.ts +++ b/x-pack/plugins/ml/server/models/data_frame_analytics/types.ts @@ -4,6 +4,29 @@ * you may not use this file except in compliance with the Elastic License. */ +import { JobMapNodeTypes } from '../../../common/constants/data_frame_analytics'; + +interface AnalyticsMapArg { + analyticsId: string; +} +interface GetAnalyticsJobIdArg extends AnalyticsMapArg { + modelId?: never; +} +interface GetAnalyticsModelIdArg { + analyticsId?: never; + modelId: string; +} +interface ExtendAnalyticsJobIdArg extends AnalyticsMapArg { + index?: never; +} +interface ExtendAnalyticsIndexArg { + analyticsId?: never; + index: string; +} + +export type GetAnalyticsMapArgs = GetAnalyticsJobIdArg | GetAnalyticsModelIdArg; +export type ExtendAnalyticsMapArgs = ExtendAnalyticsJobIdArg | ExtendAnalyticsIndexArg; + export interface IndexPatternLinkReturnType { isWildcardIndexPattern: boolean; isIndexPattern: boolean; @@ -26,9 +49,27 @@ export type NextLinkReturnType = export type MapElements = AnalyticsMapNodeElement | AnalyticsMapEdgeElement; export interface AnalyticsMapReturnType { elements: MapElements[]; - details: object; // transform, job, or index details + details: Record; // transform, job, or index details error: null | any; } + +interface BasicInitialElementsReturnType { + data: any; + details: object; + resultElements: MapElements[]; + modelElements: MapElements[]; +} + +export interface InitialElementsReturnType extends BasicInitialElementsReturnType { + nextLinkId?: string; + nextType?: JobMapNodeTypes; + previousNodeId?: string; +} +interface CompleteInitialElementsReturnType extends BasicInitialElementsReturnType { + nextLinkId: string; + nextType: JobMapNodeTypes; + previousNodeId: string; +} export interface AnalyticsMapNodeElement { data: { id: string; @@ -44,6 +85,16 @@ export interface AnalyticsMapEdgeElement { target: string; }; } +export const isCompleteInitialReturnType = (arg: any): arg is CompleteInitialElementsReturnType => { + if (typeof arg !== 'object' || arg === null) return false; + const keys = Object.keys(arg); + return ( + keys.length > 0 && + keys.includes('nextLinkId') && + keys.includes('nextType') && + keys.includes('previousNodeId') + ); +}; export const isAnalyticsMapNodeElement = (arg: any): arg is AnalyticsMapNodeElement => { if (typeof arg !== 'object' || arg === null) return false; const keys = Object.keys(arg); diff --git a/x-pack/plugins/ml/server/routes/apidoc.json b/x-pack/plugins/ml/server/routes/apidoc.json index 8d6dd692cc13..c157ae9e8200 100644 --- a/x-pack/plugins/ml/server/routes/apidoc.json +++ b/x-pack/plugins/ml/server/routes/apidoc.json @@ -17,6 +17,7 @@ "UpdateDataFrameAnalytics", "DeleteDataFrameAnalytics", "JobsExist", + "GetDataFrameAnalyticsIdMap", "DataVisualizer", "GetOverallStats", diff --git a/x-pack/plugins/ml/server/routes/data_frame_analytics.ts b/x-pack/plugins/ml/server/routes/data_frame_analytics.ts index 8e00ae706840..0abba7a429ae 100644 --- a/x-pack/plugins/ml/server/routes/data_frame_analytics.ts +++ b/x-pack/plugins/ml/server/routes/data_frame_analytics.ts @@ -8,6 +8,7 @@ import { RequestHandlerContext, IScopedClusterClient } from 'kibana/server'; import { wrapError } from '../client/error_wrapper'; import { analyticsAuditMessagesProvider } from '../models/data_frame_analytics/analytics_audit_messages'; import { RouteInitialization } from '../types'; +import { JOB_MAP_NODE_TYPES } from '../../common/constants/data_frame_analytics'; import { dataAnalyticsJobConfigSchema, dataAnalyticsJobUpdateSchema, @@ -19,6 +20,7 @@ import { deleteDataFrameAnalyticsJobSchema, jobsExistSchema, } from './schemas/data_analytics_schema'; +import { GetAnalyticsMapArgs, ExtendAnalyticsMapArgs } from '../models/data_frame_analytics/types'; import { IndexPatternHandler } from '../models/data_frame_analytics/index_patterns'; import { AnalyticsManager } from '../models/data_frame_analytics/analytics_manager'; import { DeleteDataFrameAnalyticsWithIndexStatus } from '../../common/types/data_frame_analytics'; @@ -36,14 +38,22 @@ function deleteDestIndexPatternById(context: RequestHandlerContext, indexPattern return iph.deleteIndexPatternById(indexPatternId); } -function getAnalyticsMap(mlClient: MlClient, client: IScopedClusterClient, analyticsId: string) { +function getAnalyticsMap( + mlClient: MlClient, + client: IScopedClusterClient, + idOptions: GetAnalyticsMapArgs +) { const analytics = new AnalyticsManager(mlClient, client.asInternalUser); - return analytics.getAnalyticsMap(analyticsId); + return analytics.getAnalyticsMap(idOptions); } -function getExtendedMap(mlClient: MlClient, client: IScopedClusterClient, analyticsId: string) { +function getExtendedMap( + mlClient: MlClient, + client: IScopedClusterClient, + idOptions: ExtendAnalyticsMapArgs +) { const analytics = new AnalyticsManager(mlClient, client.asInternalUser); - return analytics.extendAnalyticsMapForAnalyticsJob(analyticsId); + return analytics.extendAnalyticsMapForAnalyticsJob(idOptions); } /** @@ -633,10 +643,20 @@ export function dataFrameAnalyticsRoutes({ router, mlLicense, routeGuard }: Rout try { const { analyticsId } = request.params; const treatAsRoot = request.query?.treatAsRoot; - const caller = - treatAsRoot === 'true' || treatAsRoot === true ? getExtendedMap : getAnalyticsMap; + const type = request.query?.type; - const results = await caller(mlClient, client, analyticsId); + let results; + if (treatAsRoot === 'true' || treatAsRoot === true) { + results = await getExtendedMap(mlClient, client, { + analyticsId: type !== JOB_MAP_NODE_TYPES.INDEX ? analyticsId : undefined, + index: type === JOB_MAP_NODE_TYPES.INDEX ? analyticsId : undefined, + }); + } else { + results = await getAnalyticsMap(mlClient, client, { + analyticsId: type !== JOB_MAP_NODE_TYPES.TRAINED_MODEL ? analyticsId : undefined, + modelId: type === JOB_MAP_NODE_TYPES.TRAINED_MODEL ? analyticsId : undefined, + }); + } return response.ok({ body: results, diff --git a/x-pack/plugins/ml/server/routes/schemas/data_analytics_schema.ts b/x-pack/plugins/ml/server/routes/schemas/data_analytics_schema.ts index d8226b70eb2c..cf52d1cb2743 100644 --- a/x-pack/plugins/ml/server/routes/schemas/data_analytics_schema.ts +++ b/x-pack/plugins/ml/server/routes/schemas/data_analytics_schema.ts @@ -89,5 +89,5 @@ export const jobsExistSchema = schema.object({ }); export const analyticsMapQuerySchema = schema.maybe( - schema.object({ treatAsRoot: schema.maybe(schema.any()) }) + schema.object({ treatAsRoot: schema.maybe(schema.any()), type: schema.maybe(schema.string()) }) );