From de33e9770debfa496c0c3c0e25f024d3f0ce9552 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Mon, 19 Oct 2020 18:52:51 +0200 Subject: [PATCH] [ML] Fix race condition when loading index data. Clean up legacy outlier influence code. --- .../components/data_grid/common.ts | 11 +----- .../data_frame_analytics/common/fields.ts | 20 ++--------- .../common/get_index_data.ts | 18 +++++----- .../components/outlier_exploration/common.ts | 7 ++-- .../outlier_exploration.tsx | 9 ++--- .../outlier_exploration/use_outlier_data.ts | 36 +++++++++---------- 6 files changed, 34 insertions(+), 67 deletions(-) diff --git a/x-pack/plugins/ml/public/application/components/data_grid/common.ts b/x-pack/plugins/ml/public/application/components/data_grid/common.ts index f88694a1952b2..642d0ae564b85 100644 --- a/x-pack/plugins/ml/public/application/components/data_grid/common.ts +++ b/x-pack/plugins/ml/public/application/components/data_grid/common.ts @@ -33,7 +33,6 @@ import { import { FEATURE_IMPORTANCE, - FEATURE_INFLUENCE, OUTLIER_SCORE, TOP_CLASSES, } from '../../data_frame_analytics/common/constants'; @@ -112,10 +111,7 @@ export const getDataGridSchemasFromFieldTypes = (fieldTypes: FieldTypes, results schema = NON_AGGREGATABLE; } - if ( - field === `${resultsField}.${OUTLIER_SCORE}` || - field.includes(`${resultsField}.${FEATURE_INFLUENCE}`) - ) { + if (field === `${resultsField}.${OUTLIER_SCORE}`) { schema = 'numeric'; } @@ -203,11 +199,6 @@ export const useRenderCellValue = ( } function getCellValue(cId: string) { - if (cId.includes(`.${FEATURE_INFLUENCE}.`) && resultsField !== undefined) { - const results = getNestedProperty(tableItems[adjustedRowIndex], resultsField, null); - return results[cId.replace(`${resultsField}.`, '')]; - } - if (tableItems.hasOwnProperty(adjustedRowIndex)) { const item = tableItems[adjustedRowIndex]; diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/common/fields.ts b/x-pack/plugins/ml/public/application/data_frame_analytics/common/fields.ts index e4581f0a87bdd..c606cbd1cc11a 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/common/fields.ts +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/common/fields.ts @@ -17,7 +17,7 @@ import { import { ES_FIELD_TYPES, KBN_FIELD_TYPES } from '../../../../../../../src/plugins/data/public'; import { newJobCapsService } from '../../services/new_job_capabilities_service'; -import { FEATURE_IMPORTANCE, FEATURE_INFLUENCE, OUTLIER_SCORE, TOP_CLASSES } from './constants'; +import { FEATURE_IMPORTANCE, OUTLIER_SCORE, TOP_CLASSES } from './constants'; import { DataFrameAnalyticsConfig } from '../../../../common/types/data_frame_analytics'; export type EsId = string; @@ -179,7 +179,6 @@ export const getDefaultFieldsFromJobCaps = ( const resultsField = jobConfig.dest.results_field; const featureImportanceFields = []; - const featureInfluenceFields = []; const topClassesFields = []; const allFields: any = []; let type: ES_FIELD_TYPES | undefined; @@ -193,16 +192,6 @@ export const getDefaultFieldsFromJobCaps = ( name: `${resultsField}.${OUTLIER_SCORE}`, type: KBN_FIELD_TYPES.NUMBER, }); - - featureInfluenceFields.push( - ...fields - .filter((d) => !jobConfig.analyzed_fields.excludes.includes(d.id)) - .map((d) => ({ - id: `${resultsField}.${FEATURE_INFLUENCE}.${d.id}`, - name: `${resultsField}.${FEATURE_INFLUENCE}.${d.name}`, - type: KBN_FIELD_TYPES.NUMBER, - })) - ); } } @@ -247,12 +236,7 @@ export const getDefaultFieldsFromJobCaps = ( } } - allFields.push( - ...fields, - ...featureImportanceFields, - ...featureInfluenceFields, - ...topClassesFields - ); + allFields.push(...fields, ...featureImportanceFields, ...topClassesFields); allFields.sort(({ name: a }: { name: string }, { name: b }: { name: string }) => sortExplorationResultsFields(a, b, jobConfig) ); diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/common/get_index_data.ts b/x-pack/plugins/ml/public/application/data_frame_analytics/common/get_index_data.ts index 667dea27de96e..8e50aab0914db 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/common/get_index_data.ts +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/common/get_index_data.ts @@ -19,7 +19,8 @@ import { DataFrameAnalyticsConfig } from '../../../../common/types/data_frame_an export const getIndexData = async ( jobConfig: DataFrameAnalyticsConfig | undefined, dataGrid: UseDataGridReturnType, - searchQuery: SavedSearchQuery + searchQuery: SavedSearchQuery, + options: { didCancel: boolean } ) => { if (jobConfig !== undefined) { const { @@ -52,7 +53,7 @@ export const getIndexData = async ( index: jobConfig.dest.index, body: { fields: ['*'], - _source: jobConfig.dest.results_field, + _source: [], query: searchQuery, from: pageIndex * pageSize, size: pageSize, @@ -60,14 +61,11 @@ export const getIndexData = async ( }, }); - setRowCount(resp.hits.total.value); - const docs = resp.hits.hits.map((d) => ({ - ...getProcessedFields(d.fields), - [jobConfig.dest.results_field]: d._source[jobConfig.dest.results_field], - })); - - setTableItems(docs); - setStatus(INDEX_STATUS.LOADED); + if (!options.didCancel) { + setRowCount(resp.hits.total.value); + setTableItems(resp.hits.hits.map((d) => getProcessedFields(d.fields))); + setStatus(INDEX_STATUS.LOADED); + } } catch (e) { setErrorMessage(extractErrorMessage(e)); setStatus(INDEX_STATUS.ERROR); diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/outlier_exploration/common.ts b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/outlier_exploration/common.ts index d1889a8acb990..1ce3b3528e44b 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/outlier_exploration/common.ts +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/outlier_exploration/common.ts @@ -19,11 +19,8 @@ export const getFeatureCount = (resultsField: string, tableItems: DataGridItem[] const fullItem = tableItems[0]; - if ( - fullItem[resultsField] !== undefined && - Array.isArray(fullItem[resultsField][FEATURE_INFLUENCE]) - ) { - return fullItem[resultsField][FEATURE_INFLUENCE].length; + if (Array.isArray(fullItem[`${resultsField}.${FEATURE_INFLUENCE}.feature_name`])) { + return fullItem[`${resultsField}.${FEATURE_INFLUENCE}.feature_name`].length; } return 0; diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/outlier_exploration/outlier_exploration.tsx b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/outlier_exploration/outlier_exploration.tsx index 8fc2486599755..fa711316df003 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/outlier_exploration/outlier_exploration.tsx +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/outlier_exploration/outlier_exploration.tsx @@ -36,11 +36,8 @@ export const OutlierExploration: FC = React.memo(({ jobId }) = const { columnsWithCharts, tableItems } = outlierData; - const colorRange = useColorRange( - COLOR_RANGE.BLUE, - COLOR_RANGE_SCALE.INFLUENCER, - jobConfig !== undefined ? getFeatureCount(jobConfig.dest.results_field, tableItems) : 1 - ); + const featureCount = getFeatureCount(jobConfig?.dest?.results_field || '', tableItems); + const colorRange = useColorRange(COLOR_RANGE.BLUE, COLOR_RANGE_SCALE.INFLUENCER, featureCount); return ( <> @@ -59,7 +56,7 @@ export const OutlierExploration: FC = React.memo(({ jobId }) = )} {typeof jobConfig?.id === 'string' && } 0 ? colorRange : undefined} indexData={outlierData} indexPattern={indexPattern} jobConfig={jobConfig} diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/outlier_exploration/use_outlier_data.ts b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/outlier_exploration/use_outlier_data.ts index 88aa06808e8a7..148ff6b13699f 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/outlier_exploration/use_outlier_data.ts +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/outlier_exploration/use_outlier_data.ts @@ -39,11 +39,6 @@ import { import { getFeatureCount, getOutlierScoreFieldName } from './common'; -interface FeatureInfluence { - feature_name: string; - influence: number; -} - export const useOutlierData = ( indexPattern: IndexPattern | undefined, jobConfig: DataFrameAnalyticsConfig | undefined, @@ -88,8 +83,15 @@ export const useOutlierData = ( } }, [jobConfig && jobConfig.id]); + // The pattern using `didCancel` allows us to abort out of date remote request. + // We wrap `didCancel` in a object so we can mutate the value as it's being + // passed on to `getIndexData`. useEffect(() => { - getIndexData(jobConfig, dataGrid, searchQuery); + const options = { didCancel: false }; + getIndexData(jobConfig, dataGrid, searchQuery, options); + return () => { + options.didCancel = true; + }; // custom comparison }, [jobConfig && jobConfig.id, dataGrid.pagination, searchQuery, dataGrid.sortingColumns]); @@ -151,19 +153,17 @@ export const useOutlierData = ( const split = columnId.split('.'); let backgroundColor; + const featureNames = fullItem[`${resultsField}.${FEATURE_INFLUENCE}.feature_name`]; + // column with feature values get color coded by its corresponding influencer value - if ( - fullItem[resultsField] !== undefined && - fullItem[resultsField][FEATURE_INFLUENCE] !== undefined && - fullItem[resultsField][FEATURE_INFLUENCE].find( - (d: FeatureInfluence) => d.feature_name === columnId - ) !== undefined - ) { - backgroundColor = colorRange( - fullItem[resultsField][FEATURE_INFLUENCE].find( - (d: FeatureInfluence) => d.feature_name === columnId - ).influence - ); + if (Array.isArray(featureNames)) { + const featureIndex = featureNames.indexOf(columnId); + + if (featureIndex > -1) { + backgroundColor = colorRange( + fullItem[`${resultsField}.${FEATURE_INFLUENCE}.influence`][featureIndex] + ); + } } // column with influencer values get color coded by its own value