diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/common/analytics.ts b/x-pack/plugins/ml/public/application/data_frame_analytics/common/analytics.ts index 511ebb7e1647a..3c959b827bb1c 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/common/analytics.ts +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/common/analytics.ts @@ -13,7 +13,6 @@ import { ml } from '../../services/ml_api_service'; import { Dictionary } from '../../../../common/types/common'; import { getErrorMessage } from '../../../../common/util/errors'; import { SavedSearchQuery } from '../../contexts/ml'; -import { SortDirection } from '../../components/ml_in_memory_table'; export type IndexName = string; export type IndexPattern = string; @@ -53,13 +52,9 @@ export interface ClassificationAnalysis { classification: Classification; } -export interface LoadExploreDataArg { - field: string; - direction: SortDirection; +export interface LoadRegressionExploreDataArg { + filterByIsTraining?: boolean; searchQuery: SavedSearchQuery; - requiresKeyword?: boolean; - pageIndex?: number; - pageSize?: number; } export const SEARCH_SIZE = 1000; @@ -272,6 +267,11 @@ export const isResultsSearchBoolQuery = (arg: any): arg is ResultsSearchBoolQuer return keys.length === 1 && keys[0] === 'bool'; }; +export const isQueryStringQuery = (arg: any): arg is QueryStringQuery => { + const keys = Object.keys(arg); + return keys.length === 1 && keys[0] === 'query_string'; +}; + export const isRegressionEvaluateResponse = (arg: any): arg is RegressionEvaluateResponse => { const keys = Object.keys(arg); return ( @@ -396,6 +396,10 @@ interface ResultsSearchTermQuery { term: Dictionary; } +interface QueryStringQuery { + query_string: Dictionary; +} + export type ResultsSearchQuery = ResultsSearchBoolQuery | ResultsSearchTermQuery | SavedSearchQuery; export function getEvalQueryBody({ @@ -409,16 +413,34 @@ export function getEvalQueryBody({ searchQuery?: ResultsSearchQuery; ignoreDefaultQuery?: boolean; }) { - let query: ResultsSearchQuery = { + let query; + + const trainingQuery: ResultsSearchQuery = { term: { [`${resultsField}.is_training`]: { value: isTraining } }, }; - if (searchQuery !== undefined && ignoreDefaultQuery === true) { - query = searchQuery; - } else if (searchQuery !== undefined && isResultsSearchBoolQuery(searchQuery)) { - const searchQueryClone = cloneDeep(searchQuery); - searchQueryClone.bool.must.push(query); + const searchQueryClone = cloneDeep(searchQuery); + + if (isResultsSearchBoolQuery(searchQueryClone)) { + if (searchQueryClone.bool.must === undefined) { + searchQueryClone.bool.must = []; + } + + searchQueryClone.bool.must.push(trainingQuery); query = searchQueryClone; + } else if (isQueryStringQuery(searchQueryClone)) { + query = { + bool: { + must: [searchQueryClone, trainingQuery], + }, + }; + } else { + // Not a bool or string query so we need to create it so can add the trainingQuery + query = { + bool: { + must: [trainingQuery], + }, + }; } return query; } diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/common/fields.ts b/x-pack/plugins/ml/public/application/data_frame_analytics/common/fields.ts index 92d8731959895..f165669bdd674 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/common/fields.ts +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/common/fields.ts @@ -246,8 +246,15 @@ export const getDefaultFieldsFromJobCaps = ( fields: Field[], jobConfig: DataFrameAnalyticsConfig, needsDestIndexFields: boolean -): { selectedFields: Field[]; docFields: Field[]; depVarType?: ES_FIELD_TYPES } => { - const fieldsObj = { selectedFields: [], docFields: [] }; +): { + selectedFields: Field[]; + docFields: Field[]; + depVarType?: ES_FIELD_TYPES; +} => { + const fieldsObj = { + selectedFields: [], + docFields: [], + }; if (fields.length === 0) { return fieldsObj; } @@ -267,38 +274,37 @@ export const getDefaultFieldsFromJobCaps = ( const featureImportanceFields = []; if ((numTopFeatureImportanceValues ?? 0) > 0) { - featureImportanceFields.push( - ...fields.map(d => ({ - id: `${resultsField}.feature_importance.${d.id}`, - name: `${resultsField}.feature_importance.${d.name}`, - type: KBN_FIELD_TYPES.NUMBER, - })) - ); + featureImportanceFields.push({ + id: `${resultsField}.feature_importance`, + name: `${resultsField}.feature_importance`, + type: KBN_FIELD_TYPES.NUMBER, + }); } + let allFields: any = []; // Only need to add these fields if we didn't use dest index pattern to get the fields - const allFields: any = - needsDestIndexFields === true - ? [ - { - id: `${resultsField}.is_training`, - name: `${resultsField}.is_training`, - type: ES_FIELD_TYPES.BOOLEAN, - }, - { id: predictedField, name: predictedField, type }, - ...featureImportanceFields, - ] - : []; - - allFields.push(...fields); + if (needsDestIndexFields === true) { + allFields.push( + { + id: `${resultsField}.is_training`, + name: `${resultsField}.is_training`, + type: ES_FIELD_TYPES.BOOLEAN, + }, + { id: predictedField, name: predictedField, type } + ); + } + + allFields.push(...fields, ...featureImportanceFields); allFields.sort(({ name: a }: { name: string }, { name: b }: { name: string }) => sortRegressionResultsFields(a, b, jobConfig) ); + // Remove feature_importance fields provided by dest index since feature_importance is an array the path is not valid + if (needsDestIndexFields === false) { + allFields = allFields.filter((field: any) => !field.name.includes('.feature_importance.')); + } let selectedFields = allFields.filter( - (field: any) => - field.name === predictedField || - (!field.name.includes('.keyword') && !field.name.includes('.feature_importance.')) + (field: any) => field.name === predictedField || !field.name.includes('.keyword') ); if (selectedFields.length > DEFAULT_REGRESSION_COLUMNS) { diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/classification_exploration/use_explore_data.ts b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/classification_exploration/use_explore_data.ts index 6038def592e5c..9527a9adb98ce 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/classification_exploration/use_explore_data.ts +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/classification_exploration/use_explore_data.ts @@ -22,7 +22,6 @@ import { newJobCapsService } from '../../../../../services/new_job_capabilities_ import { Field } from '../../../../../../../common/types/fields'; import { ES_FIELD_TYPES } from '../../../../../../../../../../src/plugins/data/public'; import { - LoadExploreDataArg, defaultSearchQuery, ResultsSearchQuery, isResultsSearchBoolQuery, @@ -37,12 +36,23 @@ import { SEARCH_SIZE, SearchQuery, } from '../../../../common'; +import { SavedSearchQuery } from '../../../../../contexts/ml'; + +interface LoadClassificationExploreDataArg { + direction: SortDirection; + filterByIsTraining?: boolean; + field: string; + searchQuery: SavedSearchQuery; + requiresKeyword?: boolean; + pageIndex?: number; + pageSize?: number; +} export type TableItem = Record; export interface UseExploreDataReturnType { errorMessage: string; - loadExploreData: (arg: LoadExploreDataArg) => void; + loadExploreData: (arg: LoadClassificationExploreDataArg) => void; sortField: EsFieldName; sortDirection: SortDirection; status: INDEX_STATUS; @@ -84,7 +94,7 @@ export const useExploreData = ( direction, searchQuery, requiresKeyword, - }: LoadExploreDataArg) => { + }: LoadClassificationExploreDataArg) => { if (jobConfig !== undefined) { setErrorMessage(''); setStatus(INDEX_STATUS.LOADING); diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/regression_exploration/evaluate_panel.tsx b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/regression_exploration/evaluate_panel.tsx index 9f235ae6c45c0..6ef6666be5ec6 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/regression_exploration/evaluate_panel.tsx +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/regression_exploration/evaluate_panel.tsx @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import React, { FC, Fragment, useEffect, useState } from 'react'; +import React, { FC, useEffect, useState } from 'react'; import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n/react'; import { @@ -17,7 +17,7 @@ import { EuiTitle, } from '@elastic/eui'; import { useMlKibana } from '../../../../../contexts/kibana'; -import { ErrorCallout } from '../error_callout'; +import { SavedSearchQuery } from '../../../../../contexts/ml'; import { getValuesFromResponse, getDependentVar, @@ -33,14 +33,13 @@ import { EvaluateStat } from './evaluate_stat'; import { isResultsSearchBoolQuery, isRegressionEvaluateResponse, - ResultsSearchQuery, ANALYSIS_CONFIG_TYPE, } from '../../../../common/analytics'; interface Props { jobConfig: DataFrameAnalyticsConfig; jobStatus?: DATA_FRAME_TASK_STATE; - searchQuery: ResultsSearchQuery; + searchQuery: SavedSearchQuery; } const defaultEval: Eval = { meanSquaredError: '', rSquared: '', error: null }; @@ -54,6 +53,7 @@ export const EvaluatePanel: FC = ({ jobConfig, jobStatus, searchQuery }) const [generalizationEval, setGeneralizationEval] = useState(defaultEval); const [isLoadingTraining, setIsLoadingTraining] = useState(false); const [isLoadingGeneralization, setIsLoadingGeneralization] = useState(false); + const [isTrainingFilter, setIsTrainingFilter] = useState(undefined); const [trainingDocsCount, setTrainingDocsCount] = useState(null); const [generalizationDocsCount, setGeneralizationDocsCount] = useState(null); @@ -92,8 +92,8 @@ export const EvaluatePanel: FC = ({ jobConfig, jobStatus, searchQuery }) } else { setIsLoadingGeneralization(false); setGeneralizationEval({ - meanSquaredError: '', - rSquared: '', + meanSquaredError: '--', + rSquared: '--', error: genErrorEval.error, }); } @@ -128,108 +128,78 @@ export const EvaluatePanel: FC = ({ jobConfig, jobStatus, searchQuery }) } else { setIsLoadingTraining(false); setTrainingEval({ - meanSquaredError: '', - rSquared: '', + meanSquaredError: '--', + rSquared: '--', error: trainingErrorEval.error, }); } }; - const loadData = async ({ - isTrainingClause, - }: { - isTrainingClause?: { query: string; operator: string }; - }) => { - // searchBar query is filtering for testing data - if (isTrainingClause !== undefined && isTrainingClause.query === 'false') { - loadGeneralizationData(); - - const docsCountResp = await loadDocsCount({ - isTraining: false, - searchQuery, - resultsField, - destIndex: jobConfig.dest.index, - }); - - if (docsCountResp.success === true) { - setGeneralizationDocsCount(docsCountResp.docsCount); - } else { - setGeneralizationDocsCount(null); - } - - setTrainingDocsCount(0); - setTrainingEval({ - meanSquaredError: '--', - rSquared: '--', - error: null, - }); - } else if (isTrainingClause !== undefined && isTrainingClause.query === 'true') { - // searchBar query is filtering for training data - loadTrainingData(); - - const docsCountResp = await loadDocsCount({ - isTraining: true, - searchQuery, - resultsField, - destIndex: jobConfig.dest.index, - }); - - if (docsCountResp.success === true) { - setTrainingDocsCount(docsCountResp.docsCount); - } else { - setTrainingDocsCount(null); - } - - setGeneralizationDocsCount(0); - setGeneralizationEval({ - meanSquaredError: '--', - rSquared: '--', - error: null, - }); + const loadData = async () => { + loadGeneralizationData(false); + const genDocsCountResp = await loadDocsCount({ + ignoreDefaultQuery: false, + isTraining: false, + searchQuery, + resultsField, + destIndex: jobConfig.dest.index, + }); + if (genDocsCountResp.success === true) { + setGeneralizationDocsCount(genDocsCountResp.docsCount); } else { - // No is_training clause/filter from search bar so load both - loadGeneralizationData(false); - const genDocsCountResp = await loadDocsCount({ - ignoreDefaultQuery: false, - isTraining: false, - searchQuery, - resultsField, - destIndex: jobConfig.dest.index, - }); - if (genDocsCountResp.success === true) { - setGeneralizationDocsCount(genDocsCountResp.docsCount); - } else { - setGeneralizationDocsCount(null); - } + setGeneralizationDocsCount(null); + } - loadTrainingData(false); - const trainDocsCountResp = await loadDocsCount({ - ignoreDefaultQuery: false, - isTraining: true, - searchQuery, - resultsField, - destIndex: jobConfig.dest.index, - }); - if (trainDocsCountResp.success === true) { - setTrainingDocsCount(trainDocsCountResp.docsCount); - } else { - setTrainingDocsCount(null); - } + loadTrainingData(false); + const trainDocsCountResp = await loadDocsCount({ + ignoreDefaultQuery: false, + isTraining: true, + searchQuery, + resultsField, + destIndex: jobConfig.dest.index, + }); + if (trainDocsCountResp.success === true) { + setTrainingDocsCount(trainDocsCountResp.docsCount); + } else { + setTrainingDocsCount(null); } }; useEffect(() => { - const hasIsTrainingClause = - isResultsSearchBoolQuery(searchQuery) && - searchQuery.bool.must.filter( - (clause: any) => clause.match && clause.match[`${resultsField}.is_training`] !== undefined - ); - const isTrainingClause = - hasIsTrainingClause && - hasIsTrainingClause[0] && - hasIsTrainingClause[0].match[`${resultsField}.is_training`]; + let isTraining: boolean | undefined; + const query = + isResultsSearchBoolQuery(searchQuery) && (searchQuery.bool.should || searchQuery.bool.filter); + + if (query !== undefined && query !== false) { + for (let i = 0; i < query.length; i++) { + const clause = query[i]; - loadData({ isTrainingClause }); + if (clause.match && clause.match[`${resultsField}.is_training`] !== undefined) { + isTraining = clause.match[`${resultsField}.is_training`]; + break; + } else if ( + clause.bool && + (clause.bool.should !== undefined || clause.bool.filter !== undefined) + ) { + const innerQuery = clause.bool.should || clause.bool.filter; + if (innerQuery !== undefined) { + for (let j = 0; j < innerQuery.length; j++) { + const innerClause = innerQuery[j]; + if ( + innerClause.match && + innerClause.match[`${resultsField}.is_training`] !== undefined + ) { + isTraining = innerClause.match[`${resultsField}.is_training`]; + break; + } + } + } + } + } + } + + setIsTrainingFilter(isTraining); + loadData(); }, [JSON.stringify(searchQuery)]); return ( @@ -293,13 +263,18 @@ export const EvaluatePanel: FC = ({ jobConfig, jobStatus, searchQuery }) defaultMessage="{docsCount, plural, one {# doc} other {# docs}} evaluated" values={{ docsCount: generalizationDocsCount }} /> + {isTrainingFilter === true && generalizationDocsCount === 0 && ( + + )} )} - - {generalizationEval.error !== null && } - {generalizationEval.error === null && ( - + + + = ({ jobConfig, jobStatus, searchQuery }) isMSE={false} /> - + + + {generalizationEval.error !== null && ( + + + {generalizationEval.error} + + )} @@ -338,13 +320,18 @@ export const EvaluatePanel: FC = ({ jobConfig, jobStatus, searchQuery }) defaultMessage="{docsCount, plural, one {# doc} other {# docs}} evaluated" values={{ docsCount: trainingDocsCount }} /> + {isTrainingFilter === false && trainingDocsCount === 0 && ( + + )} )} - - {trainingEval.error !== null && } - {trainingEval.error === null && ( - + + + = ({ jobConfig, jobStatus, searchQuery }) isMSE={false} /> - + + + {trainingEval.error !== null && ( + + + {trainingEval.error} + + )} diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/regression_exploration/regression_exploration_data_grid.tsx b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/regression_exploration/regression_exploration_data_grid.tsx new file mode 100644 index 0000000000000..0fcb1ed600719 --- /dev/null +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/regression_exploration/regression_exploration_data_grid.tsx @@ -0,0 +1,135 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { Dispatch, FC, SetStateAction, useCallback, useMemo } from 'react'; + +import { i18n } from '@kbn/i18n'; + +import { EuiDataGrid, EuiDataGridPaginationProps, EuiDataGridSorting } from '@elastic/eui'; + +import { euiDataGridStyle, euiDataGridToolbarSettings } from '../../../../common'; + +import { mlFieldFormatService } from '../../../../../services/field_format_service'; + +import { IndexPattern } from '../../../../../../../../../../src/plugins/data/public'; + +const PAGE_SIZE_OPTIONS = [5, 10, 25, 50]; + +type Pagination = Pick; +type TableItem = Record; + +interface ExplorationDataGridProps { + colorRange?: (d: number) => string; + columns: any[]; + indexPattern: IndexPattern; + pagination: Pagination; + resultsField: string; + rowCount: number; + selectedFields: string[]; + setPagination: Dispatch>; + setSelectedFields: Dispatch>; + setSortingColumns: Dispatch>; + sortingColumns: EuiDataGridSorting['columns']; + tableItems: TableItem[]; +} + +export const RegressionExplorationDataGrid: FC = ({ + columns, + indexPattern, + pagination, + resultsField, + rowCount, + selectedFields, + setPagination, + setSelectedFields, + setSortingColumns, + sortingColumns, + tableItems, +}) => { + const renderCellValue = useMemo(() => { + return ({ rowIndex, columnId }: { rowIndex: number; columnId: string; setCellProps: any }) => { + const adjustedRowIndex = rowIndex - pagination.pageIndex * pagination.pageSize; + + const fullItem = tableItems[adjustedRowIndex]; + + if (fullItem === undefined) { + return null; + } + + let format: any; + + if (indexPattern !== undefined) { + format = mlFieldFormatService.getFieldFormatFromIndexPattern(indexPattern, columnId, ''); + } + + const cellValue = + fullItem.hasOwnProperty(columnId) && fullItem[columnId] !== undefined + ? fullItem[columnId] + : null; + + if (format !== undefined) { + return format.convert(cellValue, 'text'); + } + + if (typeof cellValue === 'string' || cellValue === null) { + return cellValue; + } + + if (typeof cellValue === 'boolean') { + return cellValue ? 'true' : 'false'; + } + + if (typeof cellValue === 'object' && cellValue !== null) { + return JSON.stringify(cellValue); + } + + return cellValue; + }; + }, [resultsField, rowCount, tableItems, pagination.pageIndex, pagination.pageSize]); + + const onChangeItemsPerPage = useCallback( + pageSize => { + setPagination(p => { + const pageIndex = Math.floor((p.pageSize * p.pageIndex) / pageSize); + return { pageIndex, pageSize }; + }); + }, + [setPagination] + ); + + const onChangePage = useCallback(pageIndex => setPagination(p => ({ ...p, pageIndex })), [ + setPagination, + ]); + + const onSort = useCallback(sc => setSortingColumns(sc), [setSortingColumns]); + + return ( + + ); +}; diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/regression_exploration/results_table.tsx b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/regression_exploration/results_table.tsx index a35be5400f46b..43fa50b2e4df5 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/regression_exploration/results_table.tsx +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/regression_exploration/results_table.tsx @@ -4,72 +4,41 @@ * you may not use this file except in compliance with the Elastic License. */ -import React, { Fragment, FC, useEffect, useState } from 'react'; -import moment from 'moment-timezone'; +import React, { Fragment, FC, useEffect } from 'react'; import { i18n } from '@kbn/i18n'; import { - EuiBadge, - EuiButtonIcon, EuiCallOut, - EuiCheckbox, EuiFlexGroup, EuiFlexItem, EuiFormRow, EuiPanel, - EuiPopover, - EuiPopoverTitle, EuiProgress, EuiSpacer, EuiText, - EuiToolTip, - Query, } from '@elastic/eui'; -import { Query as QueryType } from '../../../analytics_management/components/analytics_list/common'; -import { ES_FIELD_TYPES } from '../../../../../../../../../../src/plugins/data/public'; -import { mlFieldFormatService } from '../../../../../services/field_format_service'; - -import { - ColumnType, - mlInMemoryTableBasicFactory, - OnTableChangeArg, - SortingPropType, - SORT_DIRECTION, -} from '../../../../../components/ml_in_memory_table'; - -import { formatHumanReadableDateTimeSeconds } from '../../../../../util/date_utils'; -import { Field } from '../../../../../../../common/types/fields'; -import { SavedSearchQuery } from '../../../../../contexts/ml'; import { BASIC_NUMERICAL_TYPES, EXTENDED_NUMERICAL_TYPES, - toggleSelectedField, - isKeywordAndTextType, sortRegressionResultsFields, } from '../../../../common/fields'; import { DataFrameAnalyticsConfig, - EsFieldName, - EsDoc, MAX_COLUMNS, - getPredictedFieldName, INDEX_STATUS, SEARCH_SIZE, defaultSearchQuery, - getDependentVar, } from '../../../../common'; import { getTaskStateBadge } from '../../../analytics_management/components/analytics_list/columns'; import { DATA_FRAME_TASK_STATE } from '../../../analytics_management/components/analytics_list/common'; import { IndexPattern } from '../../../../../../../../../../src/plugins/data/public'; -import { useExploreData, TableItem } from './use_explore_data'; +import { useExploreData } from './use_explore_data'; import { ExplorationTitle } from './regression_exploration'; - -const PAGE_SIZE_OPTIONS = [5, 10, 25, 50]; - -const MlInMemoryTableBasic = mlInMemoryTableBasicFactory(); +import { RegressionExplorationDataGrid } from './regression_exploration_data_grid'; +import { ExplorationQueryBar } from '../exploration_query_bar'; const showingDocs = i18n.translate( 'xpack.ml.dataframe.analytics.regressionExploration.documentsShownHelpText', @@ -95,308 +64,65 @@ interface Props { export const ResultsTable: FC = React.memo( ({ indexPattern, jobConfig, jobStatus, setEvaluateSearchQuery }) => { - const [pageIndex, setPageIndex] = useState(0); - const [pageSize, setPageSize] = useState(25); - const [selectedFields, setSelectedFields] = useState([] as Field[]); - const [docFields, setDocFields] = useState([] as Field[]); - const [depVarType, setDepVarType] = useState(undefined); - const [isColumnsPopoverVisible, setColumnsPopoverVisible] = useState(false); - const [searchQuery, setSearchQuery] = useState(defaultSearchQuery); - const [searchError, setSearchError] = useState(undefined); - const [searchString, setSearchString] = useState(undefined); - - const predictedFieldName = getPredictedFieldName( - jobConfig.dest.results_field, - jobConfig.analysis - ); - - const dependentVariable = getDependentVar(jobConfig.analysis); - - function toggleColumnsPopover() { - setColumnsPopoverVisible(!isColumnsPopoverVisible); - } - - function closeColumnsPopover() { - setColumnsPopoverVisible(false); - } - - function toggleColumn(column: EsFieldName) { - if (tableItems.length > 0 && jobConfig !== undefined) { - // spread to a new array otherwise the component wouldn't re-render - setSelectedFields([ - ...toggleSelectedField(selectedFields, column, jobConfig.dest.results_field, depVarType), - ]); - } - } - const needsDestIndexFields = indexPattern && indexPattern.title === jobConfig.source.index[0]; - + const resultsField = jobConfig.dest.results_field; const { errorMessage, - loadExploreData, - sortField, - sortDirection, - status, - tableItems, - } = useExploreData( - jobConfig, - needsDestIndexFields, + fieldTypes, + pagination, + searchQuery, selectedFields, + rowCount, + setPagination, + setSearchQuery, setSelectedFields, - setDocFields, - setDepVarType - ); - - const columns: Array> = selectedFields - .sort(({ name: a }, { name: b }) => sortRegressionResultsFields(a, b, jobConfig)) - .map(field => { - const { type } = field; - let format: any; + setSortingColumns, + sortingColumns, + status, + tableFields, + tableItems, + } = useExploreData(jobConfig, needsDestIndexFields); - if (indexPattern !== undefined) { - format = mlFieldFormatService.getFieldFormatFromIndexPattern(indexPattern, field.id, ''); - } + useEffect(() => { + setEvaluateSearchQuery(searchQuery); + }, [JSON.stringify(searchQuery)]); + const columns = tableFields + .sort((a: any, b: any) => sortRegressionResultsFields(a, b, jobConfig)) + .map((field: any) => { + // Built-in values are ['boolean', 'currency', 'datetime', 'numeric', 'json'] + // To fall back to the default string schema it needs to be undefined. + let schema; + let isSortable = true; + const type = fieldTypes[field]; const isNumber = type !== undefined && (BASIC_NUMERICAL_TYPES.has(type) || EXTENDED_NUMERICAL_TYPES.has(type)); - const column: ColumnType = { - field: field.name, - name: field.name, - sortable: true, - truncateText: true, - }; - - const render = (d: any, fullItem: EsDoc) => { - if (format !== undefined) { - d = format.convert(d, 'text'); - return d; - } - - if (Array.isArray(d) && d.every(item => typeof item === 'string')) { - // If the cells data is an array of strings, return as a comma separated list. - // The list will get limited to 5 items with `…` at the end if there's more in the original array. - return `${d.slice(0, 5).join(', ')}${d.length > 5 ? ', …' : ''}`; - } else if (Array.isArray(d)) { - // If the cells data is an array of e.g. objects, display a 'array' badge with a - // tooltip that explains that this type of field is not supported in this table. - return ( - - - {i18n.translate( - 'xpack.ml.dataframe.analytics.regressionExploration.indexArrayBadgeContent', - { - defaultMessage: 'array', - } - )} - - - ); - } - - return d; - }; - if (isNumber) { - column.dataType = 'number'; - column.render = render; - } else if (typeof type !== 'undefined') { - switch (type) { - case ES_FIELD_TYPES.BOOLEAN: - column.dataType = ES_FIELD_TYPES.BOOLEAN; - column.render = d => (d ? 'true' : 'false'); - break; - case ES_FIELD_TYPES.DATE: - column.align = 'right'; - if (format !== undefined) { - column.render = render; - } else { - column.render = (d: any) => { - if (d !== undefined) { - return formatHumanReadableDateTimeSeconds(moment(d).unix() * 1000); - } - return d; - }; - } - break; - default: - column.render = render; - break; - } - } else { - column.render = render; + schema = 'numeric'; } - return column; - }); - - const docFieldsCount = docFields.length; - - useEffect(() => { - if ( - jobConfig !== undefined && - columns.length > 0 && - selectedFields.length > 0 && - sortField !== undefined && - sortDirection !== undefined && - selectedFields.some(field => field.name === sortField) - ) { - let field = sortField; - // If sorting by predictedField use dependentVar type - if (predictedFieldName === sortField) { - field = dependentVariable; + switch (type) { + case 'date': + schema = 'datetime'; + break; + case 'geo_point': + schema = 'json'; + break; + case 'boolean': + schema = 'boolean'; + break; } - const requiresKeyword = isKeywordAndTextType(field); - - loadExploreData({ - field: sortField, - direction: sortDirection, - searchQuery, - requiresKeyword, - }); - } - }, [JSON.stringify(searchQuery)]); - - useEffect(() => { - // By default set sorting to descending on the prediction field (`_prediction`). - // if that's not available sort ascending on the first column. Check if the current sorting field is still available. - if ( - jobConfig !== undefined && - columns.length > 0 && - selectedFields.length > 0 && - !selectedFields.some(field => field.name === sortField) - ) { - const predictedFieldSelected = selectedFields.some( - field => field.name === predictedFieldName - ); - - // CHECK IF keyword suffix is needed (if predicted field is selected we have to check the dependent variable type) - let sortByField = predictedFieldSelected ? dependentVariable : selectedFields[0].name; - - const requiresKeyword = isKeywordAndTextType(sortByField); - - sortByField = predictedFieldSelected ? predictedFieldName : sortByField; - - const direction = predictedFieldSelected ? SORT_DIRECTION.DESC : SORT_DIRECTION.ASC; - loadExploreData({ field: sortByField, direction, searchQuery, requiresKeyword }); - } - }, [ - jobConfig, - columns.length, - selectedFields.length, - sortField, - sortDirection, - tableItems.length, - ]); - - let sorting: SortingPropType = false; - let onTableChange; - - if (columns.length > 0 && sortField !== '' && sortField !== undefined) { - sorting = { - sort: { - field: sortField, - direction: sortDirection, - }, - }; - - onTableChange = ({ - page = { index: 0, size: 10 }, - sort = { field: sortField, direction: sortDirection }, - }: OnTableChangeArg) => { - const { index, size } = page; - setPageIndex(index); - setPageSize(size); - - if (sort.field !== sortField || sort.direction !== sortDirection) { - let field = sort.field; - // If sorting by predictedField use depVar for type check - if (predictedFieldName === sort.field) { - field = dependentVariable; - } - loadExploreData({ - ...sort, - searchQuery, - requiresKeyword: isKeywordAndTextType(field), - }); + if (field === `${resultsField}.feature_importance`) { + isSortable = false; } - }; - } - - const pagination = { - initialPageIndex: pageIndex, - initialPageSize: pageSize, - totalItemCount: tableItems.length, - pageSizeOptions: PAGE_SIZE_OPTIONS, - hidePerPageOptions: false, - }; - const onQueryChange = ({ query, error }: { query: QueryType; error: any }) => { - if (error) { - setSearchError(error.message); - } else { - try { - const esQueryDsl = Query.toESQuery(query); - setSearchQuery(esQueryDsl); - setSearchString(query.text); - setSearchError(undefined); - // set query for use in evaluate panel - setEvaluateSearchQuery(esQueryDsl); - } catch (e) { - setSearchError(e.toString()); - } - } - }; + return { id: field, schema, isSortable }; + }); - const search = { - onChange: onQueryChange, - defaultQuery: searchString, - box: { - incremental: false, - placeholder: i18n.translate( - 'xpack.ml.dataframe.analytics.regressionExploration.searchBoxPlaceholder', - { - defaultMessage: 'E.g. avg>0.5', - } - ), - }, - filters: [ - { - type: 'field_value_toggle_group', - field: `${jobConfig.dest.results_field}.is_training`, - items: [ - { - value: false, - name: i18n.translate( - 'xpack.ml.dataframe.analytics.regressionExploration.isTestingLabel', - { - defaultMessage: 'Testing', - } - ), - }, - { - value: true, - name: i18n.translate( - 'xpack.ml.dataframe.analytics.regressionExploration.isTrainingLabel', - { - defaultMessage: 'Training', - } - ), - }, - ], - }, - ], - }; + const docFieldsCount = tableFields.length; if (jobConfig === undefined) { return null; @@ -428,11 +154,6 @@ export const ResultsTable: FC = React.memo( ); } - const tableError = - status === INDEX_STATUS.ERROR && errorMessage.includes('parsing_exception') - ? errorMessage - : searchError; - return ( @@ -464,52 +185,6 @@ export const ResultsTable: FC = React.memo( )} - - - - } - isOpen={isColumnsPopoverVisible} - closePopover={closeColumnsPopover} - ownFocus - > - - {i18n.translate( - 'xpack.ml.dataframe.analytics.regressionExploration.selectFieldsPopoverTitle', - { - defaultMessage: 'Select fields', - } - )} - -
- {docFields.map(({ name }) => ( - field.name === name)} - onChange={() => toggleColumn(name)} - disabled={ - selectedFields.some(field => field.name === name) && - selectedFields.length === 1 - } - /> - ))} -
- - - @@ -518,29 +193,39 @@ export const ResultsTable: FC = React.memo( )} {(columns.length > 0 || searchQuery !== defaultSearchQuery) && ( - - - - - - - - + + + + + + + + + + + + + + + + )} ); diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/regression_exploration/use_explore_data.ts b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/regression_exploration/use_explore_data.ts index e158e952c1c18..c68fe5b2cbee8 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/regression_exploration/use_explore_data.ts +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/regression_exploration/use_explore_data.ts @@ -4,97 +4,156 @@ * you may not use this file except in compliance with the Elastic License. */ -import React, { useEffect, useState } from 'react'; +import { useEffect, useState, Dispatch, SetStateAction } from 'react'; +import { EuiDataGridPaginationProps, EuiDataGridSorting } from '@elastic/eui'; import { SearchResponse } from 'elasticsearch'; import { cloneDeep } from 'lodash'; -import { SortDirection, SORT_DIRECTION } from '../../../../../components/ml_in_memory_table'; - import { ml } from '../../../../../services/ml_api_service'; import { getNestedProperty } from '../../../../../util/object_utils'; import { newJobCapsService } from '../../../../../services/new_job_capabilities_service'; +import { SORT_DIRECTION } from '../../../../../components/ml_in_memory_table'; +import { SavedSearchQuery } from '../../../../../contexts/ml'; import { getDefaultFieldsFromJobCaps, + getDependentVar, getFlattenedFields, + getPredictedFieldName, DataFrameAnalyticsConfig, EsFieldName, INDEX_STATUS, - SEARCH_SIZE, - SearchQuery, } from '../../../../common'; -import { Field } from '../../../../../../../common/types/fields'; +import { Dictionary } from '../../../../../../../common/types/common'; +import { isKeywordAndTextType } from '../../../../common/fields'; import { ES_FIELD_TYPES } from '../../../../../../../../../../src/plugins/data/public'; import { - LoadExploreDataArg, + LoadRegressionExploreDataArg, defaultSearchQuery, ResultsSearchQuery, isResultsSearchBoolQuery, } from '../../../../common/analytics'; export type TableItem = Record; +type Pagination = Pick; export interface UseExploreDataReturnType { errorMessage: string; - loadExploreData: (arg: LoadExploreDataArg) => void; - sortField: EsFieldName; - sortDirection: SortDirection; + fieldTypes: { [key: string]: ES_FIELD_TYPES }; + pagination: Pagination; + rowCount: number; + searchQuery: SavedSearchQuery; + selectedFields: EsFieldName[]; + setFilterByIsTraining: Dispatch>; + setPagination: Dispatch>; + setSearchQuery: Dispatch>; + setSelectedFields: Dispatch>; + setSortingColumns: Dispatch>; + sortingColumns: EuiDataGridSorting['columns']; status: INDEX_STATUS; + tableFields: string[]; tableItems: TableItem[]; } +type EsSorting = Dictionary<{ + order: 'asc' | 'desc'; +}>; + +// The types specified in `@types/elasticsearch` are out of date and still have `total: number`. +interface SearchResponse7 extends SearchResponse { + hits: SearchResponse['hits'] & { + total: { + value: number; + relation: string; + }; + }; +} + export const useExploreData = ( - jobConfig: DataFrameAnalyticsConfig | undefined, - needsDestIndexFields: boolean, - selectedFields: Field[], - setSelectedFields: React.Dispatch>, - setDocFields: React.Dispatch>, - setDepVarType: React.Dispatch> + jobConfig: DataFrameAnalyticsConfig, + needsDestIndexFields: boolean ): UseExploreDataReturnType => { const [errorMessage, setErrorMessage] = useState(''); const [status, setStatus] = useState(INDEX_STATUS.UNUSED); + + const [selectedFields, setSelectedFields] = useState([] as EsFieldName[]); + const [tableFields, setTableFields] = useState([]); const [tableItems, setTableItems] = useState([]); - const [sortField, setSortField] = useState(''); - const [sortDirection, setSortDirection] = useState(SORT_DIRECTION.ASC); + const [fieldTypes, setFieldTypes] = useState<{ [key: string]: ES_FIELD_TYPES }>({}); + const [rowCount, setRowCount] = useState(0); + + const [pagination, setPagination] = useState({ pageIndex: 0, pageSize: 25 }); + const [searchQuery, setSearchQuery] = useState(defaultSearchQuery); + const [filterByIsTraining, setFilterByIsTraining] = useState(undefined); + const [sortingColumns, setSortingColumns] = useState([]); + + const predictedFieldName = getPredictedFieldName( + jobConfig.dest.results_field, + jobConfig.analysis + ); + const dependentVariable = getDependentVar(jobConfig.analysis); const getDefaultSelectedFields = () => { const { fields } = newJobCapsService; - if (selectedFields.length === 0 && jobConfig !== undefined) { - const { - selectedFields: defaultSelected, - docFields, - depVarType, - } = getDefaultFieldsFromJobCaps(fields, jobConfig, needsDestIndexFields); - - setDepVarType(depVarType); - setSelectedFields(defaultSelected); - setDocFields(docFields); + const { selectedFields: defaultSelected, docFields } = getDefaultFieldsFromJobCaps( + fields, + jobConfig, + needsDestIndexFields + ); + + const types: { [key: string]: ES_FIELD_TYPES } = {}; + const allFields: string[] = []; + + docFields.forEach(field => { + types[field.id] = field.type; + allFields.push(field.id); + }); + + setFieldTypes(types); + setSelectedFields(defaultSelected.map(field => field.id)); + setTableFields(allFields); } }; const loadExploreData = async ({ - field, - direction, - searchQuery, - requiresKeyword, - }: LoadExploreDataArg) => { + filterByIsTraining: isTraining, + searchQuery: incomingQuery, + }: LoadRegressionExploreDataArg) => { if (jobConfig !== undefined) { setErrorMessage(''); setStatus(INDEX_STATUS.LOADING); try { const resultsField = jobConfig.dest.results_field; - const searchQueryClone: ResultsSearchQuery = cloneDeep(searchQuery); + const searchQueryClone: ResultsSearchQuery = cloneDeep(incomingQuery); let query: ResultsSearchQuery; + const { pageIndex, pageSize } = pagination; + // If filterByIsTraining is defined - add that in to the final query + const trainingQuery = + isTraining !== undefined + ? { + term: { [`${resultsField}.is_training`]: { value: isTraining } }, + } + : undefined; - if (JSON.stringify(searchQuery) === JSON.stringify(defaultSearchQuery)) { - query = { + if (JSON.stringify(incomingQuery) === JSON.stringify(defaultSearchQuery)) { + const existsQuery = { exists: { field: resultsField, }, }; + + query = { + bool: { + must: [existsQuery], + }, + }; + + if (trainingQuery !== undefined && isResultsSearchBoolQuery(query)) { + query.bool.must.push(trainingQuery); + } } else if (isResultsSearchBoolQuery(searchQueryClone)) { if (searchQueryClone.bool.must === undefined) { searchQueryClone.bool.must = []; @@ -106,32 +165,37 @@ export const useExploreData = ( }, }); + if (trainingQuery !== undefined) { + searchQueryClone.bool.must.push(trainingQuery); + } + query = searchQueryClone; } else { query = searchQueryClone; } - const body: SearchQuery = { - query, - }; - - if (field !== undefined) { - body.sort = [ - { - [`${field}${requiresKeyword ? '.keyword' : ''}`]: { - order: direction, - }, - }, - ]; - } - const resp: SearchResponse = await ml.esSearch({ + const sort: EsSorting = sortingColumns + .map(column => { + const { id } = column; + column.id = isKeywordAndTextType(id) ? `${id}.keyword` : id; + return column; + }) + .reduce((s, column) => { + s[column.id] = { order: column.direction }; + return s; + }, {} as EsSorting); + + const resp: SearchResponse7 = await ml.esSearch({ index: jobConfig.dest.index, - size: SEARCH_SIZE, - body, + body: { + query, + from: pageIndex * pageSize, + size: pageSize, + ...(Object.keys(sort).length > 0 ? { sort } : {}), + }, }); - setSortField(field); - setSortDirection(direction); + setRowCount(resp.hits.total.value); const docs = resp.hits.hits; @@ -183,10 +247,45 @@ export const useExploreData = ( }; useEffect(() => { - if (jobConfig !== undefined) { - getDefaultSelectedFields(); - } + getDefaultSelectedFields(); }, [jobConfig && jobConfig.id]); - return { errorMessage, loadExploreData, sortField, sortDirection, status, tableItems }; + // By default set sorting to descending on the prediction field (`_prediction`). + useEffect(() => { + const sortByField = isKeywordAndTextType(dependentVariable) + ? `${predictedFieldName}.keyword` + : predictedFieldName; + const direction = SORT_DIRECTION.DESC; + + setSortingColumns([{ id: sortByField, direction }]); + }, [jobConfig && jobConfig.id]); + + useEffect(() => { + loadExploreData({ filterByIsTraining, searchQuery }); + }, [ + filterByIsTraining, + jobConfig && jobConfig.id, + pagination, + searchQuery, + selectedFields, + sortingColumns, + ]); + + return { + errorMessage, + fieldTypes, + pagination, + searchQuery, + selectedFields, + rowCount, + setFilterByIsTraining, + setPagination, + setSelectedFields, + setSortingColumns, + setSearchQuery, + sortingColumns, + status, + tableItems, + tableFields, + }; }; diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index 8e4b74c4c08fd..752902febe2a2 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -9577,8 +9577,6 @@ "xpack.ml.dataframe.analytics.regressionExploration.generalError": "データの読み込み中にエラーが発生しました。", "xpack.ml.dataframe.analytics.regressionExploration.generalizationDocsCount": "{docsCount, plural, one {# doc} other {# docs}}が評価されました", "xpack.ml.dataframe.analytics.regressionExploration.generalizationErrorTitle": "一般化エラー", - "xpack.ml.dataframe.analytics.regressionExploration.indexArrayBadgeContent": "配列", - "xpack.ml.dataframe.analytics.regressionExploration.indexArrayToolTipContent": "この配列ベースの列の完全なコンテンツは表示できません。", "xpack.ml.dataframe.analytics.regressionExploration.indexError": "インデックスデータの読み込み中にエラーが発生しました。", "xpack.ml.dataframe.analytics.regressionExploration.isTestingLabel": "テスト", "xpack.ml.dataframe.analytics.regressionExploration.isTrainingLabel": "トレーニング", diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index 4cf18fc287cc6..ff4fa8e81ea28 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -9580,8 +9580,6 @@ "xpack.ml.dataframe.analytics.regressionExploration.generalError": "加载数据时出错。", "xpack.ml.dataframe.analytics.regressionExploration.generalizationDocsCount": "{docsCount, plural, one {# 个文档} other {# 个文档}}已评估", "xpack.ml.dataframe.analytics.regressionExploration.generalizationErrorTitle": "泛化误差", - "xpack.ml.dataframe.analytics.regressionExploration.indexArrayBadgeContent": "数组", - "xpack.ml.dataframe.analytics.regressionExploration.indexArrayToolTipContent": "此基于数组的列的完整内容无法显示。", "xpack.ml.dataframe.analytics.regressionExploration.indexError": "加载索引数据时出错。", "xpack.ml.dataframe.analytics.regressionExploration.isTestingLabel": "测试", "xpack.ml.dataframe.analytics.regressionExploration.isTrainingLabel": "培训",