Skip to content

Commit

Permalink
[ML] DF Analytics Regression exploration: replace table with data grid (
Browse files Browse the repository at this point in the history
#63650)

* add feature_importance column correctly

* wip: switch regression table to datagrid

* add search bar to regression view

* ensure feature importance fields show up correctly

* wip: filter by training/testing

* remove separate testing/training filter

* make error more clear

* handle lucene queries

* remove unnecessary comment

* ensure boolean shows up correctly.no sorting by feature importance

* remove unused translations
  • Loading branch information
alvarezmelissa87 authored Apr 16, 2020
1 parent 50228df commit dde3d96
Show file tree
Hide file tree
Showing 9 changed files with 549 additions and 602 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@ import { ml } from '../../services/ml_api_service';
import { Dictionary } from '../../../../common/types/common';
import { getErrorMessage } from '../../../../common/util/errors';
import { SavedSearchQuery } from '../../contexts/ml';
import { SortDirection } from '../../components/ml_in_memory_table';

export type IndexName = string;
export type IndexPattern = string;
Expand Down Expand Up @@ -53,13 +52,9 @@ export interface ClassificationAnalysis {
classification: Classification;
}

export interface LoadExploreDataArg {
field: string;
direction: SortDirection;
export interface LoadRegressionExploreDataArg {
filterByIsTraining?: boolean;
searchQuery: SavedSearchQuery;
requiresKeyword?: boolean;
pageIndex?: number;
pageSize?: number;
}

export const SEARCH_SIZE = 1000;
Expand Down Expand Up @@ -272,6 +267,11 @@ export const isResultsSearchBoolQuery = (arg: any): arg is ResultsSearchBoolQuer
return keys.length === 1 && keys[0] === 'bool';
};

export const isQueryStringQuery = (arg: any): arg is QueryStringQuery => {
const keys = Object.keys(arg);
return keys.length === 1 && keys[0] === 'query_string';
};

export const isRegressionEvaluateResponse = (arg: any): arg is RegressionEvaluateResponse => {
const keys = Object.keys(arg);
return (
Expand Down Expand Up @@ -396,6 +396,10 @@ interface ResultsSearchTermQuery {
term: Dictionary<any>;
}

interface QueryStringQuery {
query_string: Dictionary<any>;
}

export type ResultsSearchQuery = ResultsSearchBoolQuery | ResultsSearchTermQuery | SavedSearchQuery;

export function getEvalQueryBody({
Expand All @@ -409,16 +413,34 @@ export function getEvalQueryBody({
searchQuery?: ResultsSearchQuery;
ignoreDefaultQuery?: boolean;
}) {
let query: ResultsSearchQuery = {
let query;

const trainingQuery: ResultsSearchQuery = {
term: { [`${resultsField}.is_training`]: { value: isTraining } },
};

if (searchQuery !== undefined && ignoreDefaultQuery === true) {
query = searchQuery;
} else if (searchQuery !== undefined && isResultsSearchBoolQuery(searchQuery)) {
const searchQueryClone = cloneDeep(searchQuery);
searchQueryClone.bool.must.push(query);
const searchQueryClone = cloneDeep(searchQuery);

if (isResultsSearchBoolQuery(searchQueryClone)) {
if (searchQueryClone.bool.must === undefined) {
searchQueryClone.bool.must = [];
}

searchQueryClone.bool.must.push(trainingQuery);
query = searchQueryClone;
} else if (isQueryStringQuery(searchQueryClone)) {
query = {
bool: {
must: [searchQueryClone, trainingQuery],
},
};
} else {
// Not a bool or string query so we need to create it so can add the trainingQuery
query = {
bool: {
must: [trainingQuery],
},
};
}
return query;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -246,8 +246,15 @@ export const getDefaultFieldsFromJobCaps = (
fields: Field[],
jobConfig: DataFrameAnalyticsConfig,
needsDestIndexFields: boolean
): { selectedFields: Field[]; docFields: Field[]; depVarType?: ES_FIELD_TYPES } => {
const fieldsObj = { selectedFields: [], docFields: [] };
): {
selectedFields: Field[];
docFields: Field[];
depVarType?: ES_FIELD_TYPES;
} => {
const fieldsObj = {
selectedFields: [],
docFields: [],
};
if (fields.length === 0) {
return fieldsObj;
}
Expand All @@ -267,38 +274,37 @@ export const getDefaultFieldsFromJobCaps = (
const featureImportanceFields = [];

if ((numTopFeatureImportanceValues ?? 0) > 0) {
featureImportanceFields.push(
...fields.map(d => ({
id: `${resultsField}.feature_importance.${d.id}`,
name: `${resultsField}.feature_importance.${d.name}`,
type: KBN_FIELD_TYPES.NUMBER,
}))
);
featureImportanceFields.push({
id: `${resultsField}.feature_importance`,
name: `${resultsField}.feature_importance`,
type: KBN_FIELD_TYPES.NUMBER,
});
}

let allFields: any = [];
// Only need to add these fields if we didn't use dest index pattern to get the fields
const allFields: any =
needsDestIndexFields === true
? [
{
id: `${resultsField}.is_training`,
name: `${resultsField}.is_training`,
type: ES_FIELD_TYPES.BOOLEAN,
},
{ id: predictedField, name: predictedField, type },
...featureImportanceFields,
]
: [];

allFields.push(...fields);
if (needsDestIndexFields === true) {
allFields.push(
{
id: `${resultsField}.is_training`,
name: `${resultsField}.is_training`,
type: ES_FIELD_TYPES.BOOLEAN,
},
{ id: predictedField, name: predictedField, type }
);
}

allFields.push(...fields, ...featureImportanceFields);
allFields.sort(({ name: a }: { name: string }, { name: b }: { name: string }) =>
sortRegressionResultsFields(a, b, jobConfig)
);
// Remove feature_importance fields provided by dest index since feature_importance is an array the path is not valid
if (needsDestIndexFields === false) {
allFields = allFields.filter((field: any) => !field.name.includes('.feature_importance.'));
}

let selectedFields = allFields.filter(
(field: any) =>
field.name === predictedField ||
(!field.name.includes('.keyword') && !field.name.includes('.feature_importance.'))
(field: any) => field.name === predictedField || !field.name.includes('.keyword')
);

if (selectedFields.length > DEFAULT_REGRESSION_COLUMNS) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@ import { newJobCapsService } from '../../../../../services/new_job_capabilities_
import { Field } from '../../../../../../../common/types/fields';
import { ES_FIELD_TYPES } from '../../../../../../../../../../src/plugins/data/public';
import {
LoadExploreDataArg,
defaultSearchQuery,
ResultsSearchQuery,
isResultsSearchBoolQuery,
Expand All @@ -37,12 +36,23 @@ import {
SEARCH_SIZE,
SearchQuery,
} from '../../../../common';
import { SavedSearchQuery } from '../../../../../contexts/ml';

interface LoadClassificationExploreDataArg {
direction: SortDirection;
filterByIsTraining?: boolean;
field: string;
searchQuery: SavedSearchQuery;
requiresKeyword?: boolean;
pageIndex?: number;
pageSize?: number;
}

export type TableItem = Record<string, any>;

export interface UseExploreDataReturnType {
errorMessage: string;
loadExploreData: (arg: LoadExploreDataArg) => void;
loadExploreData: (arg: LoadClassificationExploreDataArg) => void;
sortField: EsFieldName;
sortDirection: SortDirection;
status: INDEX_STATUS;
Expand Down Expand Up @@ -84,7 +94,7 @@ export const useExploreData = (
direction,
searchQuery,
requiresKeyword,
}: LoadExploreDataArg) => {
}: LoadClassificationExploreDataArg) => {
if (jobConfig !== undefined) {
setErrorMessage('');
setStatus(INDEX_STATUS.LOADING);
Expand Down
Loading

0 comments on commit dde3d96

Please sign in to comment.