Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[ML] Fixing filtering of categorization fields #49184

Merged
merged 1 commit into from
Oct 24, 2019
Merged

[ML] Fixing filtering of categorization fields #49184

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 2 additions & 3 deletions x-pack/legacy/plugins/ml/public/jobs/new_job_new/common/job_creator/util/general.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,10 +77,9 @@ export function getRichDetectors(
});
}

export function createFieldOptions(fields: Field[], filterOverride?: (f: Field) => boolean) {
const filter = filterOverride || (f => f.id !== EVENT_RATE_FIELD_ID);
export function createFieldOptions(fields: Field[]) {
return fields
.filter(filter)
.filter(f => f.id !== EVENT_RATE_FIELD_ID)
.map(f => ({
label: f.name,
}))
Expand Down
4 changes: 2 additions & 2 deletions ...blic/jobs/new_job_new/pages/components/datafeed_step/components/time_field/time_field.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ import { Description } from './description';
export const TimeField: FC = () => {
const { jobCreator: jc, jobCreatorUpdate, jobCreatorUpdated } = useContext(JobCreatorContext);
const jobCreator = jc as AdvancedJobCreator;
const { fields } = newJobCapsService;
const { dateFields } = newJobCapsService;
const [timeFieldName, setTimeFieldName] = useState(jobCreator.timeFieldName);

useEffect(() => {
Expand All @@ -30,7 +30,7 @@ export const TimeField: FC = () => {
return (
<Description>
<TimeFieldSelect
fields={fields}
fields={dateFields}
changeHandler={setTimeFieldName}
selectedField={timeFieldName}
/>
Expand Down
8 changes: 2 additions & 6 deletions ...bs/new_job_new/pages/components/datafeed_step/components/time_field/time_field_select.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,7 @@
import React, { FC } from 'react';
import { EuiComboBox, EuiComboBoxOptionProps } from '@elastic/eui';

import { Field, EVENT_RATE_FIELD_ID } from '../../../../../../../../common/types/fields';
import { ES_FIELD_TYPES } from '../../../../../../../../../../../../src/plugins/data/public';
import { Field } from '../../../../../../../../common/types/fields';
import { createFieldOptions } from '../../../../../common/job_creator/util/general';

interface Props {
Expand All @@ -18,10 +17,7 @@ interface Props {
}

export const TimeFieldSelect: FC<Props> = ({ fields, changeHandler, selectedField }) => {
const options: EuiComboBoxOptionProps[] = createFieldOptions(
fields,
f => f.id !== EVENT_RATE_FIELD_ID && f.type === ES_FIELD_TYPES.DATE
);
const options: EuiComboBoxOptionProps[] = createFieldOptions(fields);

const selection: EuiComboBoxOptionProps[] = [];
if (selectedField !== null) {
Expand Down
4 changes: 2 additions & 2 deletions ...ages/components/pick_fields_step/components/categorization_field/categorization_field.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ import { Description } from './description';
export const CategorizationField: FC = () => {
const { jobCreator: jc, jobCreatorUpdate, jobCreatorUpdated } = useContext(JobCreatorContext);
const jobCreator = jc as MultiMetricJobCreator | PopulationJobCreator | AdvancedJobCreator;
const { fields } = newJobCapsService;
const { catFields } = newJobCapsService;
const [categorizationFieldName, setCategorizationFieldName] = useState(
jobCreator.categorizationFieldName
);
Expand All @@ -36,7 +36,7 @@ export const CategorizationField: FC = () => {
return (
<Description>
<CategorizationFieldSelect
fields={fields}
fields={catFields}
changeHandler={setCategorizationFieldName}
selectedField={categorizationFieldName}
/>
Expand Down
10 changes: 2 additions & 8 deletions ...mponents/pick_fields_step/components/categorization_field/categorization_field_select.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,7 @@ import React, { FC, useContext } from 'react';
import { EuiComboBox, EuiComboBoxOptionProps } from '@elastic/eui';

import { JobCreatorContext } from '../../../job_creator_context';
import { Field, EVENT_RATE_FIELD_ID } from '../../../../../../../../common/types/fields';
import { ES_FIELD_TYPES } from '../../../../../../../../../../../../src/plugins/data/public';
import { Field } from '../../../../../../../../common/types/fields';
import {
createFieldOptions,
createScriptFieldOptions,
Expand All @@ -24,12 +23,7 @@ interface Props {
export const CategorizationFieldSelect: FC<Props> = ({ fields, changeHandler, selectedField }) => {
const { jobCreator } = useContext(JobCreatorContext);
const options: EuiComboBoxOptionProps[] = [
...createFieldOptions(
fields,
f =>
f.id !== EVENT_RATE_FIELD_ID &&
(f.type === ES_FIELD_TYPES.KEYWORD || f.type === ES_FIELD_TYPES.TEXT)
),
...createFieldOptions(fields),
...createScriptFieldOptions(jobCreator.scriptFields),
];

Expand Down
51 changes: 42 additions & 9 deletions x-pack/legacy/plugins/ml/public/services/new_job_capabilities_service.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,8 @@ const categoryFieldTypes = [ES_FIELD_TYPES.TEXT, ES_FIELD_TYPES.KEYWORD, ES_FIEL

class NewJobCapsService {
private _fields: Field[] = [];
private _catFields: Field[] = [];
private _dateFields: Field[] = [];
private _aggs: Aggregation[] = [];
private _includeEventRateField: boolean = true;
private _removeTextFields: boolean = true;
Expand All @@ -59,6 +61,14 @@ class NewJobCapsService {
return this._fields;
}

public get catFields(): Field[] {
return this._catFields;
}

public get dateFields(): Field[] {
return this._dateFields;
}

public get aggs(): Aggregation[] {
return this._aggs;
}
Expand All @@ -84,16 +94,28 @@ class NewJobCapsService {
this._removeTextFields = removeTextFields;

const resp = await ml.jobs.newJobCaps(indexPattern.title, indexPattern.type === 'rollup');
const { fields, aggs } = createObjects(resp, indexPattern.title);
const { fields: allFields, aggs } = createObjects(resp, indexPattern.title);

if (this._includeEventRateField === true) {
addEventRateField(aggs, fields);
addEventRateField(aggs, allFields);
}

// remove any text fields which have a keyword equivalents
const processedFields = this._removeTextFields ? processTextFields(fields) : fields;

this._fields = processedFields;
const { fieldsPreferringKeyword, fieldsPreferringText } = processTextAndKeywordFields(
allFields
);
const catFields = fieldsPreferringText.filter(
f => f.type === ES_FIELD_TYPES.KEYWORD || f.type === ES_FIELD_TYPES.TEXT
);
const dateFields = fieldsPreferringText.filter(f => f.type === ES_FIELD_TYPES.DATE);
const fields = this._removeTextFields ? fieldsPreferringKeyword : allFields;

// set the main fields list to contain fields which have been filtered to prefer
// keyword fields over text fields.
// e.g. if foo.keyword and foo exist, don't add foo to the list.
this._fields = fields;
// set the category fields to contain fields which have been filtered to prefer text fields.
this._catFields = catFields;
this._dateFields = dateFields;
this._aggs = aggs;
} catch (error) {
console.error('Unable to load new job capabilities', error); // eslint-disable-line no-console
Expand Down Expand Up @@ -204,14 +226,25 @@ function addEventRateField(aggs: Aggregation[], fields: Field[]) {
fields.splice(0, 0, eventRateField);
}

// remove fields which are text and have a keyword equivalent
function processTextFields(fields: Field[]) {
// create two lists, one removing text fields if there are keyword equivalents and vice versa
function processTextAndKeywordFields(fields: Field[]) {
const keywordIds = fields.filter(f => f.type === ES_FIELD_TYPES.KEYWORD).map(f => f.id);
return fields.filter(
const textIds = fields.filter(f => f.type === ES_FIELD_TYPES.TEXT).map(f => f.id);

const fieldsPreferringKeyword = fields.filter(
f =>
f.type !== ES_FIELD_TYPES.TEXT ||
(f.type === ES_FIELD_TYPES.TEXT && keywordIds.includes(`${f.id}.keyword`) === false)
);

const fieldsPreferringText = fields.filter(
f =>
f.type !== ES_FIELD_TYPES.KEYWORD ||
(f.type === ES_FIELD_TYPES.KEYWORD &&
textIds.includes(f.id.replace(/\.keyword$/, '')) === false)
);

return { fieldsPreferringKeyword, fieldsPreferringText };
}

export const newJobCapsService = new NewJobCapsService();