Skip to content

Commit

Permalink
[ML] Fixing filtering of categorization fields (#49184)
Browse files Browse the repository at this point in the history
jgowdyelastic authored Oct 24, 2019

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature. The key has expired.
1 parent a859577 commit d9133c1
Showing 6 changed files with 52 additions and 30 deletions.
Original file line number Diff line number Diff line change
@@ -77,10 +77,9 @@ export function getRichDetectors(
});
}

export function createFieldOptions(fields: Field[], filterOverride?: (f: Field) => boolean) {
const filter = filterOverride || (f => f.id !== EVENT_RATE_FIELD_ID);
export function createFieldOptions(fields: Field[]) {
return fields
.filter(filter)
.filter(f => f.id !== EVENT_RATE_FIELD_ID)
.map(f => ({
label: f.name,
}))
Original file line number Diff line number Diff line change
@@ -15,7 +15,7 @@ import { Description } from './description';
export const TimeField: FC = () => {
const { jobCreator: jc, jobCreatorUpdate, jobCreatorUpdated } = useContext(JobCreatorContext);
const jobCreator = jc as AdvancedJobCreator;
const { fields } = newJobCapsService;
const { dateFields } = newJobCapsService;
const [timeFieldName, setTimeFieldName] = useState(jobCreator.timeFieldName);

useEffect(() => {
@@ -30,7 +30,7 @@ export const TimeField: FC = () => {
return (
<Description>
<TimeFieldSelect
fields={fields}
fields={dateFields}
changeHandler={setTimeFieldName}
selectedField={timeFieldName}
/>
Original file line number Diff line number Diff line change
@@ -7,8 +7,7 @@
import React, { FC } from 'react';
import { EuiComboBox, EuiComboBoxOptionProps } from '@elastic/eui';

import { Field, EVENT_RATE_FIELD_ID } from '../../../../../../../../common/types/fields';
import { ES_FIELD_TYPES } from '../../../../../../../../../../../../src/plugins/data/public';
import { Field } from '../../../../../../../../common/types/fields';
import { createFieldOptions } from '../../../../../common/job_creator/util/general';

interface Props {
@@ -18,10 +17,7 @@ interface Props {
}

export const TimeFieldSelect: FC<Props> = ({ fields, changeHandler, selectedField }) => {
const options: EuiComboBoxOptionProps[] = createFieldOptions(
fields,
f => f.id !== EVENT_RATE_FIELD_ID && f.type === ES_FIELD_TYPES.DATE
);
const options: EuiComboBoxOptionProps[] = createFieldOptions(fields);

const selection: EuiComboBoxOptionProps[] = [];
if (selectedField !== null) {
Original file line number Diff line number Diff line change
@@ -19,7 +19,7 @@ import { Description } from './description';
export const CategorizationField: FC = () => {
const { jobCreator: jc, jobCreatorUpdate, jobCreatorUpdated } = useContext(JobCreatorContext);
const jobCreator = jc as MultiMetricJobCreator | PopulationJobCreator | AdvancedJobCreator;
const { fields } = newJobCapsService;
const { catFields } = newJobCapsService;
const [categorizationFieldName, setCategorizationFieldName] = useState(
jobCreator.categorizationFieldName
);
@@ -36,7 +36,7 @@ export const CategorizationField: FC = () => {
return (
<Description>
<CategorizationFieldSelect
fields={fields}
fields={catFields}
changeHandler={setCategorizationFieldName}
selectedField={categorizationFieldName}
/>
Original file line number Diff line number Diff line change
@@ -8,8 +8,7 @@ import React, { FC, useContext } from 'react';
import { EuiComboBox, EuiComboBoxOptionProps } from '@elastic/eui';

import { JobCreatorContext } from '../../../job_creator_context';
import { Field, EVENT_RATE_FIELD_ID } from '../../../../../../../../common/types/fields';
import { ES_FIELD_TYPES } from '../../../../../../../../../../../../src/plugins/data/public';
import { Field } from '../../../../../../../../common/types/fields';
import {
createFieldOptions,
createScriptFieldOptions,
@@ -24,12 +23,7 @@ interface Props {
export const CategorizationFieldSelect: FC<Props> = ({ fields, changeHandler, selectedField }) => {
const { jobCreator } = useContext(JobCreatorContext);
const options: EuiComboBoxOptionProps[] = [
...createFieldOptions(
fields,
f =>
f.id !== EVENT_RATE_FIELD_ID &&
(f.type === ES_FIELD_TYPES.KEYWORD || f.type === ES_FIELD_TYPES.TEXT)
),
...createFieldOptions(fields),
...createScriptFieldOptions(jobCreator.scriptFields),
];

Original file line number Diff line number Diff line change
@@ -51,6 +51,8 @@ const categoryFieldTypes = [ES_FIELD_TYPES.TEXT, ES_FIELD_TYPES.KEYWORD, ES_FIEL

class NewJobCapsService {
private _fields: Field[] = [];
private _catFields: Field[] = [];
private _dateFields: Field[] = [];
private _aggs: Aggregation[] = [];
private _includeEventRateField: boolean = true;
private _removeTextFields: boolean = true;
@@ -59,6 +61,14 @@ class NewJobCapsService {
return this._fields;
}

public get catFields(): Field[] {
return this._catFields;
}

public get dateFields(): Field[] {
return this._dateFields;
}

public get aggs(): Aggregation[] {
return this._aggs;
}
@@ -84,16 +94,28 @@ class NewJobCapsService {
this._removeTextFields = removeTextFields;

const resp = await ml.jobs.newJobCaps(indexPattern.title, indexPattern.type === 'rollup');
const { fields, aggs } = createObjects(resp, indexPattern.title);
const { fields: allFields, aggs } = createObjects(resp, indexPattern.title);

if (this._includeEventRateField === true) {
addEventRateField(aggs, fields);
addEventRateField(aggs, allFields);
}

// remove any text fields which have a keyword equivalents
const processedFields = this._removeTextFields ? processTextFields(fields) : fields;

this._fields = processedFields;
const { fieldsPreferringKeyword, fieldsPreferringText } = processTextAndKeywordFields(
allFields
);
const catFields = fieldsPreferringText.filter(
f => f.type === ES_FIELD_TYPES.KEYWORD || f.type === ES_FIELD_TYPES.TEXT
);
const dateFields = fieldsPreferringText.filter(f => f.type === ES_FIELD_TYPES.DATE);
const fields = this._removeTextFields ? fieldsPreferringKeyword : allFields;

// set the main fields list to contain fields which have been filtered to prefer
// keyword fields over text fields.
// e.g. if foo.keyword and foo exist, don't add foo to the list.
this._fields = fields;
// set the category fields to contain fields which have been filtered to prefer text fields.
this._catFields = catFields;
this._dateFields = dateFields;
this._aggs = aggs;
} catch (error) {
console.error('Unable to load new job capabilities', error); // eslint-disable-line no-console
@@ -204,14 +226,25 @@ function addEventRateField(aggs: Aggregation[], fields: Field[]) {
fields.splice(0, 0, eventRateField);
}

// remove fields which are text and have a keyword equivalent
function processTextFields(fields: Field[]) {
// create two lists, one removing text fields if there are keyword equivalents and vice versa
function processTextAndKeywordFields(fields: Field[]) {
const keywordIds = fields.filter(f => f.type === ES_FIELD_TYPES.KEYWORD).map(f => f.id);
return fields.filter(
const textIds = fields.filter(f => f.type === ES_FIELD_TYPES.TEXT).map(f => f.id);

const fieldsPreferringKeyword = fields.filter(
f =>
f.type !== ES_FIELD_TYPES.TEXT ||
(f.type === ES_FIELD_TYPES.TEXT && keywordIds.includes(`${f.id}.keyword`) === false)
);

const fieldsPreferringText = fields.filter(
f =>
f.type !== ES_FIELD_TYPES.KEYWORD ||
(f.type === ES_FIELD_TYPES.KEYWORD &&
textIds.includes(f.id.replace(/\.keyword$/, '')) === false)
);

return { fieldsPreferringKeyword, fieldsPreferringText };
}

export const newJobCapsService = new NewJobCapsService();

0 comments on commit d9133c1

Please sign in to comment.