[ML] Fixing filtering of categorization fields (#49184) (#49206)

elastic · Oct 24, 2019 · 952b7c8 · 952b7c8
1 parent 97ee4c3
commit 952b7c8
Show file tree

Hide file tree

Showing 6 changed files with 52 additions and 30 deletions.
diff --git a/x-pack/legacy/plugins/ml/public/jobs/new_job_new/common/job_creator/util/general.ts b/x-pack/legacy/plugins/ml/public/jobs/new_job_new/common/job_creator/util/general.ts
@@ -77,10 +77,9 @@ export function getRichDetectors(
   });
 }
 
-export function createFieldOptions(fields: Field[], filterOverride?: (f: Field) => boolean) {
-  const filter = filterOverride || (f => f.id !== EVENT_RATE_FIELD_ID);
+export function createFieldOptions(fields: Field[]) {
   return fields
-    .filter(filter)
+    .filter(f => f.id !== EVENT_RATE_FIELD_ID)
     .map(f => ({
       label: f.name,
     }))

diff --git a/...blic/jobs/new_job_new/pages/components/datafeed_step/components/time_field/time_field.tsx b/...blic/jobs/new_job_new/pages/components/datafeed_step/components/time_field/time_field.tsx
@@ -15,7 +15,7 @@ import { Description } from './description';
 export const TimeField: FC = () => {
   const { jobCreator: jc, jobCreatorUpdate, jobCreatorUpdated } = useContext(JobCreatorContext);
   const jobCreator = jc as AdvancedJobCreator;
-  const { fields } = newJobCapsService;
+  const { dateFields } = newJobCapsService;
   const [timeFieldName, setTimeFieldName] = useState(jobCreator.timeFieldName);
 
   useEffect(() => {
@@ -30,7 +30,7 @@ export const TimeField: FC = () => {
   return (
     <Description>
       <TimeFieldSelect
-        fields={fields}
+        fields={dateFields}
         changeHandler={setTimeFieldName}
         selectedField={timeFieldName}
       />

diff --git a/...bs/new_job_new/pages/components/datafeed_step/components/time_field/time_field_select.tsx b/...bs/new_job_new/pages/components/datafeed_step/components/time_field/time_field_select.tsx
@@ -7,8 +7,7 @@
 import React, { FC } from 'react';
 import { EuiComboBox, EuiComboBoxOptionProps } from '@elastic/eui';
 
-import { Field, EVENT_RATE_FIELD_ID } from '../../../../../../../../common/types/fields';
-import { ES_FIELD_TYPES } from '../../../../../../../../../../../../src/plugins/data/public';
+import { Field } from '../../../../../../../../common/types/fields';
 import { createFieldOptions } from '../../../../../common/job_creator/util/general';
 
 interface Props {
@@ -18,10 +17,7 @@ interface Props {
 }
 
 export const TimeFieldSelect: FC<Props> = ({ fields, changeHandler, selectedField }) => {
-  const options: EuiComboBoxOptionProps[] = createFieldOptions(
-    fields,
-    f => f.id !== EVENT_RATE_FIELD_ID && f.type === ES_FIELD_TYPES.DATE
-  );
+  const options: EuiComboBoxOptionProps[] = createFieldOptions(fields);
 
   const selection: EuiComboBoxOptionProps[] = [
     {

diff --git a/...ages/components/pick_fields_step/components/categorization_field/categorization_field.tsx b/...ages/components/pick_fields_step/components/categorization_field/categorization_field.tsx
@@ -19,7 +19,7 @@ import { Description } from './description';
 export const CategorizationField: FC = () => {
   const { jobCreator: jc, jobCreatorUpdate, jobCreatorUpdated } = useContext(JobCreatorContext);
   const jobCreator = jc as MultiMetricJobCreator | PopulationJobCreator | AdvancedJobCreator;
-  const { fields } = newJobCapsService;
+  const { catFields } = newJobCapsService;
   const [categorizationFieldName, setCategorizationFieldName] = useState(
     jobCreator.categorizationFieldName
   );
@@ -36,7 +36,7 @@ export const CategorizationField: FC = () => {
   return (
     <Description>
       <CategorizationFieldSelect
-        fields={fields}
+        fields={catFields}
         changeHandler={setCategorizationFieldName}
         selectedField={categorizationFieldName}
       />

diff --git a/...mponents/pick_fields_step/components/categorization_field/categorization_field_select.tsx b/...mponents/pick_fields_step/components/categorization_field/categorization_field_select.tsx
@@ -8,8 +8,7 @@ import React, { FC, useContext } from 'react';
 import { EuiComboBox, EuiComboBoxOptionProps } from '@elastic/eui';
 
 import { JobCreatorContext } from '../../../job_creator_context';
-import { Field, EVENT_RATE_FIELD_ID } from '../../../../../../../../common/types/fields';
-import { ES_FIELD_TYPES } from '../../../../../../../../../../../../src/plugins/data/public';
+import { Field } from '../../../../../../../../common/types/fields';
 import {
   createFieldOptions,
   createScriptFieldOptions,
@@ -24,12 +23,7 @@ interface Props {
 export const CategorizationFieldSelect: FC<Props> = ({ fields, changeHandler, selectedField }) => {
   const { jobCreator } = useContext(JobCreatorContext);
   const options: EuiComboBoxOptionProps[] = [
-    ...createFieldOptions(
-      fields,
-      f =>
-        f.id !== EVENT_RATE_FIELD_ID &&
-        (f.type === ES_FIELD_TYPES.KEYWORD || f.type === ES_FIELD_TYPES.TEXT)
-    ),
+    ...createFieldOptions(fields),
     ...createScriptFieldOptions(jobCreator.scriptFields),
   ];
 

diff --git a/x-pack/legacy/plugins/ml/public/services/new_job_capabilities_service.ts b/x-pack/legacy/plugins/ml/public/services/new_job_capabilities_service.ts
@@ -51,6 +51,8 @@ const categoryFieldTypes = [ES_FIELD_TYPES.TEXT, ES_FIELD_TYPES.KEYWORD, ES_FIEL
 
 class NewJobCapsService {
   private _fields: Field[] = [];
+  private _catFields: Field[] = [];
+  private _dateFields: Field[] = [];
   private _aggs: Aggregation[] = [];
   private _includeEventRateField: boolean = true;
   private _removeTextFields: boolean = true;
@@ -59,6 +61,14 @@ class NewJobCapsService {
     return this._fields;
   }
 
+  public get catFields(): Field[] {
+    return this._catFields;
+  }
+
+  public get dateFields(): Field[] {
+    return this._dateFields;
+  }
+
   public get aggs(): Aggregation[] {
     return this._aggs;
   }
@@ -84,16 +94,28 @@ class NewJobCapsService {
       this._removeTextFields = removeTextFields;
 
       const resp = await ml.jobs.newJobCaps(indexPattern.title, indexPattern.type === 'rollup');
-      const { fields, aggs } = createObjects(resp, indexPattern.title);
+      const { fields: allFields, aggs } = createObjects(resp, indexPattern.title);
 
       if (this._includeEventRateField === true) {
-        addEventRateField(aggs, fields);
+        addEventRateField(aggs, allFields);
       }
 
-      // remove any text fields which have a keyword equivalents
-      const processedFields = this._removeTextFields ? processTextFields(fields) : fields;
-
-      this._fields = processedFields;
+      const { fieldsPreferringKeyword, fieldsPreferringText } = processTextAndKeywordFields(
+        allFields
+      );
+      const catFields = fieldsPreferringText.filter(
+        f => f.type === ES_FIELD_TYPES.KEYWORD || f.type === ES_FIELD_TYPES.TEXT
+      );
+      const dateFields = fieldsPreferringText.filter(f => f.type === ES_FIELD_TYPES.DATE);
+      const fields = this._removeTextFields ? fieldsPreferringKeyword : allFields;
+
+      // set the main fields list to contain fields which have been filtered to prefer
+      // keyword fields over text fields.
+      // e.g. if foo.keyword and foo exist, don't add foo to the list.
+      this._fields = fields;
+      // set the category fields to contain fields which have been filtered to prefer text fields.
+      this._catFields = catFields;
+      this._dateFields = dateFields;
       this._aggs = aggs;
     } catch (error) {
       console.error('Unable to load new job capabilities', error); // eslint-disable-line no-console
@@ -204,14 +226,25 @@ function addEventRateField(aggs: Aggregation[], fields: Field[]) {
   fields.splice(0, 0, eventRateField);
 }
 
-// remove fields which are text and have a keyword equivalent
-function processTextFields(fields: Field[]) {
+// create two lists, one removing text fields if there are keyword equivalents and vice versa
+function processTextAndKeywordFields(fields: Field[]) {
   const keywordIds = fields.filter(f => f.type === ES_FIELD_TYPES.KEYWORD).map(f => f.id);
-  return fields.filter(
+  const textIds = fields.filter(f => f.type === ES_FIELD_TYPES.TEXT).map(f => f.id);
+
+  const fieldsPreferringKeyword = fields.filter(
     f =>
       f.type !== ES_FIELD_TYPES.TEXT ||
       (f.type === ES_FIELD_TYPES.TEXT && keywordIds.includes(`${f.id}.keyword`) === false)
   );
+
+  const fieldsPreferringText = fields.filter(
+    f =>
+      f.type !== ES_FIELD_TYPES.KEYWORD ||
+      (f.type === ES_FIELD_TYPES.KEYWORD &&
+        textIds.includes(f.id.replace(/\.keyword$/, '')) === false)
+  );
+
+  return { fieldsPreferringKeyword, fieldsPreferringText };
 }
 
 export const newJobCapsService = new NewJobCapsService();