[ML] Analytics: ensure both keyword/text types are excluded for selec…

…ted excluded field (elastic#62712) * exclude keyword and text types of field selected for exclusion * only show keyword type fields of accepted fields for depVar * make excludes field logic generic * fix regex to ensure escaped dot. reset regex and mainfield * ensure cloned jobs get correct excluded fields * add clarifying comments
nreese · Apr 8, 2020 · 3e4469c · 3e4469c
1 parent 9d89a4f
commit 3e4469c
Show file tree

Hide file tree

Showing 3 changed files with 53 additions and 3 deletions.
diff --git a/...ics/pages/analytics_management/components/create_analytics_form/create_analytics_form.tsx b/...ics/pages/analytics_management/components/create_analytics_form/create_analytics_form.tsx
@@ -250,7 +250,7 @@ export const CreateAnalyticsForm: FC<CreateAnalyticsFormProps> = ({ actions, sta
           dependentVariableOptions: [] as State['form']['dependentVariableOptions'],
         };
 
-        await newJobCapsService.initializeFromIndexPattern(indexPattern);
+        await newJobCapsService.initializeFromIndexPattern(indexPattern, false, false);
         // Get fields and filter for supported types for job type
         const { fields } = newJobCapsService;
 

diff --git a/...cs/pages/analytics_management/components/create_analytics_form/form_options_validation.ts b/...cs/pages/analytics_management/components/create_analytics_form/form_options_validation.ts
@@ -10,7 +10,7 @@ import { ANALYSIS_CONFIG_TYPE } from '../../../../common/analytics';
 import { AnalyticsJobType } from '../../hooks/use_create_analytics_form/state';
 import { BASIC_NUMERICAL_TYPES, EXTENDED_NUMERICAL_TYPES } from '../../../../common/fields';
 
-const CATEGORICAL_TYPES = new Set(['ip', 'keyword', 'text']);
+const CATEGORICAL_TYPES = new Set(['ip', 'keyword']);
 
 // List of system fields we want to ignore for the numeric field check.
 export const OMIT_FIELDS: string[] = ['_source', '_type', '_index', '_id', '_version', '_score'];

diff --git a/.../data_frame_analytics/pages/analytics_management/hooks/use_create_analytics_form/state.ts b/.../data_frame_analytics/pages/analytics_management/hooks/use_create_analytics_form/state.ts
@@ -8,6 +8,7 @@ import { EuiComboBoxOptionOption } from '@elastic/eui';
 import { DeepPartial, DeepReadonly } from '../../../../../../../common/types/common';
 import { checkPermission } from '../../../../../privilege/check_privilege';
 import { mlNodesAvailable } from '../../../../../ml_nodes_check';
+import { newJobCapsService } from '../../../../../services/new_job_capabilities_service';
 
 import {
   isClassificationAnalysis,
@@ -158,6 +159,55 @@ export const getInitialState = (): State => ({
   estimatedModelMemoryLimit: '',
 });
 
+const getExcludesFields = (excluded: string[]) => {
+  const { fields } = newJobCapsService;
+  const updatedExcluded: string[] = [];
+  // Loop through excluded fields to check for multiple types of same field
+  for (let i = 0; i < excluded.length; i++) {
+    const fieldName = excluded[i];
+    let mainField;
+
+    // No dot in fieldName - it is the main field
+    if (fieldName.includes('.') === false) {
+      mainField = fieldName;
+    } else {
+      // Dot in fieldName - check if there's a field whose name equals the fieldName with the last dot suffix removed
+      const regex = /\.[^.]*$/;
+      const suffixRemovedField = fieldName.replace(regex, '');
+      const fieldMatch = newJobCapsService.getFieldById(suffixRemovedField);
+
+      // There's a match - set as the main field
+      if (fieldMatch !== null) {
+        mainField = suffixRemovedField;
+      } else {
+        // No main field to be found - add the fieldName to updatedExcluded array if it's not already there
+        if (updatedExcluded.includes(fieldName) === false) {
+          updatedExcluded.push(fieldName);
+        }
+      }
+    }
+
+    if (mainField !== undefined) {
+      // Add the main field to the updatedExcluded array if it's not already there
+      if (updatedExcluded.includes(mainField) === false) {
+        updatedExcluded.push(mainField);
+      }
+      // Create regex to find all other fields whose names begin with main field followed by a dot
+      const regex = new RegExp(`${mainField}\\..+`);
+
+      // Loop through fields and add fields matching the pattern to updatedExcluded array
+      for (let j = 0; j < fields.length; j++) {
+        const field = fields[j].name;
+        if (updatedExcluded.includes(field) === false && field.match(regex) !== null) {
+          updatedExcluded.push(field);
+        }
+      }
+    }
+  }
+
+  return updatedExcluded;
+};
+
 export const getJobConfigFromFormState = (
   formState: State['form']
 ): DeepPartial<DataFrameAnalyticsConfig> => {
@@ -175,7 +225,7 @@ export const getJobConfigFromFormState = (
       index: formState.destinationIndex,
     },
     analyzed_fields: {
-      excludes: formState.excludes,
+      excludes: getExcludesFields(formState.excludes),
     },
     analysis: {
       outlier_detection: {},