[ML] Regression dependent variable must be numeric

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/DataFrameAnalysis.java

@@ -8,8 +8,8 @@
 import org.elasticsearch.common.io.stream.NamedWriteable;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 
+import java.util.List;
 import java.util.Map;
-import java.util.Set;
 
 public interface DataFrameAnalysis extends ToXContentObject, NamedWriteable {
 
@@ -24,9 +24,9 @@ public interface DataFrameAnalysis extends ToXContentObject, NamedWriteable {
     boolean supportsCategoricalFields();
 
     /**
-     * @return The set of fields that analyzed documents must have for the analysis to operate
+     * @return The names and types of the fields that analyzed documents must have for the analysis to operate
      */
-    Set<String> getRequiredFields();
+    List<RequiredField> getRequiredFields();
 
     /**
      * @return {@code true} if this analysis supports data frame rows with missing values

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/OutlierDetection.java

@@ -18,10 +18,10 @@
 import java.io.IOException;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
-import java.util.Set;
 
 public class OutlierDetection implements DataFrameAnalysis {
 
@@ -160,8 +160,8 @@ public boolean supportsCategoricalFields() {
     }
 
     @Override
-    public Set<String> getRequiredFields() {
-        return Collections.emptySet();
+    public List<RequiredField> getRequiredFields() {
+        return Collections.emptyList();
     }
 
     @Override

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/Regression.java

@@ -17,9 +17,10 @@
 import java.io.IOException;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Objects;
-import java.util.Set;
+import java.util.TreeSet;
 
 public class Regression implements DataFrameAnalysis {
 
@@ -201,8 +202,8 @@ public boolean supportsCategoricalFields() {
     }
 
     @Override
-    public Set<String> getRequiredFields() {
-        return Collections.singleton(dependentVariable);
+    public List<RequiredField> getRequiredFields() {
+        return Collections.singletonList(new RequiredField(dependentVariable, new TreeSet<>(Types.numerical())));
     }
 
     @Override

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/RequiredField.java

@@ -0,0 +1,31 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.dataframe.analyses;
+
+import java.util.Collections;
+import java.util.Objects;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+public class RequiredField {
+
+    private final String name;
+    private final SortedSet<String> types;
+
+    public RequiredField(String name, Set<String> types) {
+        this.name = Objects.requireNonNull(name);
+        this.types = Collections.unmodifiableSortedSet(new TreeSet<>(types));
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public SortedSet<String> getTypes() {
+        return types;
+    }
+}

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/Types.java

@@ -0,0 +1,43 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.dataframe.analyses;
+
+import org.elasticsearch.index.mapper.NumberFieldMapper;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+/**
+ * Helper class that defines groups of types
+ */
+public final class Types {
+
+    private Types() {}
+
+    private static final Set<String> CATEGORICAL_TYPES = Collections.unmodifiableSet(new HashSet<>(Arrays.asList("text", "keyword", "ip")));
+
+    private static final Set<String> NUMERICAL_TYPES;
+
+    static {
+        Set<String> numericalTypes = Stream.of(NumberFieldMapper.NumberType.values())
+            .map(NumberFieldMapper.NumberType::typeName)
+            .collect(Collectors.toSet());
+        numericalTypes.add("scaled_float");
+        NUMERICAL_TYPES = Collections.unmodifiableSet(numericalTypes);
+    }
+
+    public static Set<String> categorical() {
+        return CATEGORICAL_TYPES;
+    }
+
+    public static Set<String> numerical() {
+        return NUMERICAL_TYPES;
+    }
+}

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractor.java

@@ -23,6 +23,7 @@
 import org.elasticsearch.search.fetch.StoredFieldsContext;
 import org.elasticsearch.search.sort.SortOrder;
 import org.elasticsearch.xpack.core.ClientHelper;
+import org.elasticsearch.xpack.core.ml.dataframe.analyses.Types;
 import org.elasticsearch.xpack.ml.datafeed.extractor.fields.ExtractedField;
 import org.elasticsearch.xpack.ml.dataframe.DataFrameAnalyticsIndex;
 
@@ -268,7 +269,7 @@ public Set<String> getCategoricalFields() {
         Set<String> categoricalFields = new HashSet<>();
         for (ExtractedField extractedField : context.extractedFields.getAllFields()) {
             String fieldName = extractedField.getName();
-            if (ExtractedFieldsDetector.CATEGORICAL_TYPES.containsAll(extractedField.getTypes())) {
+            if (Types.categorical().containsAll(extractedField.getTypes())) {
                 categoricalFields.add(fieldName);
             }
         }

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetector.java

@@ -15,11 +15,12 @@
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.mapper.BooleanFieldMapper;
-import org.elasticsearch.index.mapper.NumberFieldMapper;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
 import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig;
 import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsDest;
+import org.elasticsearch.xpack.core.ml.dataframe.analyses.RequiredField;
+import org.elasticsearch.xpack.core.ml.dataframe.analyses.Types;
 import org.elasticsearch.xpack.core.ml.job.messages.Messages;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
 import org.elasticsearch.xpack.core.ml.utils.NameResolver;
@@ -38,7 +39,6 @@
 import java.util.Set;
 import java.util.TreeSet;
 import java.util.stream.Collectors;
-import java.util.stream.Stream;
 
 public class ExtractedFieldsDetector {
 
@@ -50,18 +50,6 @@ public class ExtractedFieldsDetector {
     private static final List<String> IGNORE_FIELDS = Arrays.asList("_id", "_field_names", "_index", "_parent", "_routing", "_seq_no",
         "_source", "_type", "_uid", "_version", "_feature", "_ignored", DataFrameAnalyticsIndex.ID_COPY);
 
-    public static final Set<String> CATEGORICAL_TYPES = Collections.unmodifiableSet(new HashSet<>(Arrays.asList("text", "keyword", "ip")));
-
-    private static final Set<String> NUMERICAL_TYPES;
-
-    static {
-        Set<String> numericalTypes = Stream.of(NumberFieldMapper.NumberType.values())
-            .map(NumberFieldMapper.NumberType::typeName)
-            .collect(Collectors.toSet());
-        numericalTypes.add("scaled_float");
-        NUMERICAL_TYPES = Collections.unmodifiableSet(numericalTypes);
-    }
-
     private final String[] index;
     private final DataFrameAnalyticsConfig config;
     private final String resultsField;
@@ -85,7 +73,7 @@ public ExtractedFields detect() {
         removeFieldsUnderResultsField(fields);
         includeAndExcludeFields(fields);
         removeFieldsWithIncompatibleTypes(fields);
-        checkRequiredFieldsArePresent(fields);
+        checkRequiredFields();
 
         if (fields.isEmpty()) {
             throw ExceptionsHelper.badRequestException("No compatible fields could be detected in index {}. Supported types are {}.",
@@ -145,9 +133,9 @@ private void removeFieldsWithIncompatibleTypes(Set<String> fields) {
                 fieldsIterator.remove();
             } else {
                 Set<String> fieldTypes = fieldCaps.keySet();
-                if (NUMERICAL_TYPES.containsAll(fieldTypes)) {
+                if (Types.numerical().containsAll(fieldTypes)) {
                     LOGGER.debug("[{}] field [{}] is compatible as it is numerical", config.getId(), field);
-                } else if (config.getAnalysis().supportsCategoricalFields() && CATEGORICAL_TYPES.containsAll(fieldTypes)) {
+                } else if (config.getAnalysis().supportsCategoricalFields() && Types.categorical().containsAll(fieldTypes)) {
                     LOGGER.debug("[{}] field [{}] is compatible as it is categorical", config.getId(), field);
                 } else if (isBoolean(fieldTypes)) {
                     LOGGER.debug("[{}] field [{}] is compatible as it is boolean", config.getId(), field);
@@ -161,9 +149,9 @@ private void removeFieldsWithIncompatibleTypes(Set<String> fields) {
     }
 
     private Set<String> getSupportedTypes() {
-        Set<String> supportedTypes = new TreeSet<>(NUMERICAL_TYPES);
+        Set<String> supportedTypes = new TreeSet<>(Types.numerical());
         if (config.getAnalysis().supportsCategoricalFields()) {
-            supportedTypes.addAll(CATEGORICAL_TYPES);
+            supportedTypes.addAll(Types.categorical());
         }
         supportedTypes.add(BooleanFieldMapper.CONTENT_TYPE);
         return supportedTypes;
@@ -202,13 +190,20 @@ private void includeAndExcludeFields(Set<String> fields) {
         }
     }
 
-    private void checkRequiredFieldsArePresent(Set<String> fields) {
-        List<String> missingFields = config.getAnalysis().getRequiredFields()
-            .stream()
-            .filter(f -> fields.contains(f) == false)
-            .collect(Collectors.toList());
-        if (missingFields.isEmpty() == false) {
-            throw ExceptionsHelper.badRequestException("required fields {} are missing", missingFields);
+    private void checkRequiredFields() {
+        List<RequiredField> requiredFields = config.getAnalysis().getRequiredFields();
+        for (RequiredField requiredField : requiredFields) {
+            Map<String, FieldCapabilities> fieldCaps = fieldCapabilitiesResponse.getField(requiredField.getName());
+            if (fieldCaps == null || fieldCaps.isEmpty()) {
+                List<String> requiredFieldNames = requiredFields.stream().map(RequiredField::getName).collect(Collectors.toList());
+                throw ExceptionsHelper.badRequestException("required field [{}] is missing; analysis requires fields {}",
+                    requiredField.getName(), requiredFieldNames);
+            }
+            Set<String> fieldTypes = fieldCaps.keySet();
+            if (requiredField.getTypes().containsAll(fieldTypes) == false) {
+                throw ExceptionsHelper.badRequestException("invalid types {} for required field [{}]; expected types are {}",
+                    fieldTypes, requiredField.getName(), requiredField.getTypes());
+            }
         }
     }
 

x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetectorTests.java

@@ -120,7 +120,7 @@ public void testDetect_GivenRegressionAndMultipleFields() {
             .addAggregatableField("some_long", "long")
             .addAggregatableField("some_keyword", "keyword")
             .addAggregatableField("some_boolean", "boolean")
-            .addAggregatableField("foo", "keyword")
+            .addAggregatableField("foo", "double")
             .build();
 
         ExtractedFieldsDetector extractedFieldsDetector = new ExtractedFieldsDetector(
@@ -146,7 +146,23 @@ public void testDetect_GivenRegressionAndRequiredFieldMissing() {
             SOURCE_INDEX, buildRegressionConfig("foo"), RESULTS_FIELD, false, 100, fieldCapabilities);
         ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class, () -> extractedFieldsDetector.detect());
 
-        assertThat(e.getMessage(), equalTo("required fields [foo] are missing"));
+        assertThat(e.getMessage(), equalTo("required field [foo] is missing; analysis requires fields [foo]"));
+    }
+
+    public void testDetect_GivenRegressionAndRequiredFieldHasInvalidType() {
+        FieldCapabilitiesResponse fieldCapabilities = new MockFieldCapsResponseBuilder()
+            .addAggregatableField("some_float", "float")
+            .addAggregatableField("some_long", "long")
+            .addAggregatableField("some_keyword", "keyword")
+            .addAggregatableField("foo", "keyword")
+            .build();
+
+        ExtractedFieldsDetector extractedFieldsDetector = new ExtractedFieldsDetector(
+            SOURCE_INDEX, buildRegressionConfig("foo"), RESULTS_FIELD, false, 100, fieldCapabilities);
+        ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class, () -> extractedFieldsDetector.detect());
+
+        assertThat(e.getMessage(), equalTo("invalid types [keyword] for required field [foo]; " +
+            "expected types are [byte, double, float, half_float, integer, long, scaled_float, short]"));
     }
 
     public void testDetect_GivenIgnoredField() {