[7.x][ML] Change outlier detection feature influence format to nested…

… object (elastic#1475) This changes the format of `feature_influence` for outlier detection so that the feature name is not part of the field name. This helps reduce field explosion in the results index. Feature influence is now an array with nested objects. Each of them contains the `feature_name` and the `influence` value. Backport of elastic#1475
dimitris-athanasiou · Sep 8, 2020 · 951b050 · 951b050
1 parent a053675
commit 951b050
Show file tree

Hide file tree

Showing 3 changed files with 24 additions and 7 deletions.
diff --git a/docs/CHANGELOG.asciidoc b/docs/CHANGELOG.asciidoc
@@ -33,6 +33,7 @@
 === Enhancements
 
 * Calculate total feature importance to store with model metadata. (See {ml-pull}1387[#1387].)
+* Change outlier detection feature_influence format to array with nested objects. (See {ml-pull}1475[#1475], {es-pull}62068[#62068].)
 
 === Bug Fixes
 

diff --git a/lib/api/CDataFrameOutliersRunner.cc b/lib/api/CDataFrameOutliersRunner.cc
@@ -52,7 +52,9 @@ const CDataFrameAnalysisConfigReader& parameterReader() {
 
 // Output
 const std::string OUTLIER_SCORE_FIELD_NAME{"outlier_score"};
-const std::string FEATURE_INFLUENCE_FIELD_NAME_PREFIX{"feature_influence."};
+const std::string FEATURE_NAME_FIELD_NAME{"feature_name"};
+const std::string FEATURE_INFLUENCE_FIELD_NAME{"feature_influence"};
+const std::string INFLUENCE_FIELD_NAME{"influence"};
 }
 
 CDataFrameOutliersRunner::CDataFrameOutliersRunner(const CDataFrameAnalysisSpecification& spec,
@@ -93,11 +95,19 @@ void CDataFrameOutliersRunner::writeOneRow(const core::CDataFrame& frame,
     writer.StartObject();
     writer.Key(OUTLIER_SCORE_FIELD_NAME);
     writer.Double(row[scoreColumn]);
-    if (row[scoreColumn] > m_FeatureInfluenceThreshold) {
+    if (row[scoreColumn] > m_FeatureInfluenceThreshold && numberFeatureScoreColumns > 0) {
+        writer.Key(FEATURE_INFLUENCE_FIELD_NAME);
+        writer.StartArray();
+
         for (std::size_t i = 0; i < numberFeatureScoreColumns; ++i) {
-            writer.Key(FEATURE_INFLUENCE_FIELD_NAME_PREFIX + frame.columnNames()[i]);
+            writer.StartObject();
+            writer.Key(FEATURE_NAME_FIELD_NAME);
+            writer.String(frame.columnNames()[i]);
+            writer.Key(INFLUENCE_FIELD_NAME);
             writer.Double(row[beginFeatureScoreColumns + i]);
+            writer.EndObject();
         }
+        writer.EndArray();
     }
     writer.EndObject();
 }

diff --git a/lib/api/unittest/CDataFrameAnalyzerOutlierTest.cc b/lib/api/unittest/CDataFrameAnalyzerOutlierTest.cc
@@ -285,8 +285,7 @@ BOOST_AUTO_TEST_CASE(testRunOutlierFeatureInfluences) {
 
     TDoubleVec expectedScores;
     TDoubleVecVec expectedFeatureInfluences;
-    TStrVec expectedNames{"feature_influence.c1", "feature_influence.c2", "feature_influence.c3",
-                          "feature_influence.c4", "feature_influence.c5"};
+    TStrVec expectedNames{"c1", "c2", "c3", "c4", "c5"};
 
     TStrVec fieldNames{"c1", "c2", "c3", "c4", "c5", ".", "."};
     TStrVec fieldValues{"", "", "", "", "", "0", ""};
@@ -301,12 +300,19 @@ BOOST_AUTO_TEST_CASE(testRunOutlierFeatureInfluences) {
     auto expectedFeatureInfluence = expectedFeatureInfluences.begin();
     for (const auto& result : results.GetArray()) {
         if (result.HasMember("row_results")) {
+
             BOOST_TEST_REQUIRE(expectedFeatureInfluence !=
                                expectedFeatureInfluences.end());
-            for (std::size_t i = 0; i < 5; ++i) {
+            for (int i = 0; i < 5; ++i) {
+                BOOST_REQUIRE_EQUAL(
+                    expectedNames[i].c_str(),
+                    result["row_results"]["results"]["ml"]["feature_influence"][i]["feature_name"]
+                        .GetString());
+
                 BOOST_REQUIRE_CLOSE_ABSOLUTE(
                     (*expectedFeatureInfluence)[i],
-                    result["row_results"]["results"]["ml"][expectedNames[i]].GetDouble(),
+                    result["row_results"]["results"]["ml"]["feature_influence"][i]["influence"]
+                        .GetDouble(),
                     1e-4 * (*expectedFeatureInfluence)[i]);
             }
             ++expectedFeatureInfluence;