[APM] Use model_plot as a signal for anomaly scores (elastic#77756)

spalger · Sep 21, 2020 · ce7922b · ce7922b
1 parent ec1f4e5
commit ce7922b
Show file tree

Hide file tree

Showing 4 changed files with 547 additions and 28 deletions.
diff --git a/x-pack/plugins/apm/server/lib/service_map/get_service_anomalies.ts b/x-pack/plugins/apm/server/lib/service_map/get_service_anomalies.ts
@@ -60,11 +60,16 @@ export async function getServiceAnomalies({
       query: {
         bool: {
           filter: [
-            { term: { result_type: 'record' } },
+            { terms: { result_type: ['model_plot', 'record'] } },
             { terms: { job_id: mlJobIds } },
             {
               range: {
-                timestamp: { gte: start, lte: end, format: 'epoch_millis' },
+                timestamp: {
+                  // fetch data for at least 30 minutes
+                  gte: Math.min(end - 30 * 60 * 1000, start),
+                  lte: end,
+                  format: 'epoch_millis',
+                },
               },
             },
             {
@@ -83,7 +88,13 @@ export async function getServiceAnomalies({
             top_score: {
               top_hits: {
                 sort: { record_score: 'desc' },
-                _source: { includes: ['actual', 'job_id', 'by_field_value'] },
+                _source: [
+                  'actual',
+                  'job_id',
+                  'by_field_value',
+                  'result_type',
+                  'record_score',
+                ],
                 size: 1,
               },
             },
@@ -113,10 +124,19 @@ interface ServiceAnomaliesAggResponse {
             hits: Array<{
               sort: [number];
               _source: {
-                actual: [number];
                 job_id: string;
                 by_field_value: string;
-              };
+              } & (
+                | {
+                    record_score: number | null;
+                    result_type: 'record';
+                    actual: number[];
+                  }
+                | {
+                    result_type: 'model_plot';
+                    actual?: number;
+                  }
+              );
             }>;
           };
         };
@@ -127,29 +147,36 @@ interface ServiceAnomaliesAggResponse {
 
 function transformResponseToServiceAnomalies(
   response: ServiceAnomaliesAggResponse
-): Record<string, ServiceAnomalyStats> {
+) {
   const serviceAnomaliesMap = (
     response.aggregations?.services.buckets ?? []
-  ).reduce(
+  ).reduce<Record<string, ServiceAnomalyStats>>(
     (statsByServiceName, { key: serviceName, top_score: topScoreAgg }) => {
-      const anomalyScore = topScoreAgg.hits.hits[0]?.sort?.[0];
+      const mlResult = topScoreAgg.hits.hits[0]._source;
+
+      const anomalyScore =
+        (mlResult.result_type === 'record' && mlResult.record_score) || 0;
 
       const severity = getSeverity(anomalyScore);
       const healthStatus = getServiceHealthStatus({ severity });
 
       return {
         ...statsByServiceName,
         [serviceName]: {
-          transactionType: topScoreAgg.hits.hits[0]?._source?.by_field_value,
+          transactionType: mlResult.by_field_value,
+          jobId: mlResult.job_id,
+          actualValue:
+            mlResult.result_type === 'record'
+              ? mlResult.actual[0]
+              : mlResult.actual,
           anomalyScore,
-          actualValue: topScoreAgg.hits.hits[0]?._source?.actual?.[0],
-          jobId: topScoreAgg.hits.hits[0]?._source?.job_id,
           healthStatus,
         },
       };
     },
     {}
   );
+
   return serviceAnomaliesMap;
 }