[ResponseOps] add parallel number arrays to task manager metric histo…

…grams (#176881)

towards: elastic/response-ops-team#151

## Summary

As we've found the elasticsearch histogram mapping type to be difficult
to use in practice, we're switching to an "array of numbers" approach -
keeping the histograms for now. We'll eventually remove the histograms,
once we ensure they aren't being used.

x-pack/plugins/task_manager/server/metrics/lib/simple_histogram.test.ts

@@ -28,6 +28,7 @@ describe('SimpleHistogram', () => {
       { value: 90, count: 0 },
       { value: 100, count: 0 },
     ]);
+    expect(histogram.getAllValues()).toEqual([]);
   });
 
   test('should correctly initialize when bucketSize does not evenly divides range', () => {
@@ -49,6 +50,7 @@ describe('SimpleHistogram', () => {
       { value: 98, count: 0 },
       { value: 105, count: 0 },
     ]);
+    expect(histogram.getAllValues()).toEqual([]);
   });
 
   test('should correctly record values', () => {
@@ -77,6 +79,7 @@ describe('SimpleHistogram', () => {
       { value: 90, count: 0 },
       { value: 100, count: 1 },
     ]);
+    expect(histogram.getAllValues()).toEqual([0, 10, 23, 34, 21, 56, 78, 33, 99, 1, 2]);
   });
 
   test('should correctly record values with specific increment', () => {
@@ -104,6 +107,9 @@ describe('SimpleHistogram', () => {
       { value: 90, count: 0 },
       { value: 100, count: 5 },
     ]);
+    expect(histogram.getAllValues()).toEqual([
+      0, 23, 23, 34, 21, 56, 78, 33, 33, 33, 33, 99, 99, 99, 99, 99, 1, 2,
+    ]);
   });
 
   test('should ignore values less than 0 and greater than max', () => {
@@ -119,11 +125,13 @@ describe('SimpleHistogram', () => {
     histogram.record(2);
 
     const hist1 = histogram.get();
+    const hist1AllValues = histogram.getAllValues();
 
     histogram.record(-1);
     histogram.record(200);
 
     expect(histogram.get()).toEqual(hist1);
+    expect(histogram.getAllValues()).toEqual(hist1AllValues);
   });
 
   test('should correctly reset values', () => {
@@ -150,6 +158,7 @@ describe('SimpleHistogram', () => {
       { value: 90, count: 0 },
       { value: 100, count: 1 },
     ]);
+    expect(histogram.getAllValues()).toEqual([23, 34, 21, 56, 78, 33, 99, 1, 2]);
 
     histogram.reset();
 
@@ -165,6 +174,7 @@ describe('SimpleHistogram', () => {
       { value: 90, count: 0 },
       { value: 100, count: 0 },
     ]);
+    expect(histogram.getAllValues()).toEqual([]);
   });
 
   test('should correctly truncate zero values', () => {
@@ -189,6 +199,7 @@ describe('SimpleHistogram', () => {
       { value: 90, count: 0 },
       { value: 100, count: 0 },
     ]);
+    expect(histogram.getAllValues()).toEqual([23, 34, 21, 56, 33, 1, 2]);
 
     expect(histogram.get(true)).toEqual([
       { value: 10, count: 2 },
@@ -204,6 +215,7 @@ describe('SimpleHistogram', () => {
     const histogram = new SimpleHistogram(100, 10);
 
     expect(histogram.get(true)).toEqual([]);
+    expect(histogram.getAllValues()).toEqual([]);
   });
 
   test('should correctly serialize histogram data', () => {

x-pack/plugins/task_manager/server/metrics/lib/simple_histogram.ts

@@ -23,6 +23,7 @@ export class SimpleHistogram {
   private maxValue: number;
   private bucketSize: number;
   private histogramBuckets: Bucket[] = [];
+  private allValues: number[] = [];
 
   constructor(max: number, bucketSize: number) {
     if (bucketSize > max) {
@@ -38,6 +39,7 @@ export class SimpleHistogram {
     for (let i = 0; i < this.histogramBuckets.length; i++) {
       this.histogramBuckets[i].count = 0;
     }
+    this.allValues = [];
   }
 
   public record(value: number, increment: number = 1) {
@@ -52,6 +54,10 @@ export class SimpleHistogram {
         break;
       }
     }
+
+    for (let i = 0; i < increment; i++) {
+      this.allValues.push(value);
+    }
   }
 
   public get(truncate: boolean = false) {
@@ -74,6 +80,10 @@ export class SimpleHistogram {
     }));
   }
 
+  public getAllValues(truncate: boolean = false) {
+    return this.allValues.slice();
+  }
+
   public serialize(): SerializedHistogram {
     const counts: number[] = [];
     const values: number[] = [];

x-pack/plugins/task_manager/server/metrics/task_claim_metrics_aggregator.test.ts

@@ -47,6 +47,7 @@ describe('TaskClaimMetricsAggregator', () => {
       success: 0,
       total: 0,
       duration: { counts: [], values: [] },
+      duration_values: [],
     });
   });
 
@@ -55,6 +56,7 @@ describe('TaskClaimMetricsAggregator', () => {
       success: 0,
       total: 0,
       duration: { counts: [], values: [] },
+      duration_values: [],
     });
   });
 
@@ -65,6 +67,7 @@ describe('TaskClaimMetricsAggregator', () => {
       success: 2,
       total: 2,
       duration: { counts: [2], values: [100] },
+      duration_values: [10, 10],
     });
   });
 
@@ -75,6 +78,7 @@ describe('TaskClaimMetricsAggregator', () => {
       success: 0,
       total: 2,
       duration: { counts: [], values: [] },
+      duration_values: [],
     });
   });
 
@@ -90,13 +94,15 @@ describe('TaskClaimMetricsAggregator', () => {
       success: 4,
       total: 7,
       duration: { counts: [4], values: [100] },
+      duration_values: [10, 10, 10, 10],
     });
 
     taskClaimMetricsAggregator.reset();
     expect(taskClaimMetricsAggregator.collect()).toEqual({
       success: 0,
       total: 0,
       duration: { counts: [], values: [] },
+      duration_values: [],
     });
   });
 });

x-pack/plugins/task_manager/server/metrics/task_claim_metrics_aggregator.ts

@@ -26,6 +26,7 @@ interface TaskClaimCounts extends JsonObject {
 
 export type TaskClaimMetric = TaskClaimCounts & {
   duration: SerializedHistogram;
+  duration_values: number[];
 };
 
 export class TaskClaimMetricsAggregator implements ITaskMetricsAggregator<TaskClaimMetric> {
@@ -38,12 +39,14 @@ export class TaskClaimMetricsAggregator implements ITaskMetricsAggregator<TaskCl
     return {
       ...this.counter.initialMetrics(),
       duration: { counts: [], values: [] },
+      duration_values: [],
     };
   }
   public collect(): TaskClaimMetric {
     return {
       ...this.counter.collect(),
       duration: this.durationHistogram.serialize(),
+      duration_values: this.durationHistogram.getAllValues(),
     };
   }
 

x-pack/plugins/task_manager/server/metrics/task_overdue_metrics_aggregator.test.ts

@@ -22,14 +22,20 @@ describe('TaskOverdueMetricsAggregator', () => {
 
   test('should correctly initialize', () => {
     expect(taskOverdueMetricsAggregator.collect()).toEqual({
-      overall: { overdue_by: { counts: [], values: [] } },
+      overall: {
+        overdue_by: { counts: [], values: [] },
+        overdue_by_values: [],
+      },
       by_type: {},
     });
   });
 
   test('should correctly return initialMetrics', () => {
     expect(taskOverdueMetricsAggregator.initialMetric()).toEqual({
-      overall: { overdue_by: { counts: [], values: [] } },
+      overall: {
+        overdue_by: { counts: [], values: [] },
+        overdue_by_values: [],
+      },
       by_type: {},
     });
   });
@@ -50,9 +56,15 @@ describe('TaskOverdueMetricsAggregator', () => {
       })
     );
     expect(taskOverdueMetricsAggregator.collect()).toEqual({
-      overall: { overdue_by: { counts: [1, 0, 1], values: [10, 20, 30] } },
+      overall: {
+        overdue_by: { counts: [1, 0, 1], values: [10, 20, 30] },
+        overdue_by_values: [0, 20],
+      },
       by_type: {
-        telemetry: { overdue_by: { counts: [1, 0, 1], values: [10, 20, 30] } },
+        telemetry: {
+          overdue_by: { counts: [1, 0, 1], values: [10, 20, 30] },
+          overdue_by_values: [0, 20],
+        },
       },
     });
   });
@@ -66,7 +78,10 @@ describe('TaskOverdueMetricsAggregator', () => {
       })
     );
     expect(taskOverdueMetricsAggregator.collect()).toEqual({
-      overall: { overdue_by: { counts: [], values: [] } },
+      overall: {
+        overdue_by: { counts: [], values: [] },
+        overdue_by_values: [],
+      },
       by_type: {},
     });
   });
@@ -95,9 +110,15 @@ describe('TaskOverdueMetricsAggregator', () => {
       })
     );
     expect(taskOverdueMetricsAggregator.collect()).toEqual({
-      overall: { overdue_by: { counts: [0, 0, 0, 0, 1], values: [10, 20, 30, 40, 50] } },
+      overall: {
+        overdue_by: { counts: [0, 0, 0, 0, 1], values: [10, 20, 30, 40, 50] },
+        overdue_by_values: [40],
+      },
       by_type: {
-        telemetry: { overdue_by: { counts: [0, 0, 0, 0, 1], values: [10, 20, 30, 40, 50] } },
+        telemetry: {
+          overdue_by: { counts: [0, 0, 0, 0, 1], values: [10, 20, 30, 40, 50] },
+          overdue_by_values: [40],
+        },
       },
     });
   });
@@ -129,43 +150,50 @@ describe('TaskOverdueMetricsAggregator', () => {
           counts: [3, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1],
           values: [10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120, 130],
         },
+        overdue_by_values: [0, 0, 0, 20, 20, 40, 120],
       },
       by_type: {
         'alerting:example': {
           overdue_by: {
             counts: [0, 0, 0, 0, 1],
             values: [10, 20, 30, 40, 50],
           },
+          overdue_by_values: [40],
         },
         'alerting:__index-threshold': {
           overdue_by: {
             counts: [0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1],
             values: [10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120, 130],
           },
+          overdue_by_values: [20, 20, 120],
         },
         alerting: {
           overdue_by: {
             counts: [0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1],
             values: [10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120, 130],
           },
+          overdue_by_values: [40, 20, 20, 120],
         },
         'actions:webhook': {
           overdue_by: {
             counts: [2],
             values: [10],
           },
+          overdue_by_values: [0, 0],
         },
         'actions:__email': {
           overdue_by: {
             counts: [1],
             values: [10],
           },
+          overdue_by_values: [0],
         },
         actions: {
           overdue_by: {
             counts: [3],
             values: [10],
           },
+          overdue_by_values: [0, 0, 0],
         },
       },
     });
@@ -198,44 +226,50 @@ describe('TaskOverdueMetricsAggregator', () => {
           counts: [3, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1],
           values: [10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120, 130],
         },
+        overdue_by_values: [0, 0, 0, 20, 20, 40, 120],
       },
       by_type: {
         'alerting:example': {
           overdue_by: {
             counts: [0, 0, 0, 0, 1],
-
             values: [10, 20, 30, 40, 50],
           },
+          overdue_by_values: [40],
         },
         'alerting:__index-threshold': {
           overdue_by: {
             counts: [0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1],
             values: [10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120, 130],
           },
+          overdue_by_values: [20, 20, 120],
         },
         alerting: {
           overdue_by: {
             counts: [0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1],
             values: [10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120, 130],
           },
+          overdue_by_values: [40, 20, 20, 120],
         },
         'actions:webhook': {
           overdue_by: {
             counts: [2],
             values: [10],
           },
+          overdue_by_values: [0, 0],
         },
         'actions:__email': {
           overdue_by: {
             counts: [1],
             values: [10],
           },
+          overdue_by_values: [0],
         },
         actions: {
           overdue_by: {
             counts: [3],
             values: [10],
           },
+          overdue_by_values: [0, 0, 0],
         },
       },
     });