From b94d718ed72909d64eb4f34e0c86764bac588c9a Mon Sep 17 00:00:00 2001 From: Patrick Mueller Date: Mon, 1 Apr 2024 08:18:28 -0400 Subject: [PATCH] [ResponseOps] add parallel number arrays to task manager metric histograms (#176881) towards: https://github.com/elastic/response-ops-team/issues/151 ## Summary As we've found the elasticsearch histogram mapping type to be difficult to use in practice, we're switching to an "array of numbers" approach - keeping the histograms for now. We'll eventually remove the histograms, once we ensure they aren't being used. --- .../server/metrics/create_aggregator.test.ts | 311 ++++++++++++++++-- .../metrics/lib/simple_histogram.test.ts | 12 + .../server/metrics/lib/simple_histogram.ts | 10 + .../task_claim_metrics_aggregator.test.ts | 6 + .../metrics/task_claim_metrics_aggregator.ts | 3 + .../task_overdue_metrics_aggregator.test.ts | 50 ++- .../task_overdue_metrics_aggregator.ts | 32 +- .../task_run_metrics_aggregator.test.ts | 12 + .../metrics/task_run_metrics_aggregator.ts | 13 +- 9 files changed, 392 insertions(+), 57 deletions(-) diff --git a/x-pack/plugins/task_manager/server/metrics/create_aggregator.test.ts b/x-pack/plugins/task_manager/server/metrics/create_aggregator.test.ts index 196170f716d29..07323cb5b203d 100644 --- a/x-pack/plugins/task_manager/server/metrics/create_aggregator.test.ts +++ b/x-pack/plugins/task_manager/server/metrics/create_aggregator.test.ts @@ -117,47 +117,102 @@ describe('createAggregator', () => { .subscribe((metrics: Array>) => { expect(metrics[0]).toEqual({ key: 'task_claim', - value: { success: 1, total: 1, duration: { counts: [1], values: [100] } }, + value: { + success: 1, + total: 1, + duration: { counts: [1], values: [100] }, + duration_values: [10], + }, }); expect(metrics[1]).toEqual({ key: 'task_claim', - value: { success: 2, total: 2, duration: { counts: [2], values: [100] } }, + value: { + success: 2, + total: 2, + duration: { counts: [2], values: [100] }, + duration_values: [10, 10], + }, }); expect(metrics[2]).toEqual({ key: 'task_claim', - value: { success: 3, total: 3, duration: { counts: [3], values: [100] } }, + value: { + success: 3, + total: 3, + duration: { counts: [3], values: [100] }, + duration_values: [10, 10, 10], + }, }); expect(metrics[3]).toEqual({ key: 'task_claim', - value: { success: 4, total: 4, duration: { counts: [4], values: [100] } }, + value: { + success: 4, + total: 4, + duration: { counts: [4], values: [100] }, + duration_values: [10, 10, 10, 10], + }, }); expect(metrics[4]).toEqual({ key: 'task_claim', - value: { success: 4, total: 5, duration: { counts: [4], values: [100] } }, + value: { + success: 4, + total: 5, + duration: { counts: [4], values: [100] }, + duration_values: [10, 10, 10, 10], + }, }); expect(metrics[5]).toEqual({ key: 'task_claim', - value: { success: 5, total: 6, duration: { counts: [5], values: [100] } }, + value: { + success: 5, + total: 6, + duration: { counts: [5], values: [100] }, + duration_values: [10, 10, 10, 10, 10], + }, }); expect(metrics[6]).toEqual({ key: 'task_claim', - value: { success: 6, total: 7, duration: { counts: [6], values: [100] } }, + value: { + success: 6, + total: 7, + duration: { counts: [6], values: [100] }, + duration_values: [10, 10, 10, 10, 10, 10], + }, }); expect(metrics[7]).toEqual({ key: 'task_claim', - value: { success: 7, total: 8, duration: { counts: [7], values: [100] } }, + value: { + success: 7, + total: 8, + duration: { counts: [7], values: [100] }, + duration_values: [10, 10, 10, 10, 10, 10, 10], + }, }); expect(metrics[8]).toEqual({ key: 'task_claim', - value: { success: 8, total: 9, duration: { counts: [8], values: [100] } }, + value: { + success: 8, + total: 9, + duration: { counts: [8], values: [100] }, + duration_values: [10, 10, 10, 10, 10, 10, 10, 10], + }, }); expect(metrics[9]).toEqual({ key: 'task_claim', - value: { success: 8, total: 10, duration: { counts: [8], values: [100] } }, + value: { + success: 8, + total: 10, + duration: { counts: [8], values: [100] }, + duration_values: [10, 10, 10, 10, 10, 10, 10, 10], + }, }); expect(metrics[10]).toEqual({ key: 'task_claim', - value: { success: 9, total: 11, duration: { counts: [9], values: [100] } }, + value: { + success: 9, + total: 11, + duration: { counts: [9], values: [100] }, + duration_values: [10, 10, 10, 10, 10, 10, 10, 10, 10], + }, }); resolve(); }); @@ -209,48 +264,103 @@ describe('createAggregator', () => { .subscribe((metrics: Array>) => { expect(metrics[0]).toEqual({ key: 'task_claim', - value: { success: 1, total: 1, duration: { counts: [1], values: [100] } }, + value: { + success: 1, + total: 1, + duration: { counts: [1], values: [100] }, + duration_values: [10], + }, }); expect(metrics[1]).toEqual({ key: 'task_claim', - value: { success: 2, total: 2, duration: { counts: [2], values: [100] } }, + value: { + success: 2, + total: 2, + duration: { counts: [2], values: [100] }, + duration_values: [10, 10], + }, }); expect(metrics[2]).toEqual({ key: 'task_claim', - value: { success: 3, total: 3, duration: { counts: [3], values: [100] } }, + value: { + success: 3, + total: 3, + duration: { counts: [3], values: [100] }, + duration_values: [10, 10, 10], + }, }); expect(metrics[3]).toEqual({ key: 'task_claim', - value: { success: 4, total: 4, duration: { counts: [4], values: [100] } }, + value: { + success: 4, + total: 4, + duration: { counts: [4], values: [100] }, + duration_values: [10, 10, 10, 10], + }, }); expect(metrics[4]).toEqual({ key: 'task_claim', - value: { success: 4, total: 5, duration: { counts: [4], values: [100] } }, + value: { + success: 4, + total: 5, + duration: { counts: [4], values: [100] }, + duration_values: [10, 10, 10, 10], + }, }); expect(metrics[5]).toEqual({ key: 'task_claim', - value: { success: 5, total: 6, duration: { counts: [5], values: [100] } }, + value: { + success: 5, + total: 6, + duration: { counts: [5], values: [100] }, + duration_values: [10, 10, 10, 10, 10], + }, }); // reset event should have been received here expect(metrics[6]).toEqual({ key: 'task_claim', - value: { success: 1, total: 1, duration: { counts: [1], values: [100] } }, + value: { + success: 1, + total: 1, + duration: { counts: [1], values: [100] }, + duration_values: [10], + }, }); expect(metrics[7]).toEqual({ key: 'task_claim', - value: { success: 1, total: 2, duration: { counts: [1], values: [100] } }, + value: { + success: 1, + total: 2, + duration: { counts: [1], values: [100] }, + duration_values: [10], + }, }); expect(metrics[8]).toEqual({ key: 'task_claim', - value: { success: 1, total: 3, duration: { counts: [1], values: [100] } }, + value: { + success: 1, + total: 3, + duration: { counts: [1], values: [100] }, + duration_values: [10], + }, }); expect(metrics[9]).toEqual({ key: 'task_claim', - value: { success: 2, total: 4, duration: { counts: [2], values: [100] } }, + value: { + success: 2, + total: 4, + duration: { counts: [2], values: [100] }, + duration_values: [10, 10], + }, }); expect(metrics[10]).toEqual({ key: 'task_claim', - value: { success: 3, total: 5, duration: { counts: [3], values: [100] } }, + value: { + success: 3, + total: 5, + duration: { counts: [3], values: [100] }, + duration_values: [10, 10, 10], + }, }); resolve(); }); @@ -310,48 +420,103 @@ describe('createAggregator', () => { .subscribe((metrics: Array>) => { expect(metrics[0]).toEqual({ key: 'task_claim', - value: { success: 1, total: 1, duration: { counts: [1], values: [100] } }, + value: { + success: 1, + total: 1, + duration: { counts: [1], values: [100] }, + duration_values: [10], + }, }); expect(metrics[1]).toEqual({ key: 'task_claim', - value: { success: 2, total: 2, duration: { counts: [2], values: [100] } }, + value: { + success: 2, + total: 2, + duration: { counts: [2], values: [100] }, + duration_values: [10, 10], + }, }); expect(metrics[2]).toEqual({ key: 'task_claim', - value: { success: 3, total: 3, duration: { counts: [3], values: [100] } }, + value: { + success: 3, + total: 3, + duration: { counts: [3], values: [100] }, + duration_values: [10, 10, 10], + }, }); expect(metrics[3]).toEqual({ key: 'task_claim', - value: { success: 4, total: 4, duration: { counts: [4], values: [100] } }, + value: { + success: 4, + total: 4, + duration: { counts: [4], values: [100] }, + duration_values: [10, 10, 10, 10], + }, }); expect(metrics[4]).toEqual({ key: 'task_claim', - value: { success: 4, total: 5, duration: { counts: [4], values: [100] } }, + value: { + success: 4, + total: 5, + duration: { counts: [4], values: [100] }, + duration_values: [10, 10, 10, 10], + }, }); expect(metrics[5]).toEqual({ key: 'task_claim', - value: { success: 5, total: 6, duration: { counts: [5], values: [100] } }, + value: { + success: 5, + total: 6, + duration: { counts: [5], values: [100] }, + duration_values: [10, 10, 10, 10, 10], + }, }); // reset interval should have fired here expect(metrics[6]).toEqual({ key: 'task_claim', - value: { success: 1, total: 1, duration: { counts: [1], values: [100] } }, + value: { + success: 1, + total: 1, + duration: { counts: [1], values: [100] }, + duration_values: [10], + }, }); expect(metrics[7]).toEqual({ key: 'task_claim', - value: { success: 1, total: 2, duration: { counts: [1], values: [100] } }, + value: { + success: 1, + total: 2, + duration: { counts: [1], values: [100] }, + duration_values: [10], + }, }); expect(metrics[8]).toEqual({ key: 'task_claim', - value: { success: 1, total: 3, duration: { counts: [1], values: [100] } }, + value: { + success: 1, + total: 3, + duration: { counts: [1], values: [100] }, + duration_values: [10], + }, }); expect(metrics[9]).toEqual({ key: 'task_claim', - value: { success: 2, total: 4, duration: { counts: [2], values: [100] } }, + value: { + success: 2, + total: 4, + duration: { counts: [2], values: [100] }, + duration_values: [10, 10], + }, }); expect(metrics[10]).toEqual({ key: 'task_claim', - value: { success: 3, total: 5, duration: { counts: [3], values: [100] } }, + value: { + success: 3, + total: 5, + duration: { counts: [3], values: [100] }, + duration_values: [10, 10, 10], + }, }); resolve(); }); @@ -423,6 +588,7 @@ describe('createAggregator', () => { not_timed_out: 0, total: 0, delay: { counts: [1], values: [10] }, + delay_values: [3], framework_errors: 0, user_errors: 0, total_errors: 0, @@ -437,6 +603,7 @@ describe('createAggregator', () => { not_timed_out: 1, total: 1, delay: { counts: [1], values: [10] }, + delay_values: [3], framework_errors: 0, user_errors: 0, total_errors: 0, @@ -469,6 +636,7 @@ describe('createAggregator', () => { not_timed_out: 1, total: 1, delay: { counts: [1, 1], values: [10, 20] }, + delay_values: [3, 10], framework_errors: 0, user_errors: 0, total_errors: 0, @@ -501,6 +669,7 @@ describe('createAggregator', () => { not_timed_out: 2, total: 2, delay: { counts: [1, 1], values: [10, 20] }, + delay_values: [3, 10], framework_errors: 0, user_errors: 0, total_errors: 0, @@ -541,6 +710,7 @@ describe('createAggregator', () => { not_timed_out: 2, total: 2, delay: { counts: [2, 1], values: [10, 20] }, + delay_values: [3, 10, 3], framework_errors: 0, user_errors: 0, total_errors: 0, @@ -581,6 +751,7 @@ describe('createAggregator', () => { not_timed_out: 3, total: 3, delay: { counts: [2, 1], values: [10, 20] }, + delay_values: [3, 10, 3], framework_errors: 0, user_errors: 0, total_errors: 0, @@ -621,6 +792,7 @@ describe('createAggregator', () => { not_timed_out: 3, total: 3, delay: { counts: [2, 1, 0, 1], values: [10, 20, 30, 40] }, + delay_values: [3, 10, 3, 35], framework_errors: 0, user_errors: 0, total_errors: 0, @@ -661,6 +833,7 @@ describe('createAggregator', () => { not_timed_out: 4, total: 4, delay: { counts: [2, 1, 0, 1], values: [10, 20, 30, 40] }, + delay_values: [3, 10, 3, 35], framework_errors: 0, user_errors: 0, total_errors: 0, @@ -709,6 +882,7 @@ describe('createAggregator', () => { not_timed_out: 4, total: 4, delay: { counts: [3, 1, 0, 1], values: [10, 20, 30, 40] }, + delay_values: [3, 10, 3, 35, 9], framework_errors: 0, user_errors: 0, total_errors: 0, @@ -757,6 +931,7 @@ describe('createAggregator', () => { not_timed_out: 5, total: 5, delay: { counts: [3, 1, 0, 1], values: [10, 20, 30, 40] }, + delay_values: [3, 10, 3, 35, 9], framework_errors: 1, user_errors: 0, total_errors: 1, @@ -805,6 +980,7 @@ describe('createAggregator', () => { not_timed_out: 5, total: 5, delay: { counts: [4, 1, 0, 1], values: [10, 20, 30, 40] }, + delay_values: [3, 10, 3, 35, 9, 5], framework_errors: 1, user_errors: 0, total_errors: 1, @@ -853,6 +1029,7 @@ describe('createAggregator', () => { not_timed_out: 6, total: 6, delay: { counts: [4, 1, 0, 1], values: [10, 20, 30, 40] }, + delay_values: [3, 10, 3, 35, 9, 5], framework_errors: 1, user_errors: 0, total_errors: 1, @@ -909,6 +1086,7 @@ describe('createAggregator', () => { not_timed_out: 6, total: 6, delay: { counts: [4, 2, 0, 1], values: [10, 20, 30, 40] }, + delay_values: [3, 10, 3, 35, 9, 5, 12], framework_errors: 1, user_errors: 0, total_errors: 1, @@ -965,6 +1143,7 @@ describe('createAggregator', () => { not_timed_out: 7, total: 7, delay: { counts: [4, 2, 0, 1], values: [10, 20, 30, 40] }, + delay_values: [3, 10, 3, 35, 9, 5, 12], framework_errors: 1, user_errors: 0, total_errors: 1, @@ -1021,6 +1200,7 @@ describe('createAggregator', () => { not_timed_out: 7, total: 7, delay: { counts: [5, 2, 0, 1], values: [10, 20, 30, 40] }, + delay_values: [3, 10, 3, 35, 9, 5, 12, 4], framework_errors: 1, user_errors: 0, total_errors: 1, @@ -1077,6 +1257,7 @@ describe('createAggregator', () => { not_timed_out: 8, total: 8, delay: { counts: [5, 2, 0, 1], values: [10, 20, 30, 40] }, + delay_values: [3, 10, 3, 35, 9, 5, 12, 4], framework_errors: 2, user_errors: 0, total_errors: 2, @@ -1133,6 +1314,7 @@ describe('createAggregator', () => { not_timed_out: 8, total: 8, delay: { counts: [6, 2, 0, 1], values: [10, 20, 30, 40] }, + delay_values: [3, 10, 3, 35, 9, 5, 12, 4, 4], framework_errors: 2, user_errors: 0, total_errors: 2, @@ -1189,6 +1371,7 @@ describe('createAggregator', () => { not_timed_out: 9, total: 9, delay: { counts: [6, 2, 0, 1], values: [10, 20, 30, 40] }, + delay_values: [3, 10, 3, 35, 9, 5, 12, 4, 4], framework_errors: 2, user_errors: 0, total_errors: 2, @@ -1245,6 +1428,7 @@ describe('createAggregator', () => { not_timed_out: 9, total: 9, delay: { counts: [7, 2, 0, 1], values: [10, 20, 30, 40] }, + delay_values: [3, 10, 3, 35, 9, 5, 12, 4, 4, 3], framework_errors: 2, user_errors: 0, total_errors: 2, @@ -1301,6 +1485,7 @@ describe('createAggregator', () => { not_timed_out: 10, total: 10, delay: { counts: [7, 2, 0, 1], values: [10, 20, 30, 40] }, + delay_values: [3, 10, 3, 35, 9, 5, 12, 4, 4, 3], framework_errors: 3, user_errors: 0, total_errors: 3, @@ -1431,6 +1616,7 @@ describe('createAggregator', () => { not_timed_out: 0, total: 0, delay: { counts: [1], values: [10] }, + delay_values: [3], framework_errors: 0, user_errors: 0, total_errors: 0, @@ -1445,6 +1631,7 @@ describe('createAggregator', () => { not_timed_out: 1, total: 1, delay: { counts: [1], values: [10] }, + delay_values: [3], framework_errors: 0, user_errors: 0, total_errors: 0, @@ -1477,6 +1664,7 @@ describe('createAggregator', () => { not_timed_out: 1, total: 1, delay: { counts: [1, 1], values: [10, 20] }, + delay_values: [3, 10], framework_errors: 0, user_errors: 0, total_errors: 0, @@ -1509,6 +1697,7 @@ describe('createAggregator', () => { not_timed_out: 2, total: 2, delay: { counts: [1, 1], values: [10, 20] }, + delay_values: [3, 10], framework_errors: 0, user_errors: 0, total_errors: 0, @@ -1549,6 +1738,7 @@ describe('createAggregator', () => { not_timed_out: 2, total: 2, delay: { counts: [2, 1], values: [10, 20] }, + delay_values: [3, 10, 3], framework_errors: 0, user_errors: 0, total_errors: 0, @@ -1589,6 +1779,7 @@ describe('createAggregator', () => { not_timed_out: 3, total: 3, delay: { counts: [2, 1], values: [10, 20] }, + delay_values: [3, 10, 3], framework_errors: 0, user_errors: 0, total_errors: 0, @@ -1629,6 +1820,7 @@ describe('createAggregator', () => { not_timed_out: 3, total: 3, delay: { counts: [2, 1, 0, 1], values: [10, 20, 30, 40] }, + delay_values: [3, 10, 3, 35], framework_errors: 0, user_errors: 0, total_errors: 0, @@ -1669,6 +1861,7 @@ describe('createAggregator', () => { not_timed_out: 4, total: 4, delay: { counts: [2, 1, 0, 1], values: [10, 20, 30, 40] }, + delay_values: [3, 10, 3, 35], framework_errors: 0, user_errors: 0, total_errors: 0, @@ -1717,6 +1910,7 @@ describe('createAggregator', () => { not_timed_out: 4, total: 4, delay: { counts: [3, 1, 0, 1], values: [10, 20, 30, 40] }, + delay_values: [3, 10, 3, 35, 9], framework_errors: 0, user_errors: 0, total_errors: 0, @@ -1765,6 +1959,7 @@ describe('createAggregator', () => { not_timed_out: 5, total: 5, delay: { counts: [3, 1, 0, 1], values: [10, 20, 30, 40] }, + delay_values: [3, 10, 3, 35, 9], framework_errors: 1, user_errors: 0, total_errors: 1, @@ -1814,6 +2009,7 @@ describe('createAggregator', () => { not_timed_out: 0, total: 0, delay: { counts: [1], values: [10] }, + delay_values: [5], framework_errors: 0, user_errors: 0, total_errors: 0, @@ -1862,6 +2058,7 @@ describe('createAggregator', () => { not_timed_out: 1, total: 1, delay: { counts: [1], values: [10] }, + delay_values: [5], framework_errors: 0, user_errors: 0, total_errors: 0, @@ -1910,6 +2107,7 @@ describe('createAggregator', () => { not_timed_out: 1, total: 1, delay: { counts: [1, 1], values: [10, 20] }, + delay_values: [5, 12], framework_errors: 0, user_errors: 0, total_errors: 0, @@ -1958,6 +2156,7 @@ describe('createAggregator', () => { not_timed_out: 2, total: 2, delay: { counts: [1, 1], values: [10, 20] }, + delay_values: [5, 12], framework_errors: 0, user_errors: 0, total_errors: 0, @@ -2006,6 +2205,7 @@ describe('createAggregator', () => { not_timed_out: 2, total: 2, delay: { counts: [2, 1], values: [10, 20] }, + delay_values: [5, 12, 4], framework_errors: 0, user_errors: 0, total_errors: 0, @@ -2054,6 +2254,7 @@ describe('createAggregator', () => { not_timed_out: 3, total: 3, delay: { counts: [2, 1], values: [10, 20] }, + delay_values: [5, 12, 4], framework_errors: 1, user_errors: 0, total_errors: 1, @@ -2102,6 +2303,7 @@ describe('createAggregator', () => { not_timed_out: 3, total: 3, delay: { counts: [3, 1], values: [10, 20] }, + delay_values: [5, 12, 4, 4], framework_errors: 1, user_errors: 0, total_errors: 1, @@ -2150,6 +2352,7 @@ describe('createAggregator', () => { not_timed_out: 4, total: 4, delay: { counts: [3, 1], values: [10, 20] }, + delay_values: [5, 12, 4, 4], framework_errors: 1, user_errors: 0, total_errors: 1, @@ -2198,6 +2401,7 @@ describe('createAggregator', () => { not_timed_out: 4, total: 4, delay: { counts: [4, 1], values: [10, 20] }, + delay_values: [5, 12, 4, 4, 3], framework_errors: 1, user_errors: 0, total_errors: 1, @@ -2246,6 +2450,7 @@ describe('createAggregator', () => { not_timed_out: 5, total: 5, delay: { counts: [4, 1], values: [10, 20] }, + delay_values: [5, 12, 4, 4, 3], framework_errors: 2, user_errors: 0, total_errors: 2, @@ -2376,6 +2581,7 @@ describe('createAggregator', () => { not_timed_out: 0, total: 0, delay: { counts: [1], values: [10] }, + delay_values: [3], framework_errors: 0, user_errors: 0, total_errors: 0, @@ -2390,6 +2596,7 @@ describe('createAggregator', () => { not_timed_out: 1, total: 1, delay: { counts: [1], values: [10] }, + delay_values: [3], framework_errors: 0, user_errors: 0, total_errors: 0, @@ -2422,6 +2629,7 @@ describe('createAggregator', () => { not_timed_out: 1, total: 1, delay: { counts: [1, 1], values: [10, 20] }, + delay_values: [3, 10], framework_errors: 0, user_errors: 0, total_errors: 0, @@ -2454,6 +2662,7 @@ describe('createAggregator', () => { not_timed_out: 2, total: 2, delay: { counts: [1, 1], values: [10, 20] }, + delay_values: [3, 10], framework_errors: 0, user_errors: 0, total_errors: 0, @@ -2494,6 +2703,7 @@ describe('createAggregator', () => { not_timed_out: 2, total: 2, delay: { counts: [2, 1], values: [10, 20] }, + delay_values: [3, 10, 3], framework_errors: 0, user_errors: 0, total_errors: 0, @@ -2534,6 +2744,7 @@ describe('createAggregator', () => { not_timed_out: 3, total: 3, delay: { counts: [2, 1], values: [10, 20] }, + delay_values: [3, 10, 3], framework_errors: 0, user_errors: 0, total_errors: 0, @@ -2574,6 +2785,7 @@ describe('createAggregator', () => { not_timed_out: 3, total: 3, delay: { counts: [2, 1, 0, 1], values: [10, 20, 30, 40] }, + delay_values: [3, 10, 3, 35], framework_errors: 0, user_errors: 0, total_errors: 0, @@ -2614,6 +2826,7 @@ describe('createAggregator', () => { not_timed_out: 4, total: 4, delay: { counts: [2, 1, 0, 1], values: [10, 20, 30, 40] }, + delay_values: [3, 10, 3, 35], framework_errors: 0, user_errors: 0, total_errors: 0, @@ -2662,6 +2875,7 @@ describe('createAggregator', () => { not_timed_out: 4, total: 4, delay: { counts: [3, 1, 0, 1], values: [10, 20, 30, 40] }, + delay_values: [3, 10, 3, 35, 9], framework_errors: 0, user_errors: 0, total_errors: 0, @@ -2710,6 +2924,7 @@ describe('createAggregator', () => { not_timed_out: 5, total: 5, delay: { counts: [3, 1, 0, 1], values: [10, 20, 30, 40] }, + delay_values: [3, 10, 3, 35, 9], framework_errors: 1, user_errors: 0, total_errors: 1, @@ -2759,6 +2974,7 @@ describe('createAggregator', () => { not_timed_out: 0, total: 0, delay: { counts: [1], values: [10] }, + delay_values: [5], framework_errors: 0, user_errors: 0, total_errors: 0, @@ -2807,6 +3023,7 @@ describe('createAggregator', () => { not_timed_out: 1, total: 1, delay: { counts: [1], values: [10] }, + delay_values: [5], framework_errors: 0, user_errors: 0, total_errors: 0, @@ -2855,6 +3072,7 @@ describe('createAggregator', () => { not_timed_out: 1, total: 1, delay: { counts: [1, 1], values: [10, 20] }, + delay_values: [5, 12], framework_errors: 0, user_errors: 0, total_errors: 0, @@ -2903,6 +3121,7 @@ describe('createAggregator', () => { not_timed_out: 2, total: 2, delay: { counts: [1, 1], values: [10, 20] }, + delay_values: [5, 12], framework_errors: 0, user_errors: 0, total_errors: 0, @@ -2951,6 +3170,7 @@ describe('createAggregator', () => { not_timed_out: 2, total: 2, delay: { counts: [2, 1], values: [10, 20] }, + delay_values: [5, 12, 4], framework_errors: 0, user_errors: 0, total_errors: 0, @@ -2999,6 +3219,7 @@ describe('createAggregator', () => { not_timed_out: 3, total: 3, delay: { counts: [2, 1], values: [10, 20] }, + delay_values: [5, 12, 4], framework_errors: 1, user_errors: 0, total_errors: 1, @@ -3047,6 +3268,7 @@ describe('createAggregator', () => { not_timed_out: 3, total: 3, delay: { counts: [3, 1], values: [10, 20] }, + delay_values: [5, 12, 4, 4], framework_errors: 1, user_errors: 0, total_errors: 1, @@ -3095,6 +3317,7 @@ describe('createAggregator', () => { not_timed_out: 4, total: 4, delay: { counts: [3, 1], values: [10, 20] }, + delay_values: [5, 12, 4, 4], framework_errors: 1, user_errors: 0, total_errors: 1, @@ -3143,6 +3366,7 @@ describe('createAggregator', () => { not_timed_out: 4, total: 4, delay: { counts: [4, 1], values: [10, 20] }, + delay_values: [5, 12, 4, 4, 3], framework_errors: 1, user_errors: 0, total_errors: 1, @@ -3191,6 +3415,7 @@ describe('createAggregator', () => { not_timed_out: 5, total: 5, delay: { counts: [4, 1], values: [10, 20] }, + delay_values: [5, 12, 4, 4, 3], framework_errors: 2, user_errors: 0, total_errors: 2, @@ -3339,6 +3564,7 @@ describe('createAggregator', () => { counts: [3, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1], values: [10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120, 130], }, + overdue_by_values: [0, 0, 0, 20, 20, 40, 120], }, by_type: { 'alerting:example': { @@ -3346,36 +3572,42 @@ describe('createAggregator', () => { counts: [0, 0, 0, 0, 1], values: [10, 20, 30, 40, 50], }, + overdue_by_values: [40], }, 'alerting:__index-threshold': { overdue_by: { counts: [0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1], values: [10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120, 130], }, + overdue_by_values: [20, 20, 120], }, alerting: { overdue_by: { counts: [0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1], values: [10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120, 130], }, + overdue_by_values: [40, 20, 20, 120], }, 'actions:webhook': { overdue_by: { counts: [2], values: [10], }, + overdue_by_values: [0, 0], }, 'actions:__email': { overdue_by: { counts: [1], values: [10], }, + overdue_by_values: [0], }, actions: { overdue_by: { counts: [3], values: [10], }, + overdue_by_values: [0, 0, 0], }, }, }, @@ -3383,7 +3615,10 @@ describe('createAggregator', () => { expect(metrics[1]).toEqual({ key: 'task_overdue', value: { - overall: { overdue_by: { counts: [], values: [] } }, + overall: { + overdue_by: { counts: [], values: [] }, + overdue_by_values: [], + }, by_type: {}, }, }); @@ -3395,6 +3630,9 @@ describe('createAggregator', () => { counts: [16, 0, 1, 2, 0, 1], values: [10, 20, 30, 40, 50, 60], }, + overdue_by_values: [ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 30, 30, 50, + ], }, by_type: { telemetry: { @@ -3402,30 +3640,35 @@ describe('createAggregator', () => { counts: [1, 0, 1], values: [10, 20, 30], }, + overdue_by_values: [0, 20], }, reporting: { overdue_by: { counts: [1], values: [10], }, + overdue_by_values: [0], }, 'actions:webhook': { overdue_by: { counts: [3, 0, 0, 2, 0, 1], values: [10, 20, 30, 40, 50, 60], }, + overdue_by_values: [0, 0, 0, 30, 30, 50], }, 'actions:__email': { overdue_by: { counts: [11], values: [10], }, + overdue_by_values: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], }, actions: { overdue_by: { counts: [14, 0, 0, 2, 0, 1], values: [10, 20, 30, 40, 50, 60], }, + overdue_by_values: [0, 0, 0, 30, 30, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], }, }, }, diff --git a/x-pack/plugins/task_manager/server/metrics/lib/simple_histogram.test.ts b/x-pack/plugins/task_manager/server/metrics/lib/simple_histogram.test.ts index 9c24aabbdd575..a574eb9e809ec 100644 --- a/x-pack/plugins/task_manager/server/metrics/lib/simple_histogram.test.ts +++ b/x-pack/plugins/task_manager/server/metrics/lib/simple_histogram.test.ts @@ -28,6 +28,7 @@ describe('SimpleHistogram', () => { { value: 90, count: 0 }, { value: 100, count: 0 }, ]); + expect(histogram.getAllValues()).toEqual([]); }); test('should correctly initialize when bucketSize does not evenly divides range', () => { @@ -49,6 +50,7 @@ describe('SimpleHistogram', () => { { value: 98, count: 0 }, { value: 105, count: 0 }, ]); + expect(histogram.getAllValues()).toEqual([]); }); test('should correctly record values', () => { @@ -77,6 +79,7 @@ describe('SimpleHistogram', () => { { value: 90, count: 0 }, { value: 100, count: 1 }, ]); + expect(histogram.getAllValues()).toEqual([0, 10, 23, 34, 21, 56, 78, 33, 99, 1, 2]); }); test('should correctly record values with specific increment', () => { @@ -104,6 +107,9 @@ describe('SimpleHistogram', () => { { value: 90, count: 0 }, { value: 100, count: 5 }, ]); + expect(histogram.getAllValues()).toEqual([ + 0, 23, 23, 34, 21, 56, 78, 33, 33, 33, 33, 99, 99, 99, 99, 99, 1, 2, + ]); }); test('should ignore values less than 0 and greater than max', () => { @@ -119,11 +125,13 @@ describe('SimpleHistogram', () => { histogram.record(2); const hist1 = histogram.get(); + const hist1AllValues = histogram.getAllValues(); histogram.record(-1); histogram.record(200); expect(histogram.get()).toEqual(hist1); + expect(histogram.getAllValues()).toEqual(hist1AllValues); }); test('should correctly reset values', () => { @@ -150,6 +158,7 @@ describe('SimpleHistogram', () => { { value: 90, count: 0 }, { value: 100, count: 1 }, ]); + expect(histogram.getAllValues()).toEqual([23, 34, 21, 56, 78, 33, 99, 1, 2]); histogram.reset(); @@ -165,6 +174,7 @@ describe('SimpleHistogram', () => { { value: 90, count: 0 }, { value: 100, count: 0 }, ]); + expect(histogram.getAllValues()).toEqual([]); }); test('should correctly truncate zero values', () => { @@ -189,6 +199,7 @@ describe('SimpleHistogram', () => { { value: 90, count: 0 }, { value: 100, count: 0 }, ]); + expect(histogram.getAllValues()).toEqual([23, 34, 21, 56, 33, 1, 2]); expect(histogram.get(true)).toEqual([ { value: 10, count: 2 }, @@ -204,6 +215,7 @@ describe('SimpleHistogram', () => { const histogram = new SimpleHistogram(100, 10); expect(histogram.get(true)).toEqual([]); + expect(histogram.getAllValues()).toEqual([]); }); test('should correctly serialize histogram data', () => { diff --git a/x-pack/plugins/task_manager/server/metrics/lib/simple_histogram.ts b/x-pack/plugins/task_manager/server/metrics/lib/simple_histogram.ts index 7940ee3c48ee5..460369099def0 100644 --- a/x-pack/plugins/task_manager/server/metrics/lib/simple_histogram.ts +++ b/x-pack/plugins/task_manager/server/metrics/lib/simple_histogram.ts @@ -23,6 +23,7 @@ export class SimpleHistogram { private maxValue: number; private bucketSize: number; private histogramBuckets: Bucket[] = []; + private allValues: number[] = []; constructor(max: number, bucketSize: number) { if (bucketSize > max) { @@ -38,6 +39,7 @@ export class SimpleHistogram { for (let i = 0; i < this.histogramBuckets.length; i++) { this.histogramBuckets[i].count = 0; } + this.allValues = []; } public record(value: number, increment: number = 1) { @@ -52,6 +54,10 @@ export class SimpleHistogram { break; } } + + for (let i = 0; i < increment; i++) { + this.allValues.push(value); + } } public get(truncate: boolean = false) { @@ -74,6 +80,10 @@ export class SimpleHistogram { })); } + public getAllValues(truncate: boolean = false) { + return this.allValues.slice(); + } + public serialize(): SerializedHistogram { const counts: number[] = []; const values: number[] = []; diff --git a/x-pack/plugins/task_manager/server/metrics/task_claim_metrics_aggregator.test.ts b/x-pack/plugins/task_manager/server/metrics/task_claim_metrics_aggregator.test.ts index cfcf4bfdf8d0b..0874940675287 100644 --- a/x-pack/plugins/task_manager/server/metrics/task_claim_metrics_aggregator.test.ts +++ b/x-pack/plugins/task_manager/server/metrics/task_claim_metrics_aggregator.test.ts @@ -47,6 +47,7 @@ describe('TaskClaimMetricsAggregator', () => { success: 0, total: 0, duration: { counts: [], values: [] }, + duration_values: [], }); }); @@ -55,6 +56,7 @@ describe('TaskClaimMetricsAggregator', () => { success: 0, total: 0, duration: { counts: [], values: [] }, + duration_values: [], }); }); @@ -65,6 +67,7 @@ describe('TaskClaimMetricsAggregator', () => { success: 2, total: 2, duration: { counts: [2], values: [100] }, + duration_values: [10, 10], }); }); @@ -75,6 +78,7 @@ describe('TaskClaimMetricsAggregator', () => { success: 0, total: 2, duration: { counts: [], values: [] }, + duration_values: [], }); }); @@ -90,6 +94,7 @@ describe('TaskClaimMetricsAggregator', () => { success: 4, total: 7, duration: { counts: [4], values: [100] }, + duration_values: [10, 10, 10, 10], }); taskClaimMetricsAggregator.reset(); @@ -97,6 +102,7 @@ describe('TaskClaimMetricsAggregator', () => { success: 0, total: 0, duration: { counts: [], values: [] }, + duration_values: [], }); }); }); diff --git a/x-pack/plugins/task_manager/server/metrics/task_claim_metrics_aggregator.ts b/x-pack/plugins/task_manager/server/metrics/task_claim_metrics_aggregator.ts index 8bb1269bae2fb..486a3f9aa5fd7 100644 --- a/x-pack/plugins/task_manager/server/metrics/task_claim_metrics_aggregator.ts +++ b/x-pack/plugins/task_manager/server/metrics/task_claim_metrics_aggregator.ts @@ -26,6 +26,7 @@ interface TaskClaimCounts extends JsonObject { export type TaskClaimMetric = TaskClaimCounts & { duration: SerializedHistogram; + duration_values: number[]; }; export class TaskClaimMetricsAggregator implements ITaskMetricsAggregator { @@ -38,12 +39,14 @@ export class TaskClaimMetricsAggregator implements ITaskMetricsAggregator { test('should correctly initialize', () => { expect(taskOverdueMetricsAggregator.collect()).toEqual({ - overall: { overdue_by: { counts: [], values: [] } }, + overall: { + overdue_by: { counts: [], values: [] }, + overdue_by_values: [], + }, by_type: {}, }); }); test('should correctly return initialMetrics', () => { expect(taskOverdueMetricsAggregator.initialMetric()).toEqual({ - overall: { overdue_by: { counts: [], values: [] } }, + overall: { + overdue_by: { counts: [], values: [] }, + overdue_by_values: [], + }, by_type: {}, }); }); @@ -50,9 +56,15 @@ describe('TaskOverdueMetricsAggregator', () => { }) ); expect(taskOverdueMetricsAggregator.collect()).toEqual({ - overall: { overdue_by: { counts: [1, 0, 1], values: [10, 20, 30] } }, + overall: { + overdue_by: { counts: [1, 0, 1], values: [10, 20, 30] }, + overdue_by_values: [0, 20], + }, by_type: { - telemetry: { overdue_by: { counts: [1, 0, 1], values: [10, 20, 30] } }, + telemetry: { + overdue_by: { counts: [1, 0, 1], values: [10, 20, 30] }, + overdue_by_values: [0, 20], + }, }, }); }); @@ -66,7 +78,10 @@ describe('TaskOverdueMetricsAggregator', () => { }) ); expect(taskOverdueMetricsAggregator.collect()).toEqual({ - overall: { overdue_by: { counts: [], values: [] } }, + overall: { + overdue_by: { counts: [], values: [] }, + overdue_by_values: [], + }, by_type: {}, }); }); @@ -95,9 +110,15 @@ describe('TaskOverdueMetricsAggregator', () => { }) ); expect(taskOverdueMetricsAggregator.collect()).toEqual({ - overall: { overdue_by: { counts: [0, 0, 0, 0, 1], values: [10, 20, 30, 40, 50] } }, + overall: { + overdue_by: { counts: [0, 0, 0, 0, 1], values: [10, 20, 30, 40, 50] }, + overdue_by_values: [40], + }, by_type: { - telemetry: { overdue_by: { counts: [0, 0, 0, 0, 1], values: [10, 20, 30, 40, 50] } }, + telemetry: { + overdue_by: { counts: [0, 0, 0, 0, 1], values: [10, 20, 30, 40, 50] }, + overdue_by_values: [40], + }, }, }); }); @@ -129,6 +150,7 @@ describe('TaskOverdueMetricsAggregator', () => { counts: [3, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1], values: [10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120, 130], }, + overdue_by_values: [0, 0, 0, 20, 20, 40, 120], }, by_type: { 'alerting:example': { @@ -136,36 +158,42 @@ describe('TaskOverdueMetricsAggregator', () => { counts: [0, 0, 0, 0, 1], values: [10, 20, 30, 40, 50], }, + overdue_by_values: [40], }, 'alerting:__index-threshold': { overdue_by: { counts: [0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1], values: [10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120, 130], }, + overdue_by_values: [20, 20, 120], }, alerting: { overdue_by: { counts: [0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1], values: [10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120, 130], }, + overdue_by_values: [40, 20, 20, 120], }, 'actions:webhook': { overdue_by: { counts: [2], values: [10], }, + overdue_by_values: [0, 0], }, 'actions:__email': { overdue_by: { counts: [1], values: [10], }, + overdue_by_values: [0], }, actions: { overdue_by: { counts: [3], values: [10], }, + overdue_by_values: [0, 0, 0], }, }, }); @@ -198,44 +226,50 @@ describe('TaskOverdueMetricsAggregator', () => { counts: [3, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1], values: [10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120, 130], }, + overdue_by_values: [0, 0, 0, 20, 20, 40, 120], }, by_type: { 'alerting:example': { overdue_by: { counts: [0, 0, 0, 0, 1], - values: [10, 20, 30, 40, 50], }, + overdue_by_values: [40], }, 'alerting:__index-threshold': { overdue_by: { counts: [0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1], values: [10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120, 130], }, + overdue_by_values: [20, 20, 120], }, alerting: { overdue_by: { counts: [0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1], values: [10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120, 130], }, + overdue_by_values: [40, 20, 20, 120], }, 'actions:webhook': { overdue_by: { counts: [2], values: [10], }, + overdue_by_values: [0, 0], }, 'actions:__email': { overdue_by: { counts: [1], values: [10], }, + overdue_by_values: [0], }, actions: { overdue_by: { counts: [3], values: [10], }, + overdue_by_values: [0, 0, 0], }, }, }); diff --git a/x-pack/plugins/task_manager/server/metrics/task_overdue_metrics_aggregator.ts b/x-pack/plugins/task_manager/server/metrics/task_overdue_metrics_aggregator.ts index 4743889e7ebf9..bd2f6a97372aa 100644 --- a/x-pack/plugins/task_manager/server/metrics/task_overdue_metrics_aggregator.ts +++ b/x-pack/plugins/task_manager/server/metrics/task_overdue_metrics_aggregator.ts @@ -6,16 +6,12 @@ */ import { JsonObject } from '@kbn/utility-types'; -import { keys, mapValues } from 'lodash'; +import { keys } from 'lodash'; +import { set } from '@kbn/safer-lodash-set'; import { isOk, unwrap } from '../lib/result_type'; import { TaskLifecycleEvent } from '../polling_lifecycle'; import { TaskManagerMetric } from '../task_events'; -import { - unflattenObject, - getTaskTypeGroup, - type SerializedHistogram, - SimpleHistogram, -} from './lib'; +import { getTaskTypeGroup, type SerializedHistogram, SimpleHistogram } from './lib'; import { TaskManagerMetrics } from './task_metrics_collector'; import { ITaskMetricsAggregator } from './types'; @@ -23,6 +19,7 @@ const HDR_HISTOGRAM_MAX = 5400; // 90 minutes const HDR_HISTOGRAM_BUCKET_SIZE = 10; // 10 seconds const OVERDUE_BY_KEY = 'overdue_by'; +const OVERDUE_BY_VALUES_KEY = 'overdue_by_values'; enum TaskOverdueMetricKeys { OVERALL = 'overall', @@ -31,6 +28,7 @@ enum TaskOverdueMetricKeys { interface TaskOverdueHistogram extends JsonObject { [OVERDUE_BY_KEY]: SerializedHistogram; + [OVERDUE_BY_VALUES_KEY]: number[]; } export interface TaskOverdueMetric extends JsonObject { [TaskOverdueMetricKeys.OVERALL]: TaskOverdueHistogram; @@ -45,18 +43,26 @@ export class TaskOverdueMetricsAggregator implements ITaskMetricsAggregator hist.serialize())); + + for (const prop of Object.keys(this.histograms)) { + const hist = this.histograms[prop]; + set(result, prop, hist.serialize()); + set(result, `${prop}_values`, hist.getAllValues()); + } + + return result; } public reset() { diff --git a/x-pack/plugins/task_manager/server/metrics/task_run_metrics_aggregator.test.ts b/x-pack/plugins/task_manager/server/metrics/task_run_metrics_aggregator.test.ts index 19c022dea662f..d156685885191 100644 --- a/x-pack/plugins/task_manager/server/metrics/task_run_metrics_aggregator.test.ts +++ b/x-pack/plugins/task_manager/server/metrics/task_run_metrics_aggregator.test.ts @@ -94,6 +94,7 @@ describe('TaskRunMetricsAggregator', () => { user_errors: 0, delay: { counts: [], values: [] }, total_errors: 0, + delay_values: [], }, }); }); @@ -108,6 +109,7 @@ describe('TaskRunMetricsAggregator', () => { user_errors: 0, delay: { counts: [], values: [] }, total_errors: 0, + delay_values: [], }, by_type: {}, }); @@ -125,6 +127,7 @@ describe('TaskRunMetricsAggregator', () => { user_errors: 0, delay: { counts: [], values: [] }, total_errors: 0, + delay_values: [], }, by_type: { telemetry: { @@ -150,6 +153,7 @@ describe('TaskRunMetricsAggregator', () => { user_errors: 0, delay: { counts: [1], values: [10] }, total_errors: 0, + delay_values: [3], }, }); }); @@ -167,6 +171,7 @@ describe('TaskRunMetricsAggregator', () => { user_errors: 0, delay: { counts: [], values: [] }, total_errors: 0, + delay_values: [], }, }); }); @@ -183,6 +188,7 @@ describe('TaskRunMetricsAggregator', () => { user_errors: 0, delay: { counts: [], values: [] }, total_errors: 0, + delay_values: [], }, by_type: { telemetry: { @@ -209,6 +215,7 @@ describe('TaskRunMetricsAggregator', () => { user_errors: 0, delay: { counts: [], values: [] }, total_errors: 2, + delay_values: [], }, by_type: { telemetry: { @@ -235,6 +242,7 @@ describe('TaskRunMetricsAggregator', () => { user_errors: 0, delay: { counts: [], values: [] }, total_errors: 2, + delay_values: [], }, by_type: { telemetry: { @@ -263,6 +271,7 @@ describe('TaskRunMetricsAggregator', () => { user_errors: 0, delay: { counts: [], values: [] }, total_errors: 1, + delay_values: [], }, by_type: { report: { @@ -315,6 +324,7 @@ describe('TaskRunMetricsAggregator', () => { user_errors: 0, delay: { counts: [], values: [] }, total_errors: 3, + delay_values: [], }, by_type: { actions: { @@ -417,6 +427,7 @@ describe('TaskRunMetricsAggregator', () => { not_timed_out: 12, total: 14, delay: { counts: [3, 0, 1], values: [10, 20, 30] }, + delay_values: [3, 25, 6, 9], framework_errors: 3, user_errors: 0, total_errors: 3, @@ -499,6 +510,7 @@ describe('TaskRunMetricsAggregator', () => { user_errors: 0, delay: { counts: [], values: [] }, total_errors: 0, + delay_values: [], }, by_type: { actions: { diff --git a/x-pack/plugins/task_manager/server/metrics/task_run_metrics_aggregator.ts b/x-pack/plugins/task_manager/server/metrics/task_run_metrics_aggregator.ts index bfa230ac9bf5e..c0c80e4021498 100644 --- a/x-pack/plugins/task_manager/server/metrics/task_run_metrics_aggregator.ts +++ b/x-pack/plugins/task_manager/server/metrics/task_run_metrics_aggregator.ts @@ -57,6 +57,7 @@ export interface TaskRunMetrics extends JsonObject { export interface TaskRunMetric extends JsonObject { overall: TaskRunMetrics['overall'] & { delay: SerializedHistogram; + delay_values: number[]; }; by_type: TaskRunMetrics['by_type']; } @@ -71,12 +72,20 @@ export class TaskRunMetricsAggregator implements ITaskMetricsAggregator