From 4a2cc02464f061d492bf52b1359ea7721c15abc2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Louv-Jansen?= Date: Thu, 29 Nov 2018 21:41:33 +0100 Subject: [PATCH] [APM] Move impact calculation to Elasticsearch --- .../__snapshots__/fetcher.test.ts.snap | 7 +- .../__snapshots__/transform.test.ts.snap | 65 +++++++++++++------ .../server/lib/transaction_groups/fetcher.ts | 8 ++- .../transactionGroupsResponse.ts | 23 +++++++ .../lib/transaction_groups/transform.test.ts | 27 +++++++- .../lib/transaction_groups/transform.ts | 9 +-- 6 files changed, 111 insertions(+), 28 deletions(-) diff --git a/x-pack/plugins/apm/server/lib/transaction_groups/__snapshots__/fetcher.test.ts.snap b/x-pack/plugins/apm/server/lib/transaction_groups/__snapshots__/fetcher.test.ts.snap index 07f8bf59f53a0..42240a161b46a 100644 --- a/x-pack/plugins/apm/server/lib/transaction_groups/__snapshots__/fetcher.test.ts.snap +++ b/x-pack/plugins/apm/server/lib/transaction_groups/__snapshots__/fetcher.test.ts.snap @@ -14,6 +14,11 @@ Array [ "field": "transaction.duration.us", }, }, + "durationSum": Object { + "sum": Object { + "field": "transaction.duration.us", + }, + }, "p95": Object { "percentiles": Object { "field": "transaction.duration.us", @@ -41,7 +46,7 @@ Array [ "terms": Object { "field": "transaction.name.keyword", "order": Object { - "avg": "desc", + "durationSum": "desc", }, "size": 100, }, diff --git a/x-pack/plugins/apm/server/lib/transaction_groups/__snapshots__/transform.test.ts.snap b/x-pack/plugins/apm/server/lib/transaction_groups/__snapshots__/transform.test.ts.snap index a68d7dc80338a..ea5aa8b0aa6ec 100644 --- a/x-pack/plugins/apm/server/lib/transaction_groups/__snapshots__/transform.test.ts.snap +++ b/x-pack/plugins/apm/server/lib/transaction_groups/__snapshots__/transform.test.ts.snap @@ -3,8 +3,9 @@ exports[`transactionGroupsTransformer should match snapshot 1`] = ` Array [ Object { + "absoluteImpact": 46073935, "averageResponseTime": 255966.30555555556, - "impact": 4.369340653255684, + "impact": 4.3693406535517445, "name": "POST /api/orders", "p95": 320238.5, "sample": Object { @@ -128,6 +129,7 @@ baz", "transactionsPerMinute": 5684.210526315789, }, Object { + "absoluteImpact": 1052209442, "averageResponseTime": 48021.972616494, "impact": 100, "name": "GET /api", @@ -254,8 +256,9 @@ baz", "transactionsPerMinute": 691926.3157894736, }, Object { + "absoluteImpact": 108011563, "averageResponseTime": 33265.03326147213, - "impact": 10.256357027900046, + "impact": 10.256357027376065, "name": "GET /api/orders", "p95": 58827.489999999976, "sample": Object { @@ -375,8 +378,9 @@ baz", "transactionsPerMinute": 102536.84210526315, }, Object { + "absoluteImpact": 23030509, "averageResponseTime": 32900.72714285714, - "impact": 2.179120743402716, + "impact": 2.1791207411745854, "name": "GET /log-message", "p95": 40444, "sample": Object { @@ -496,8 +500,9 @@ baz", "transactionsPerMinute": 22105.263157894737, }, Object { + "absoluteImpact": 151019688, "averageResponseTime": 32554.36257814184, - "impact": 14.344171564855404, + "impact": 14.344171563678346, "name": "GET /api/stats", "p95": 59356.73611111111, "sample": Object { @@ -623,8 +628,9 @@ baz", "transactionsPerMinute": 146494.73684210525, }, Object { + "absoluteImpact": 23837374, "averageResponseTime": 32387.73641304348, - "impact": 2.2558112391673664, + "impact": 2.2558112380477584, "name": "GET /log-error", "p95": 40061.1, "sample": Object { @@ -744,8 +750,9 @@ baz", "transactionsPerMinute": 23242.105263157893, }, Object { + "absoluteImpact": 108250312, "averageResponseTime": 32159.926322043968, - "impact": 10.279049521913821, + "impact": 10.27904952170656, "name": "GET /api/customers", "p95": 59845.85714285714, "sample": Object { @@ -871,8 +878,9 @@ baz", "transactionsPerMinute": 106294.73684210525, }, Object { + "absoluteImpact": 101647397, "averageResponseTime": 27516.89144558744, - "impact": 9.651458993728006, + "impact": 9.651458992731666, "name": "GET /api/products/top", "p95": 56064.679999999986, "sample": Object { @@ -999,8 +1007,9 @@ baz", "transactionsPerMinute": 116652.63157894736, }, Object { + "absoluteImpact": 3135762, "averageResponseTime": 21331.714285714286, - "impact": 0.28817488008070574, + "impact": 0.28817487960409877, "name": "POST /api", "p95": 30938, "sample": Object { @@ -1122,8 +1131,9 @@ baz", "transactionsPerMinute": 4642.105263157894, }, Object { + "absoluteImpact": 36131970, "averageResponseTime": 17189.329210275926, - "impact": 3.424381788267164, + "impact": 3.424381787142002, "name": "GET /api/products/:id/customers", "p95": 39284.79999999999, "sample": Object { @@ -1249,8 +1259,9 @@ baz", "transactionsPerMinute": 66378.94736842105, }, Object { + "absoluteImpact": 18494584, "averageResponseTime": 12763.68806073154, - "impact": 1.747992435179465, + "impact": 1.7479924334286208, "name": "GET /api/types/:id", "p95": 30576.749999999996, "sample": Object { @@ -1370,8 +1381,9 @@ baz", "transactionsPerMinute": 45757.8947368421, }, Object { + "absoluteImpact": 46648776, "averageResponseTime": 12683.190864600327, - "impact": 4.4239778511514745, + "impact": 4.4239778504968, "name": "GET /api/products", "p95": 35009.67999999999, "sample": Object { @@ -1491,8 +1503,9 @@ baz", "transactionsPerMinute": 116147.36842105263, }, Object { + "absoluteImpact": 27018619, "averageResponseTime": 11257.757916666667, - "impact": 2.558180605423081, + "impact": 2.558180605569336, "name": "GET /api/types", "p95": 35222.944444444445, "sample": Object { @@ -1612,8 +1625,9 @@ baz", "transactionsPerMinute": 75789.47368421052, }, Object { + "absoluteImpact": 13579338, "averageResponseTime": 10584.05144193297, - "impact": 1.2808106158729446, + "impact": 1.280810614916383, "name": "GET /api/orders/:id", "p95": 26555.399999999998, "sample": Object { @@ -1731,8 +1745,9 @@ baz", "transactionsPerMinute": 40515.789473684206, }, Object { + "absoluteImpact": 19398174, "averageResponseTime": 10548.218597063622, - "impact": 1.8338764008269306, + "impact": 1.8338763992340905, "name": "GET /api/products/:id", "p95": 28413.383333333328, "sample": Object { @@ -1852,8 +1867,9 @@ baz", "transactionsPerMinute": 58073.68421052631, }, Object { + "absoluteImpact": 18749614, "averageResponseTime": 9868.217894736843, - "impact": 1.7722323979309487, + "impact": 1.7722323960215767, "name": "GET /api/customers/:id", "p95": 27486.5, "sample": Object { @@ -1979,6 +1995,7 @@ baz", "transactionsPerMinute": 59999.99999999999, }, Object { + "absoluteImpact": 103858, "averageResponseTime": 5192.9, "impact": 0, "name": "POST unknown route", @@ -2110,8 +2127,9 @@ baz", "transactionsPerMinute": 631.578947368421, }, Object { + "absoluteImpact": 1680454, "averageResponseTime": 4694.005586592179, - "impact": 0.1498514997591876, + "impact": 0.1498515000753004, "name": "GET /is-it-coffee-time", "p95": 11022.99999999992, "sample": Object { @@ -2232,8 +2250,9 @@ baz", "transactionsPerMinute": 11305.263157894737, }, Object { + "absoluteImpact": 1528763, "averageResponseTime": 4549.889880952381, - "impact": 0.1354336505457395, + "impact": 0.13543365054509587, "name": "GET /throw-error", "p95": 7719.700000000001, "sample": Object { @@ -2354,8 +2373,9 @@ baz", "transactionsPerMinute": 10610.526315789473, }, Object { + "absoluteImpact": 24934595, "averageResponseTime": 3504.5108924806746, - "impact": 2.36009934580083, + "impact": 2.3600993453143766, "name": "GET *", "p95": 11431.738095238095, "sample": Object { @@ -2481,8 +2501,9 @@ baz", "transactionsPerMinute": 224684.21052631576, }, Object { + "absoluteImpact": 998256, "averageResponseTime": 2742.4615384615386, - "impact": 0.08501029113483448, + "impact": 0.08501028923348058, "name": "OPTIONS unknown route", "p95": 4370.000000000002, "sample": Object { @@ -2604,8 +2625,9 @@ baz", "transactionsPerMinute": 11494.736842105262, }, Object { + "absoluteImpact": 166023502, "averageResponseTime": 2651.8784461553205, - "impact": 15.770246498769827, + "impact": 15.770246496477105, "name": "GET static file", "p95": 6140.579335038363, "sample": Object { @@ -2712,8 +2734,9 @@ baz", "transactionsPerMinute": 1977031.5789473683, }, Object { + "absoluteImpact": 10653452, "averageResponseTime": 1422.926672899693, - "impact": 1.002712481568783, + "impact": 1.0027124806135428, "name": "GET unknown route", "p95": 2311.885238095238, "sample": Object { diff --git a/x-pack/plugins/apm/server/lib/transaction_groups/fetcher.ts b/x-pack/plugins/apm/server/lib/transaction_groups/fetcher.ts index 2a534aa3315f1..8c976477de8c0 100644 --- a/x-pack/plugins/apm/server/lib/transaction_groups/fetcher.ts +++ b/x-pack/plugins/apm/server/lib/transaction_groups/fetcher.ts @@ -24,6 +24,9 @@ interface Bucket { '95.0': number; }; }; + durationSum: { + value: number; + }; sample: { hits: { total: number; @@ -57,7 +60,7 @@ export function transactionGroupsFetcher( transactions: { terms: { field: `${TRANSACTION_NAME}.keyword`, - order: { avg: 'desc' }, + order: { durationSum: 'desc' }, size: 100 }, aggs: { @@ -73,6 +76,9 @@ export function transactionGroupsFetcher( avg: { avg: { field: TRANSACTION_DURATION } }, p95: { percentiles: { field: TRANSACTION_DURATION, percents: [95] } + }, + durationSum: { + sum: { field: 'transaction.duration.us' } } } } diff --git a/x-pack/plugins/apm/server/lib/transaction_groups/mock-responses/transactionGroupsResponse.ts b/x-pack/plugins/apm/server/lib/transaction_groups/mock-responses/transactionGroupsResponse.ts index 08492ee7a7839..5a90a9f9e61a6 100644 --- a/x-pack/plugins/apm/server/lib/transaction_groups/mock-responses/transactionGroupsResponse.ts +++ b/x-pack/plugins/apm/server/lib/transaction_groups/mock-responses/transactionGroupsResponse.ts @@ -21,6 +21,7 @@ export const transactionGroupsResponse = ({ doc_count: 180, avg: { value: 255966.30555555556 }, p95: { values: { '95.0': 320238.5 } }, + durationSum: { value: 46073935 }, sample: { hits: { total: 180, @@ -140,6 +141,7 @@ export const transactionGroupsResponse = ({ doc_count: 21911, avg: { value: 48021.972616494 }, p95: { values: { '95.0': 67138.18364917398 } }, + durationSum: { value: 1052209442 }, sample: { hits: { total: 21911, @@ -259,6 +261,7 @@ export const transactionGroupsResponse = ({ doc_count: 3247, avg: { value: 33265.03326147213 }, p95: { values: { '95.0': 58827.489999999976 } }, + durationSum: { value: 108011563 }, sample: { hits: { total: 3247, @@ -374,6 +377,7 @@ export const transactionGroupsResponse = ({ doc_count: 700, avg: { value: 32900.72714285714 }, p95: { values: { '95.0': 40444 } }, + durationSum: { value: 23030509 }, sample: { hits: { total: 700, @@ -489,6 +493,7 @@ export const transactionGroupsResponse = ({ doc_count: 4639, avg: { value: 32554.36257814184 }, p95: { values: { '95.0': 59356.73611111111 } }, + durationSum: { value: 151019688 }, sample: { hits: { total: 4639, @@ -609,6 +614,7 @@ export const transactionGroupsResponse = ({ doc_count: 736, avg: { value: 32387.73641304348 }, p95: { values: { '95.0': 40061.1 } }, + durationSum: { value: 23837374 }, sample: { hits: { total: 736, @@ -724,6 +730,7 @@ export const transactionGroupsResponse = ({ doc_count: 3366, avg: { value: 32159.926322043968 }, p95: { values: { '95.0': 59845.85714285714 } }, + durationSum: { value: 108250312 }, sample: { hits: { total: 3366, @@ -844,6 +851,7 @@ export const transactionGroupsResponse = ({ doc_count: 3694, avg: { value: 27516.89144558744 }, p95: { values: { '95.0': 56064.679999999986 } }, + durationSum: { value: 101647397 }, sample: { hits: { total: 3694, @@ -965,6 +973,7 @@ export const transactionGroupsResponse = ({ doc_count: 147, avg: { value: 21331.714285714286 }, p95: { values: { '95.0': 30938 } }, + durationSum: { value: 3135762 }, sample: { hits: { total: 147, @@ -1082,6 +1091,7 @@ export const transactionGroupsResponse = ({ doc_count: 2102, avg: { value: 17189.329210275926 }, p95: { values: { '95.0': 39284.79999999999 } }, + durationSum: { value: 36131970 }, sample: { hits: { total: 2102, @@ -1203,6 +1213,7 @@ export const transactionGroupsResponse = ({ doc_count: 1449, avg: { value: 12763.68806073154 }, p95: { values: { '95.0': 30576.749999999996 } }, + durationSum: { value: 18494584 }, sample: { hits: { total: 1449, @@ -1318,6 +1329,7 @@ export const transactionGroupsResponse = ({ doc_count: 3678, avg: { value: 12683.190864600327 }, p95: { values: { '95.0': 35009.67999999999 } }, + durationSum: { value: 46648776 }, sample: { hits: { total: 3678, @@ -1433,6 +1445,7 @@ export const transactionGroupsResponse = ({ doc_count: 2400, avg: { value: 11257.757916666667 }, p95: { values: { '95.0': 35222.944444444445 } }, + durationSum: { value: 27018619 }, sample: { hits: { total: 2400, @@ -1548,6 +1561,7 @@ export const transactionGroupsResponse = ({ doc_count: 1283, avg: { value: 10584.05144193297 }, p95: { values: { '95.0': 26555.399999999998 } }, + durationSum: { value: 13579338 }, sample: { hits: { total: 1283, @@ -1661,6 +1675,7 @@ export const transactionGroupsResponse = ({ doc_count: 1839, avg: { value: 10548.218597063622 }, p95: { values: { '95.0': 28413.383333333328 } }, + durationSum: { value: 19398174 }, sample: { hits: { total: 1839, @@ -1776,6 +1791,7 @@ export const transactionGroupsResponse = ({ doc_count: 1900, avg: { value: 9868.217894736843 }, p95: { values: { '95.0': 27486.5 } }, + durationSum: { value: 18749614 }, sample: { hits: { total: 1900, @@ -1896,6 +1912,7 @@ export const transactionGroupsResponse = ({ doc_count: 20, avg: { value: 5192.9 }, p95: { values: { '95.0': 13230.5 } }, + durationSum: { value: 103858 }, sample: { hits: { total: 20, @@ -2021,6 +2038,7 @@ export const transactionGroupsResponse = ({ doc_count: 358, avg: { value: 4694.005586592179 }, p95: { values: { '95.0': 11022.99999999992 } }, + durationSum: { value: 1680454 }, sample: { hits: { total: 358, @@ -2137,6 +2155,7 @@ export const transactionGroupsResponse = ({ doc_count: 336, avg: { value: 4549.889880952381 }, p95: { values: { '95.0': 7719.700000000001 } }, + durationSum: { value: 1528763 }, sample: { hits: { total: 336, @@ -2253,6 +2272,7 @@ export const transactionGroupsResponse = ({ doc_count: 7115, avg: { value: 3504.5108924806746 }, p95: { values: { '95.0': 11431.738095238095 } }, + durationSum: { value: 24934595 }, sample: { hits: { total: 7115, @@ -2375,6 +2395,7 @@ export const transactionGroupsResponse = ({ doc_count: 364, avg: { value: 2742.4615384615386 }, p95: { values: { '95.0': 4370.000000000002 } }, + durationSum: { value: 998256 }, sample: { hits: { total: 364, @@ -2492,6 +2513,7 @@ export const transactionGroupsResponse = ({ doc_count: 62606, avg: { value: 2651.8784461553205 }, p95: { values: { '95.0': 6140.579335038363 } }, + durationSum: { value: 166023502 }, sample: { hits: { total: 62606, @@ -2596,6 +2618,7 @@ export const transactionGroupsResponse = ({ doc_count: 7487, avg: { value: 1422.926672899693 }, p95: { values: { '95.0': 2311.885238095238 } }, + durationSum: { value: 10653452 }, sample: { hits: { total: 7487, diff --git a/x-pack/plugins/apm/server/lib/transaction_groups/transform.test.ts b/x-pack/plugins/apm/server/lib/transaction_groups/transform.test.ts index 73053aa04643d..bd23fc3c60d7a 100644 --- a/x-pack/plugins/apm/server/lib/transaction_groups/transform.test.ts +++ b/x-pack/plugins/apm/server/lib/transaction_groups/transform.test.ts @@ -19,12 +19,13 @@ describe('transactionGroupsTransformer', () => { ).toMatchSnapshot(); }); - fit('should transform response correctly', () => { + it('should transform response correctly', () => { const bucket = { key: 'POST /api/orders', doc_count: 180, avg: { value: 255966.30555555556 }, p95: { values: { '95.0': 320238.5 } }, + durationSum: { value: 3000000000 }, sample: { hits: { total: 180, @@ -47,6 +48,7 @@ describe('transactionGroupsTransformer', () => { { averageResponseTime: 255966.30555555556, impact: 0, + absoluteImpact: 3000000000, name: 'POST /api/orders', p95: 320238.5, sample: 'sample source', @@ -54,4 +56,27 @@ describe('transactionGroupsTransformer', () => { } ]); }); + + it('should calculate relative impact from durationSum', () => { + const getBucket = (durationSum: number) => ({ + key: 'POST /api/orders', + doc_count: 180, + avg: { value: 300000 }, + p95: { values: { '95.0': 320000 } }, + durationSum: { value: durationSum }, + sample: { hits: { total: 180, hits: [{ _source: 'sample source' }] } } + }); + + const response = ({ + aggregations: { + transactions: { buckets: [getBucket(10), getBucket(20), getBucket(50)] } + } + } as unknown) as ESResponse; + + expect( + transactionGroupsTransformer({ response, start: 100, end: 20000 }).map( + bucket => bucket.impact + ) + ).toEqual([0, 25, 100]); + }); }); diff --git a/x-pack/plugins/apm/server/lib/transaction_groups/transform.ts b/x-pack/plugins/apm/server/lib/transaction_groups/transform.ts index 229134025b18f..cde3db16c5b5e 100644 --- a/x-pack/plugins/apm/server/lib/transaction_groups/transform.ts +++ b/x-pack/plugins/apm/server/lib/transaction_groups/transform.ts @@ -16,16 +16,17 @@ export interface ITransactionGroup { averageResponseTime: number; transactionsPerMinute: number; impact: number; + absoluteImpact: number; } function calculateRelativeImpacts(results: ITransactionGroup[]) { - const values = results.map(({ impact }) => impact); + const values = results.map(({ absoluteImpact }) => absoluteImpact); const max = Math.max(...values); const min = Math.min(...values); return results.map(bucket => ({ ...bucket, - impact: ((bucket.impact - min) / (max - min)) * 100 || 0 + impact: ((bucket.absoluteImpact - min) / (max - min)) * 100 || 0 })); } @@ -44,7 +45,6 @@ export function transactionGroupsTransformer({ const results = buckets.map(bucket => { const averageResponseTime = bucket.avg.value; const transactionsPerMinute = bucket.doc_count / minutes; - const impact = Math.round(averageResponseTime * transactionsPerMinute); const sample = bucket.sample.hits.hits[0]._source; return { @@ -53,7 +53,8 @@ export function transactionGroupsTransformer({ p95: bucket.p95.values['95.0'], averageResponseTime, transactionsPerMinute, - impact + impact: 0, + absoluteImpact: bucket.durationSum.value }; });