From f57518a48f28df9e77896bdf1c5e1cdb4612829a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Louv-Jansen?= Date: Tue, 20 Oct 2020 12:19:54 +0200 Subject: [PATCH] [APM] Add correlations API (#78882) --- .../components/app/Correlations/index.tsx | 85 +++++++++++ .../components/app/ServiceOverview/index.tsx | 4 + .../app/TransactionDetails/index.tsx | 3 + .../app/TransactionOverview/index.tsx | 2 + .../apm/server/lib/helpers/setup_request.ts | 8 + .../get_correlations_for_ranges.ts | 90 ++++++++++++ .../get_correlations_for_slow_transactions.ts | 94 ++++++++++++ .../get_duration_for_percentile.ts | 43 ++++++ .../correlations/get_significant_terms_agg.ts | 68 +++++++++ .../correlations/scoring_rt.ts | 16 ++ .../plugins/apm/server/routes/correlations.ts | 101 +++++++++++++ .../apm/server/routes/create_apm_api.ts | 8 + .../apm/typings/elasticsearch/aggregations.ts | 16 ++ .../basic/tests/correlations/ranges.ts | 96 ++++++++++++ .../tests/correlations/slow_durations.ts | 138 ++++++++++++++++++ .../apm_api_integration/basic/tests/index.ts | 5 + 16 files changed, 777 insertions(+) create mode 100644 x-pack/plugins/apm/public/components/app/Correlations/index.tsx create mode 100644 x-pack/plugins/apm/server/lib/transaction_groups/correlations/get_correlations_for_ranges.ts create mode 100644 x-pack/plugins/apm/server/lib/transaction_groups/correlations/get_correlations_for_slow_transactions.ts create mode 100644 x-pack/plugins/apm/server/lib/transaction_groups/correlations/get_duration_for_percentile.ts create mode 100644 x-pack/plugins/apm/server/lib/transaction_groups/correlations/get_significant_terms_agg.ts create mode 100644 x-pack/plugins/apm/server/lib/transaction_groups/correlations/scoring_rt.ts create mode 100644 x-pack/plugins/apm/server/routes/correlations.ts create mode 100644 x-pack/test/apm_api_integration/basic/tests/correlations/ranges.ts create mode 100644 x-pack/test/apm_api_integration/basic/tests/correlations/slow_durations.ts diff --git a/x-pack/plugins/apm/public/components/app/Correlations/index.tsx b/x-pack/plugins/apm/public/components/app/Correlations/index.tsx new file mode 100644 index 0000000000000..afee2b9f5e881 --- /dev/null +++ b/x-pack/plugins/apm/public/components/app/Correlations/index.tsx @@ -0,0 +1,85 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React from 'react'; +import url from 'url'; +import { useParams } from 'react-router-dom'; +import { useLocation } from 'react-router-dom'; +import { EuiTitle, EuiListGroup } from '@elastic/eui'; +import { useUrlParams } from '../../../hooks/useUrlParams'; +import { useApmPluginContext } from '../../../hooks/useApmPluginContext'; + +const SESSION_STORAGE_KEY = 'apm.debug.show_correlations'; + +export function Correlations() { + const location = useLocation(); + const { serviceName } = useParams<{ serviceName?: string }>(); + const { urlParams, uiFilters } = useUrlParams(); + const { core } = useApmPluginContext(); + const { transactionName, transactionType, start, end } = urlParams; + + if ( + !location.search.includes('&_show_correlations') && + sessionStorage.getItem(SESSION_STORAGE_KEY) !== 'true' + ) { + return null; + } + + sessionStorage.setItem(SESSION_STORAGE_KEY, 'true'); + + const query = { + serviceName, + transactionName, + transactionType, + start, + end, + uiFilters: JSON.stringify(uiFilters), + fieldNames: + 'user.username,user.id,host.ip,user_agent.name,kubernetes.pod.uuid,url.domain,container.id,service.node.name', + }; + + const listItems = [ + { + label: 'Show correlations between two ranges', + href: url.format({ + query: { + ...query, + gap: 24, + }, + pathname: core.http.basePath.prepend(`/api/apm/correlations/ranges`), + }), + isDisabled: false, + iconType: 'tokenRange', + size: 's' as const, + }, + + { + label: 'Show correlations for slow transactions', + href: url.format({ + query: { + ...query, + durationPercentile: 95, + }, + pathname: core.http.basePath.prepend( + `/api/apm/correlations/slow_durations` + ), + }), + isDisabled: false, + iconType: 'clock', + size: 's' as const, + }, + ]; + + return ( + <> + +

Correlations

+ + + + + ); +} diff --git a/x-pack/plugins/apm/public/components/app/ServiceOverview/index.tsx b/x-pack/plugins/apm/public/components/app/ServiceOverview/index.tsx index bc4a3212bafe2..fba5df5b16a4a 100644 --- a/x-pack/plugins/apm/public/components/app/ServiceOverview/index.tsx +++ b/x-pack/plugins/apm/public/components/app/ServiceOverview/index.tsx @@ -21,6 +21,7 @@ import { useApmPluginContext } from '../../../hooks/useApmPluginContext'; import { MLCallout } from './ServiceList/MLCallout'; import { useLocalStorage } from '../../../hooks/useLocalStorage'; import { useAnomalyDetectionJobs } from '../../../hooks/useAnomalyDetectionJobs'; +import { Correlations } from '../Correlations'; const initialData = { items: [], @@ -117,6 +118,9 @@ export function ServiceOverview() { return ( <> + + + diff --git a/x-pack/plugins/apm/public/components/app/TransactionDetails/index.tsx b/x-pack/plugins/apm/public/components/app/TransactionDetails/index.tsx index e31a2b24f1d15..b79186a90cd1d 100644 --- a/x-pack/plugins/apm/public/components/app/TransactionDetails/index.tsx +++ b/x-pack/plugins/apm/public/components/app/TransactionDetails/index.tsx @@ -31,6 +31,7 @@ import { fromQuery, toQuery } from '../../shared/Links/url_helpers'; import { useUrlParams } from '../../../hooks/useUrlParams'; import { LocalUIFilters } from '../../shared/LocalUIFilters'; import { HeightRetainer } from '../../shared/HeightRetainer'; +import { Correlations } from '../Correlations'; interface Sample { traceId: string; @@ -111,6 +112,8 @@ export function TransactionDetails({ + + diff --git a/x-pack/plugins/apm/public/components/app/TransactionOverview/index.tsx b/x-pack/plugins/apm/public/components/app/TransactionOverview/index.tsx index 8c7d088d36eb2..003df632d11b3 100644 --- a/x-pack/plugins/apm/public/components/app/TransactionOverview/index.tsx +++ b/x-pack/plugins/apm/public/components/app/TransactionOverview/index.tsx @@ -37,6 +37,7 @@ import { TransactionList } from './TransactionList'; import { useRedirect } from './useRedirect'; import { TRANSACTION_PAGE_LOAD } from '../../../../common/transaction_types'; import { UserExperienceCallout } from './user_experience_callout'; +import { Correlations } from '../Correlations'; function getRedirectLocation({ urlParams, @@ -117,6 +118,7 @@ export function TransactionOverview({ serviceName }: TransactionOverviewProps) { return ( <> + diff --git a/x-pack/plugins/apm/server/lib/helpers/setup_request.ts b/x-pack/plugins/apm/server/lib/helpers/setup_request.ts index 26896a050dd88..a8a128937fb1c 100644 --- a/x-pack/plugins/apm/server/lib/helpers/setup_request.ts +++ b/x-pack/plugins/apm/server/lib/helpers/setup_request.ts @@ -49,7 +49,15 @@ export interface SetupTimeRange { interface SetupRequestParams { query?: { _debug?: boolean; + + /** + * Timestamp in ms since epoch + */ start?: string; + + /** + * Timestamp in ms since epoch + */ end?: string; uiFilters?: string; processorEvent?: ProcessorEvent; diff --git a/x-pack/plugins/apm/server/lib/transaction_groups/correlations/get_correlations_for_ranges.ts b/x-pack/plugins/apm/server/lib/transaction_groups/correlations/get_correlations_for_ranges.ts new file mode 100644 index 0000000000000..3cf0271baa1c6 --- /dev/null +++ b/x-pack/plugins/apm/server/lib/transaction_groups/correlations/get_correlations_for_ranges.ts @@ -0,0 +1,90 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { rangeFilter } from '../../../../common/utils/range_filter'; +import { + SERVICE_NAME, + TRANSACTION_NAME, + TRANSACTION_TYPE, +} from '../../../../common/elasticsearch_fieldnames'; +import { ProcessorEvent } from '../../../../common/processor_event'; +import { Setup, SetupTimeRange } from '../../helpers/setup_request'; +import { + getSignificantTermsAgg, + formatAggregationResponse, +} from './get_significant_terms_agg'; +import { SignificantTermsScoring } from './scoring_rt'; + +export async function getCorrelationsForRanges({ + serviceName, + transactionType, + transactionName, + scoring, + gapBetweenRanges, + fieldNames, + setup, +}: { + serviceName: string | undefined; + transactionType: string | undefined; + transactionName: string | undefined; + scoring: SignificantTermsScoring; + gapBetweenRanges: number; + fieldNames: string[]; + setup: Setup & SetupTimeRange; +}) { + const { start, end, esFilter, apmEventClient } = setup; + + const baseFilters = [...esFilter]; + + if (serviceName) { + baseFilters.push({ term: { [SERVICE_NAME]: serviceName } }); + } + + if (transactionType) { + baseFilters.push({ term: { [TRANSACTION_TYPE]: transactionType } }); + } + + if (transactionName) { + baseFilters.push({ term: { [TRANSACTION_NAME]: transactionName } }); + } + + const diff = end - start + gapBetweenRanges; + const baseRangeStart = start - diff; + const baseRangeEnd = end - diff; + const backgroundFilters = [ + ...baseFilters, + { range: rangeFilter(baseRangeStart, baseRangeEnd) }, + ]; + + const params = { + apm: { events: [ProcessorEvent.transaction] }, + body: { + size: 0, + query: { + bool: { filter: [...baseFilters, { range: rangeFilter(start, end) }] }, + }, + aggs: getSignificantTermsAgg({ + fieldNames, + backgroundFilters, + backgroundIsSuperset: false, + scoring, + }), + }, + }; + + const response = await apmEventClient.search(params); + + return { + message: `Showing significant fields between the ranges`, + firstRange: `${new Date(baseRangeStart).toISOString()} - ${new Date( + baseRangeEnd + ).toISOString()}`, + lastRange: `${new Date(start).toISOString()} - ${new Date( + end + ).toISOString()}`, + response: formatAggregationResponse(response.aggregations), + }; +} diff --git a/x-pack/plugins/apm/server/lib/transaction_groups/correlations/get_correlations_for_slow_transactions.ts b/x-pack/plugins/apm/server/lib/transaction_groups/correlations/get_correlations_for_slow_transactions.ts new file mode 100644 index 0000000000000..3efc65afdfd28 --- /dev/null +++ b/x-pack/plugins/apm/server/lib/transaction_groups/correlations/get_correlations_for_slow_transactions.ts @@ -0,0 +1,94 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { asDuration } from '../../../../common/utils/formatters'; +import { ESFilter } from '../../../../typings/elasticsearch'; +import { rangeFilter } from '../../../../common/utils/range_filter'; +import { + SERVICE_NAME, + TRANSACTION_DURATION, + TRANSACTION_NAME, + TRANSACTION_TYPE, +} from '../../../../common/elasticsearch_fieldnames'; +import { ProcessorEvent } from '../../../../common/processor_event'; +import { Setup, SetupTimeRange } from '../../helpers/setup_request'; +import { getDurationForPercentile } from './get_duration_for_percentile'; +import { + formatAggregationResponse, + getSignificantTermsAgg, +} from './get_significant_terms_agg'; +import { SignificantTermsScoring } from './scoring_rt'; + +export async function getCorrelationsForSlowTransactions({ + serviceName, + transactionType, + transactionName, + durationPercentile, + fieldNames, + scoring, + setup, +}: { + serviceName: string | undefined; + transactionType: string | undefined; + transactionName: string | undefined; + scoring: SignificantTermsScoring; + durationPercentile: number; + fieldNames: string[]; + setup: Setup & SetupTimeRange; +}) { + const { start, end, esFilter, apmEventClient } = setup; + + const backgroundFilters: ESFilter[] = [ + ...esFilter, + { range: rangeFilter(start, end) }, + ]; + + if (serviceName) { + backgroundFilters.push({ term: { [SERVICE_NAME]: serviceName } }); + } + + if (transactionType) { + backgroundFilters.push({ term: { [TRANSACTION_TYPE]: transactionType } }); + } + + if (transactionName) { + backgroundFilters.push({ term: { [TRANSACTION_NAME]: transactionName } }); + } + + const durationForPercentile = await getDurationForPercentile({ + durationPercentile, + backgroundFilters, + setup, + }); + + const params = { + apm: { events: [ProcessorEvent.transaction] }, + body: { + size: 0, + query: { + bool: { + // foreground filters + filter: [ + ...backgroundFilters, + { + range: { [TRANSACTION_DURATION]: { gte: durationForPercentile } }, + }, + ], + }, + }, + aggs: getSignificantTermsAgg({ fieldNames, backgroundFilters, scoring }), + }, + }; + + const response = await apmEventClient.search(params); + + return { + message: `Showing significant fields for transactions slower than ${durationPercentile}th percentile (${asDuration( + durationForPercentile + )})`, + response: formatAggregationResponse(response.aggregations), + }; +} diff --git a/x-pack/plugins/apm/server/lib/transaction_groups/correlations/get_duration_for_percentile.ts b/x-pack/plugins/apm/server/lib/transaction_groups/correlations/get_duration_for_percentile.ts new file mode 100644 index 0000000000000..37ee19ff40f62 --- /dev/null +++ b/x-pack/plugins/apm/server/lib/transaction_groups/correlations/get_duration_for_percentile.ts @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { ESFilter } from '../../../../typings/elasticsearch'; +import { TRANSACTION_DURATION } from '../../../../common/elasticsearch_fieldnames'; +import { ProcessorEvent } from '../../../../common/processor_event'; +import { Setup, SetupTimeRange } from '../../helpers/setup_request'; + +export async function getDurationForPercentile({ + durationPercentile, + backgroundFilters, + setup, +}: { + durationPercentile: number; + backgroundFilters: ESFilter[]; + setup: Setup & SetupTimeRange; +}) { + const { apmEventClient } = setup; + const res = await apmEventClient.search({ + apm: { + events: [ProcessorEvent.transaction], + }, + body: { + size: 0, + query: { + bool: { filter: backgroundFilters }, + }, + aggs: { + percentile: { + percentiles: { + field: TRANSACTION_DURATION, + percents: [durationPercentile], + }, + }, + }, + }, + }); + + return Object.values(res.aggregations?.percentile.values || {})[0]; +} diff --git a/x-pack/plugins/apm/server/lib/transaction_groups/correlations/get_significant_terms_agg.ts b/x-pack/plugins/apm/server/lib/transaction_groups/correlations/get_significant_terms_agg.ts new file mode 100644 index 0000000000000..1cf0787c1d970 --- /dev/null +++ b/x-pack/plugins/apm/server/lib/transaction_groups/correlations/get_significant_terms_agg.ts @@ -0,0 +1,68 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { ESFilter } from '../../../../typings/elasticsearch'; +import { SignificantTermsScoring } from './scoring_rt'; + +export function getSignificantTermsAgg({ + fieldNames, + backgroundFilters, + backgroundIsSuperset = true, + scoring = 'percentage', +}: { + fieldNames: string[]; + backgroundFilters: ESFilter[]; + backgroundIsSuperset?: boolean; + scoring: SignificantTermsScoring; +}) { + return fieldNames.reduce((acc, fieldName) => { + return { + ...acc, + [fieldName]: { + significant_terms: { + size: 10, + field: fieldName, + background_filter: { bool: { filter: backgroundFilters } }, + + // indicate whether background is a superset of the foreground + mutual_information: { background_is_superset: backgroundIsSuperset }, + + // different scorings https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-bucket-significantterms-aggregation.html#significantterms-aggregation-parameters + [scoring]: {}, + min_doc_count: 5, + shard_min_doc_count: 5, + }, + }, + [`cardinality-${fieldName}`]: { + cardinality: { field: fieldName }, + }, + }; + }, {} as Record); +} + +export function formatAggregationResponse(aggs?: Record) { + if (!aggs) { + return; + } + + return Object.entries(aggs).reduce((acc, [key, value]) => { + if (key.startsWith('cardinality-')) { + if (value.value > 0) { + const fieldName = key.slice(12); + acc[fieldName] = { + ...acc[fieldName], + cardinality: value.value, + }; + } + } else if (value.buckets.length > 0) { + acc[key] = { + ...acc[key], + value, + }; + } + return acc; + }, {} as Record); +} diff --git a/x-pack/plugins/apm/server/lib/transaction_groups/correlations/scoring_rt.ts b/x-pack/plugins/apm/server/lib/transaction_groups/correlations/scoring_rt.ts new file mode 100644 index 0000000000000..cb94b6251eb07 --- /dev/null +++ b/x-pack/plugins/apm/server/lib/transaction_groups/correlations/scoring_rt.ts @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import * as t from 'io-ts'; + +export const scoringRt = t.union([ + t.literal('jlh'), + t.literal('chi_square'), + t.literal('gnd'), + t.literal('percentage'), +]); + +export type SignificantTermsScoring = t.TypeOf; diff --git a/x-pack/plugins/apm/server/routes/correlations.ts b/x-pack/plugins/apm/server/routes/correlations.ts new file mode 100644 index 0000000000000..5f8d2afd544f3 --- /dev/null +++ b/x-pack/plugins/apm/server/routes/correlations.ts @@ -0,0 +1,101 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import * as t from 'io-ts'; +import { rangeRt } from './default_api_types'; +import { getCorrelationsForSlowTransactions } from '../lib/transaction_groups/correlations/get_correlations_for_slow_transactions'; +import { getCorrelationsForRanges } from '../lib/transaction_groups/correlations/get_correlations_for_ranges'; +import { scoringRt } from '../lib/transaction_groups/correlations/scoring_rt'; +import { createRoute } from './create_route'; +import { setupRequest } from '../lib/helpers/setup_request'; + +export const correlationsForSlowTransactionsRoute = createRoute(() => ({ + path: '/api/apm/correlations/slow_durations', + params: { + query: t.intersection([ + t.partial({ + serviceName: t.string, + transactionName: t.string, + transactionType: t.string, + scoring: scoringRt, + }), + t.type({ + durationPercentile: t.string, + fieldNames: t.string, + }), + t.partial({ uiFilters: t.string }), + rangeRt, + ]), + }, + handler: async ({ context, request }) => { + const setup = await setupRequest(context, request); + const { + serviceName, + transactionType, + transactionName, + durationPercentile, + fieldNames, + scoring = 'percentage', + } = context.params.query; + + return getCorrelationsForSlowTransactions({ + serviceName, + transactionType, + transactionName, + durationPercentile: parseInt(durationPercentile, 10), + fieldNames: fieldNames.split(','), + scoring, + setup, + }); + }, +})); + +export const correlationsForRangesRoute = createRoute(() => ({ + path: '/api/apm/correlations/ranges', + params: { + query: t.intersection([ + t.partial({ + serviceName: t.string, + transactionName: t.string, + transactionType: t.string, + scoring: scoringRt, + gap: t.string, + }), + t.type({ + fieldNames: t.string, + }), + t.partial({ uiFilters: t.string }), + rangeRt, + ]), + }, + handler: async ({ context, request }) => { + const setup = await setupRequest(context, request); + + const { + serviceName, + transactionType, + transactionName, + scoring = 'percentage', + gap, + fieldNames, + } = context.params.query; + + const gapBetweenRanges = parseInt(gap || '0', 10) * 3600 * 1000; + if (gapBetweenRanges < 0) { + throw new Error('gap must be 0 or positive'); + } + + return getCorrelationsForRanges({ + serviceName, + transactionType, + transactionName, + scoring, + gapBetweenRanges, + fieldNames: fieldNames.split(','), + setup, + }); + }, +})); diff --git a/x-pack/plugins/apm/server/routes/create_apm_api.ts b/x-pack/plugins/apm/server/routes/create_apm_api.ts index c1f13ee646e49..2fbe404a70d82 100644 --- a/x-pack/plugins/apm/server/routes/create_apm_api.ts +++ b/x-pack/plugins/apm/server/routes/create_apm_api.ts @@ -41,6 +41,10 @@ import { metricsChartsRoute } from './metrics'; import { serviceNodesRoute } from './service_nodes'; import { tracesRoute, tracesByIdRoute } from './traces'; import { transactionByTraceIdRoute } from './transaction'; +import { + correlationsForRangesRoute, + correlationsForSlowTransactionsRoute, +} from './correlations'; import { transactionGroupsBreakdownRoute, transactionGroupsChartsRoute, @@ -122,6 +126,10 @@ const createApmApi = () => { .add(listAgentConfigurationServicesRoute) .add(createOrUpdateAgentConfigurationRoute) + // Correlations + .add(correlationsForSlowTransactionsRoute) + .add(correlationsForRangesRoute) + // APM indices .add(apmIndexSettingsRoute) .add(apmIndicesRoute) diff --git a/x-pack/plugins/apm/typings/elasticsearch/aggregations.ts b/x-pack/plugins/apm/typings/elasticsearch/aggregations.ts index 534321201938d..8b3163e44915a 100644 --- a/x-pack/plugins/apm/typings/elasticsearch/aggregations.ts +++ b/x-pack/plugins/apm/typings/elasticsearch/aggregations.ts @@ -158,6 +158,11 @@ export interface AggregationOptionsByType { from?: number; size?: number; }; + significant_terms: { + size?: number; + field?: string; + background_filter?: Record; + } & AggregationSourceOptions; } type AggregationType = keyof AggregationOptionsByType; @@ -334,6 +339,17 @@ interface AggregationResponsePart< ? Array<{ key: number; value: number }> : Record; }; + significant_terms: { + doc_count: number; + bg_count: number; + buckets: Array< + { + bg_count: number; + doc_count: number; + key: string | number; + } & SubAggregationResponseOf + >; + }; bucket_sort: undefined; } diff --git a/x-pack/test/apm_api_integration/basic/tests/correlations/ranges.ts b/x-pack/test/apm_api_integration/basic/tests/correlations/ranges.ts new file mode 100644 index 0000000000000..f013520fa163b --- /dev/null +++ b/x-pack/test/apm_api_integration/basic/tests/correlations/ranges.ts @@ -0,0 +1,96 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import expect from '@kbn/expect'; +import { format } from 'url'; +import { expectSnapshot } from '../../../common/match_snapshot'; +import { PromiseReturnType } from '../../../../../plugins/apm/typings/common'; +import { FtrProviderContext } from '../../../common/ftr_provider_context'; +import archives_metadata from '../../../common/archives_metadata'; + +export default function ApiTest({ getService }: FtrProviderContext) { + const supertest = getService('supertest'); + const esArchiver = getService('esArchiver'); + const archiveName = 'apm_8.0.0'; + const range = archives_metadata[archiveName]; + + // url parameters + const start = '2020-09-29T14:45:00.000Z'; + const end = range.end; + const fieldNames = + 'user.username,user.id,host.ip,user_agent.name,kubernetes.pod.uuid,url.domain,container.id,service.node.name'; + + describe('Ranges', () => { + const url = format({ + pathname: `/api/apm/correlations/ranges`, + query: { start, end, fieldNames }, + }); + + describe('when data is not loaded ', () => { + it('handles the empty state', async () => { + const response = await supertest.get(url); + expect(response.status).to.be(200); + expect(response.body.response).to.be(undefined); + }); + }); + + describe('when data is loaded', () => { + let response: PromiseReturnType; + before(async () => { + await esArchiver.load(archiveName); + response = await supertest.get(url); + }); + + after(() => esArchiver.unload(archiveName)); + + it('returns successfully', () => { + expect(response.status).to.eql(200); + }); + + it('returns fields in response', () => { + expectSnapshot(Object.keys(response.body.response)).toMatchInline(` + Array [ + "service.node.name", + "host.ip", + "user.id", + "user_agent.name", + "container.id", + "url.domain", + ] + `); + }); + + it('returns cardinality for each field', () => { + const cardinalitys = Object.values(response.body.response).map( + (field: any) => field.cardinality + ); + + expectSnapshot(cardinalitys).toMatchInline(` + Array [ + 5, + 6, + 20, + 6, + 5, + 4, + ] + `); + }); + + it('returns buckets', () => { + const { buckets } = response.body.response['user.id'].value; + expectSnapshot(buckets[0]).toMatchInline(` + Object { + "bg_count": 2, + "doc_count": 7, + "key": "20", + "score": 3.5, + } + `); + }); + }); + }); +} diff --git a/x-pack/test/apm_api_integration/basic/tests/correlations/slow_durations.ts b/x-pack/test/apm_api_integration/basic/tests/correlations/slow_durations.ts new file mode 100644 index 0000000000000..78dca5100dece --- /dev/null +++ b/x-pack/test/apm_api_integration/basic/tests/correlations/slow_durations.ts @@ -0,0 +1,138 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import expect from '@kbn/expect'; +import { format } from 'url'; +import { PromiseReturnType } from '../../../../../plugins/apm/typings/common'; +import { FtrProviderContext } from '../../../common/ftr_provider_context'; +import { expectSnapshot } from '../../../common/match_snapshot'; +import archives_metadata from '../../../common/archives_metadata'; + +export default function ApiTest({ getService }: FtrProviderContext) { + const supertest = getService('supertest'); + const esArchiver = getService('esArchiver'); + const archiveName = 'apm_8.0.0'; + const range = archives_metadata[archiveName]; + + // url parameters + const start = range.start; + const end = range.end; + const durationPercentile = 95; + const fieldNames = + 'user.username,user.id,host.ip,user_agent.name,kubernetes.pod.uuid,url.domain,container.id,service.node.name'; + + describe('Slow durations', () => { + const url = format({ + pathname: `/api/apm/correlations/slow_durations`, + query: { start, end, durationPercentile, fieldNames }, + }); + + describe('when data is not loaded ', () => { + it('handles the empty state', async () => { + const response = await supertest.get(url); + + expect(response.status).to.be(200); + expect(response.body.response).to.be(undefined); + }); + }); + + describe('with default scoring', () => { + let response: PromiseReturnType; + before(async () => { + await esArchiver.load(archiveName); + response = await supertest.get(url); + }); + + after(() => esArchiver.unload(archiveName)); + + it('returns successfully', () => { + expect(response.status).to.eql(200); + }); + + it('returns fields in response', () => { + expectSnapshot(Object.keys(response.body.response)).toMatchInline(` + Array [ + "service.node.name", + "host.ip", + "user.id", + "user_agent.name", + "container.id", + "url.domain", + ] + `); + }); + + it('returns cardinality for each field', () => { + const cardinalitys = Object.values(response.body.response).map( + (field: any) => field.cardinality + ); + + expectSnapshot(cardinalitys).toMatchInline(` + Array [ + 5, + 6, + 3, + 5, + 5, + 4, + ] + `); + }); + + it('returns buckets', () => { + const { buckets } = response.body.response['user.id'].value; + expectSnapshot(buckets[0]).toMatchInline(` + Object { + "bg_count": 32, + "doc_count": 6, + "key": "2", + "score": 0.1875, + } + `); + }); + }); + + describe('with different scoring', () => { + before(async () => esArchiver.load(archiveName)); + after(() => esArchiver.unload(archiveName)); + + it(`returns buckets for each score`, async () => { + const promises = ['percentage', 'jlh', 'chi_square', 'gnd'].map(async (scoring) => { + const response = await supertest.get( + format({ + pathname: `/api/apm/correlations/slow_durations`, + query: { start, end, durationPercentile, fieldNames, scoring }, + }) + ); + + return { name: scoring, value: response.body.response['user.id'].value.buckets[0].score }; + }); + + const res = await Promise.all(promises); + expectSnapshot(res).toMatchInline(` + Array [ + Object { + "name": "percentage", + "value": 0.1875, + }, + Object { + "name": "jlh", + "value": 3.33506905769659, + }, + Object { + "name": "chi_square", + "value": 219.192006524483, + }, + Object { + "name": "gnd", + "value": 0.671406580688819, + }, + ] + `); + }); + }); + }); +} diff --git a/x-pack/test/apm_api_integration/basic/tests/index.ts b/x-pack/test/apm_api_integration/basic/tests/index.ts index 19dd82d617bd9..df3e60d79aca5 100644 --- a/x-pack/test/apm_api_integration/basic/tests/index.ts +++ b/x-pack/test/apm_api_integration/basic/tests/index.ts @@ -56,5 +56,10 @@ export default function apmApiIntegrationTests({ loadTestFile }: FtrProviderCont describe('Metrics', function () { loadTestFile(require.resolve('./metrics_charts/metrics_charts')); }); + + describe('Correlations', function () { + loadTestFile(require.resolve('./correlations/slow_durations')); + loadTestFile(require.resolve('./correlations/ranges')); + }); }); }