diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/async_search_service.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/async_search_service.ts index 5820fd952c449..7a511fc60fd06 100644 --- a/x-pack/plugins/apm/server/lib/search_strategies/correlations/async_search_service.ts +++ b/x-pack/plugins/apm/server/lib/search_strategies/correlations/async_search_service.ts @@ -9,7 +9,7 @@ import { shuffle, range } from 'lodash'; import type { ElasticsearchClient } from 'src/core/server'; import { fetchTransactionDurationFieldCandidates } from './query_field_candidates'; import { fetchTransactionDurationFieldValuePairs } from './query_field_value_pairs'; -import { fetchTransactionDurationPecentiles } from './query_percentiles'; +import { fetchTransactionDurationPercentiles } from './query_percentiles'; import { fetchTransactionDurationCorrelation } from './query_correlation'; import { fetchTransactionDurationHistogramRangesteps } from './query_histogram_rangesteps'; import { fetchTransactionDurationRanges, HistogramItem } from './query_ranges'; @@ -59,7 +59,7 @@ export const asyncSearchServiceProvider = ( const fetchCorrelations = async () => { try { // 95th percentile to be displayed as a marker in the log log chart - const percentileThreshold = await fetchTransactionDurationPecentiles( + const percentileThreshold = await fetchTransactionDurationPercentiles( esClient, params, params.percentileThreshold ? [params.percentileThreshold] : undefined @@ -93,7 +93,7 @@ export const asyncSearchServiceProvider = ( // Create an array of ranges [2, 4, 6, ..., 98] const percents = Array.from(range(2, 100, 2)); - const percentilesRecords = await fetchTransactionDurationPecentiles( + const percentilesRecords = await fetchTransactionDurationPercentiles( esClient, params, percents diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/get_query_with_params.test.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/get_query_with_params.test.ts new file mode 100644 index 0000000000000..12e897ab3eec9 --- /dev/null +++ b/x-pack/plugins/apm/server/lib/search_strategies/correlations/get_query_with_params.test.ts @@ -0,0 +1,92 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { getQueryWithParams } from './get_query_with_params'; + +describe('correlations', () => { + describe('getQueryWithParams', () => { + it('returns the most basic query filtering on processor.event=transaction', () => { + const query = getQueryWithParams({ params: { index: 'apm-*' } }); + expect(query).toEqual({ + bool: { + filter: [{ term: { 'processor.event': 'transaction' } }], + }, + }); + }); + + it('returns a query considering additional params', () => { + const query = getQueryWithParams({ + params: { + index: 'apm-*', + serviceName: 'actualServiceName', + transactionName: 'actualTransactionName', + start: '01-01-2021', + end: '31-01-2021', + environment: 'dev', + percentileThresholdValue: 75, + }, + }); + expect(query).toEqual({ + bool: { + filter: [ + { term: { 'processor.event': 'transaction' } }, + { + term: { + 'service.name': 'actualServiceName', + }, + }, + { + term: { + 'transaction.name': 'actualTransactionName', + }, + }, + { + range: { + '@timestamp': { + gte: '01-01-2021', + lte: '31-01-2021', + }, + }, + }, + { + term: { + 'service.environment': 'dev', + }, + }, + { + range: { + 'transaction.duration.us': { + gte: 75, + }, + }, + }, + ], + }, + }); + }); + + it('returns a query considering a custom field/value pair', () => { + const query = getQueryWithParams({ + params: { index: 'apm-*' }, + fieldName: 'actualFieldName', + fieldValue: 'actualFieldValue', + }); + expect(query).toEqual({ + bool: { + filter: [ + { term: { 'processor.event': 'transaction' } }, + { + term: { + actualFieldName: 'actualFieldValue', + }, + }, + ], + }, + }); + }); + }); +}); diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/get_query_with_params.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/get_query_with_params.ts index e7cf8173b5bac..08ba4b23fec35 100644 --- a/x-pack/plugins/apm/server/lib/search_strategies/correlations/get_query_with_params.ts +++ b/x-pack/plugins/apm/server/lib/search_strategies/correlations/get_query_with_params.ts @@ -43,6 +43,10 @@ const getRangeQuery = ( start?: string, end?: string ): estypes.QueryDslQueryContainer[] => { + if (start === undefined && end === undefined) { + return []; + } + return [ { range: { diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_correlation.test.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_correlation.test.ts new file mode 100644 index 0000000000000..24741ebaa2dae --- /dev/null +++ b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_correlation.test.ts @@ -0,0 +1,103 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { estypes } from '@elastic/elasticsearch'; + +import type { ElasticsearchClient } from 'src/core/server'; + +import { + fetchTransactionDurationCorrelation, + getTransactionDurationCorrelationRequest, + BucketCorrelation, +} from './query_correlation'; + +const params = { index: 'apm-*' }; +const expectations = [1, 3, 5]; +const ranges = [{ to: 1 }, { from: 1, to: 3 }, { from: 3, to: 5 }, { from: 5 }]; +const fractions = [1, 2, 4, 5]; +const totalDocCount = 1234; + +describe('query_correlation', () => { + describe('getTransactionDurationCorrelationRequest', () => { + it('applies options to the returned query with aggregations for correlations and k-test', () => { + const query = getTransactionDurationCorrelationRequest( + params, + expectations, + ranges, + fractions, + totalDocCount + ); + + expect(query.index).toBe(params.index); + + expect(query?.body?.aggs?.latency_ranges?.range?.field).toBe( + 'transaction.duration.us' + ); + expect(query?.body?.aggs?.latency_ranges?.range?.ranges).toEqual(ranges); + + expect( + (query?.body?.aggs?.transaction_duration_correlation as { + bucket_correlation: BucketCorrelation; + })?.bucket_correlation.function.count_correlation.indicator + ).toEqual({ + fractions, + expectations, + doc_count: totalDocCount, + }); + + expect( + (query?.body?.aggs?.ks_test as any)?.bucket_count_ks_test?.fractions + ).toEqual(fractions); + }); + }); + + describe('fetchTransactionDurationCorrelation', () => { + it('returns the data from the aggregations', async () => { + const latencyRangesBuckets = [{ to: 1 }, { from: 1, to: 2 }, { from: 2 }]; + const transactionDurationCorrelationValue = 0.45; + const KsTestLess = 0.01; + + const esClientSearchMock = jest.fn((req: estypes.SearchRequest): { + body: estypes.SearchResponse; + } => { + return { + body: ({ + aggregations: { + latency_ranges: { + buckets: latencyRangesBuckets, + }, + transaction_duration_correlation: { + value: transactionDurationCorrelationValue, + }, + ks_test: { less: KsTestLess }, + }, + } as unknown) as estypes.SearchResponse, + }; + }); + + const esClientMock = ({ + search: esClientSearchMock, + } as unknown) as ElasticsearchClient; + + const resp = await fetchTransactionDurationCorrelation( + esClientMock, + params, + expectations, + ranges, + fractions, + totalDocCount + ); + + expect(resp).toEqual({ + correlation: transactionDurationCorrelationValue, + ksTest: KsTestLess, + ranges: latencyRangesBuckets, + }); + expect(esClientSearchMock).toHaveBeenCalledTimes(1); + }); + }); +}); diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_correlation.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_correlation.ts index 9894ac54eccb6..f63c36f90d728 100644 --- a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_correlation.ts +++ b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_correlation.ts @@ -26,7 +26,7 @@ interface ResponseHit { _source: ResponseHitSource; } -interface BucketCorrelation { +export interface BucketCorrelation { buckets_path: string; function: { count_correlation: { @@ -80,8 +80,7 @@ export const getTransactionDurationCorrelationRequest = ( // KS test p value = ks_test.less ks_test: { bucket_count_ks_test: { - // Remove 0 after https://github.com/elastic/elasticsearch/pull/74624 is merged - fractions: [0, ...fractions], + fractions, buckets_path: 'latency_ranges>_count', alternative: ['less', 'greater', 'two_sided'], }, diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_field_candidates.test.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_field_candidates.test.ts new file mode 100644 index 0000000000000..89bdd4280d324 --- /dev/null +++ b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_field_candidates.test.ts @@ -0,0 +1,145 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { estypes } from '@elastic/elasticsearch'; + +import type { ElasticsearchClient } from 'src/core/server'; + +import { + fetchTransactionDurationFieldCandidates, + getRandomDocsRequest, + hasPrefixToInclude, + shouldBeExcluded, +} from './query_field_candidates'; + +const params = { index: 'apm-*' }; + +describe('query_field_candidates', () => { + describe('shouldBeExcluded', () => { + it('does not exclude a completely custom field name', () => { + expect(shouldBeExcluded('myFieldName')).toBe(false); + }); + + it(`excludes a field if it's one of FIELDS_TO_EXCLUDE_AS_CANDIDATE`, () => { + expect(shouldBeExcluded('transaction.type')).toBe(true); + }); + + it(`excludes a field if it's prefixed with one of FIELD_PREFIX_TO_EXCLUDE_AS_CANDIDATE`, () => { + expect(shouldBeExcluded('observer.myFieldName')).toBe(true); + }); + }); + + describe('hasPrefixToInclude', () => { + it('identifies if a field name is prefixed to be included', () => { + expect(hasPrefixToInclude('myFieldName')).toBe(false); + expect(hasPrefixToInclude('somePrefix.myFieldName')).toBe(false); + expect(hasPrefixToInclude('cloud.myFieldName')).toBe(true); + expect(hasPrefixToInclude('labels.myFieldName')).toBe(true); + expect(hasPrefixToInclude('user_agent.myFieldName')).toBe(true); + }); + }); + + describe('getRandomDocsRequest', () => { + it('returns the most basic request body for a sample of random documents', () => { + const req = getRandomDocsRequest(params); + + expect(req).toEqual({ + body: { + _source: false, + fields: ['*'], + query: { + function_score: { + query: { + bool: { + filter: [ + { + term: { + 'processor.event': 'transaction', + }, + }, + ], + }, + }, + random_score: {}, + }, + }, + size: 1000, + }, + index: params.index, + }); + }); + }); + + describe('fetchTransactionDurationFieldCandidates', () => { + it('returns field candidates and total hits', async () => { + const esClientFieldCapsMock = jest.fn(() => ({ + body: { + fields: { + myIpFieldName: { ip: {} }, + myKeywordFieldName: { keyword: {} }, + myUnpopulatedKeywordFieldName: { keyword: {} }, + myNumericFieldName: { number: {} }, + }, + }, + })); + const esClientSearchMock = jest.fn((req: estypes.SearchRequest): { + body: estypes.SearchResponse; + } => { + return { + body: ({ + hits: { + hits: [ + { + fields: { + myIpFieldName: '1.1.1.1', + myKeywordFieldName: 'myKeywordFieldValue', + myNumericFieldName: 1234, + }, + }, + ], + }, + } as unknown) as estypes.SearchResponse, + }; + }); + + const esClientMock = ({ + fieldCaps: esClientFieldCapsMock, + search: esClientSearchMock, + } as unknown) as ElasticsearchClient; + + const resp = await fetchTransactionDurationFieldCandidates( + esClientMock, + params + ); + + expect(resp).toEqual({ + fieldCandidates: [ + // default field candidates + 'service.version', + 'service.node.name', + 'service.framework.version', + 'service.language.version', + 'service.runtime.version', + 'kubernetes.pod.name', + 'kubernetes.pod.uid', + 'container.id', + 'source.ip', + 'client.ip', + 'host.ip', + 'service.environment', + 'process.args', + 'http.response.status_code', + // field candidates identified by sample documents + 'myIpFieldName', + 'myKeywordFieldName', + ], + }); + expect(esClientFieldCapsMock).toHaveBeenCalledTimes(1); + expect(esClientSearchMock).toHaveBeenCalledTimes(1); + }); + }); +}); diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_field_candidates.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_field_candidates.ts index 4f1840971da7d..0fbdfef405e0d 100644 --- a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_field_candidates.ts +++ b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_field_candidates.ts @@ -21,7 +21,7 @@ import { POPULATED_DOC_COUNT_SAMPLE_SIZE, } from './constants'; -const shouldBeExcluded = (fieldName: string) => { +export const shouldBeExcluded = (fieldName: string) => { return ( FIELDS_TO_EXCLUDE_AS_CANDIDATE.has(fieldName) || FIELD_PREFIX_TO_EXCLUDE_AS_CANDIDATE.some((prefix) => @@ -30,7 +30,7 @@ const shouldBeExcluded = (fieldName: string) => { ); }; -const hasPrefixToInclude = (fieldName: string) => { +export const hasPrefixToInclude = (fieldName: string) => { return FIELD_PREFIX_TO_ADD_AS_CANDIDATE.some((prefix) => fieldName.startsWith(prefix) ); @@ -50,8 +50,6 @@ export const getRandomDocsRequest = ( random_score: {}, }, }, - // Required value for later correlation queries - track_total_hits: true, size: POPULATED_DOC_COUNT_SAMPLE_SIZE, }, }); @@ -59,7 +57,7 @@ export const getRandomDocsRequest = ( export const fetchTransactionDurationFieldCandidates = async ( esClient: ElasticsearchClient, params: SearchServiceParams -): Promise<{ fieldCandidates: Field[]; totalHits: number }> => { +): Promise<{ fieldCandidates: Field[] }> => { const { index } = params; // Get all fields with keyword mapping const respMapping = await esClient.fieldCaps({ @@ -100,6 +98,5 @@ export const fetchTransactionDurationFieldCandidates = async ( return { fieldCandidates: [...finalFieldCandidates], - totalHits: (resp.body.hits.total as estypes.SearchTotalHits).value, }; }; diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_field_value_pairs.test.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_field_value_pairs.test.ts new file mode 100644 index 0000000000000..ea5a1f55bc924 --- /dev/null +++ b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_field_value_pairs.test.ts @@ -0,0 +1,78 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { estypes } from '@elastic/elasticsearch'; + +import type { ElasticsearchClient } from 'src/core/server'; + +import type { AsyncSearchProviderProgress } from '../../../../common/search_strategies/correlations/types'; + +import { + fetchTransactionDurationFieldValuePairs, + getTermsAggRequest, +} from './query_field_value_pairs'; + +const params = { index: 'apm-*' }; + +describe('query_field_value_pairs', () => { + describe('getTermsAggRequest', () => { + it('returns the most basic request body for a terms aggregation', () => { + const fieldName = 'myFieldName'; + const req = getTermsAggRequest(params, fieldName); + expect(req?.body?.aggs?.attribute_terms?.terms?.field).toBe(fieldName); + }); + }); + + describe('fetchTransactionDurationFieldValuePairs', () => { + it('returns field/value pairs for field candidates', async () => { + const fieldCandidates = [ + 'myFieldCandidate1', + 'myFieldCandidate2', + 'myFieldCandidate3', + ]; + const progress = { + loadedFieldValuePairs: 0, + } as AsyncSearchProviderProgress; + + const esClientSearchMock = jest.fn((req: estypes.SearchRequest): { + body: estypes.SearchResponse; + } => { + return { + body: ({ + aggregations: { + attribute_terms: { + buckets: [{ key: 'myValue1' }, { key: 'myValue2' }], + }, + }, + } as unknown) as estypes.SearchResponse, + }; + }); + + const esClientMock = ({ + search: esClientSearchMock, + } as unknown) as ElasticsearchClient; + + const resp = await fetchTransactionDurationFieldValuePairs( + esClientMock, + params, + fieldCandidates, + progress + ); + + expect(progress.loadedFieldValuePairs).toBe(1); + expect(resp).toEqual([ + { field: 'myFieldCandidate1', value: 'myValue1' }, + { field: 'myFieldCandidate1', value: 'myValue2' }, + { field: 'myFieldCandidate2', value: 'myValue1' }, + { field: 'myFieldCandidate2', value: 'myValue2' }, + { field: 'myFieldCandidate3', value: 'myValue1' }, + { field: 'myFieldCandidate3', value: 'myValue2' }, + ]); + expect(esClientSearchMock).toHaveBeenCalledTimes(3); + }); + }); +}); diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_field_value_pairs.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_field_value_pairs.ts index 703a203c89207..8fde9d3ab1378 100644 --- a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_field_value_pairs.ts +++ b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_field_value_pairs.ts @@ -52,7 +52,7 @@ export const fetchTransactionDurationFieldValuePairs = async ( ): Promise => { const fieldValuePairs: FieldValuePairs = []; - let fieldValuePairsProgress = 0; + let fieldValuePairsProgress = 1; for (let i = 0; i < fieldCandidates.length; i++) { const fieldName = fieldCandidates[i]; diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_fractions.test.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_fractions.test.ts new file mode 100644 index 0000000000000..6052841d277c3 --- /dev/null +++ b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_fractions.test.ts @@ -0,0 +1,65 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { estypes } from '@elastic/elasticsearch'; + +import type { ElasticsearchClient } from 'src/core/server'; + +import { + fetchTransactionDurationFractions, + getTransactionDurationRangesRequest, +} from './query_fractions'; + +const params = { index: 'apm-*' }; +const ranges = [{ to: 1 }, { from: 1, to: 3 }, { from: 3, to: 5 }, { from: 5 }]; + +describe('query_fractions', () => { + describe('getTransactionDurationRangesRequest', () => { + it('returns the request body for the transaction duration ranges aggregation', () => { + const req = getTransactionDurationRangesRequest(params, ranges); + + expect(req?.body?.aggs?.latency_ranges?.range?.field).toBe( + 'transaction.duration.us' + ); + expect(req?.body?.aggs?.latency_ranges?.range?.ranges).toEqual(ranges); + }); + }); + + describe('fetchTransactionDurationFractions', () => { + it('computes the actual percentile bucket counts and actual fractions', async () => { + const esClientSearchMock = jest.fn((req: estypes.SearchRequest): { + body: estypes.SearchResponse; + } => { + return { + body: ({ + aggregations: { + latency_ranges: { + buckets: [{ doc_count: 1 }, { doc_count: 2 }], + }, + }, + } as unknown) as estypes.SearchResponse, + }; + }); + + const esClientMock = ({ + search: esClientSearchMock, + } as unknown) as ElasticsearchClient; + + const resp = await fetchTransactionDurationFractions( + esClientMock, + params, + ranges + ); + + expect(resp).toEqual({ + fractions: [0.3333333333333333, 0.6666666666666666], + totalDocCount: 3, + }); + expect(esClientSearchMock).toHaveBeenCalledTimes(1); + }); + }); +}); diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_histogram.test.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_histogram.test.ts new file mode 100644 index 0000000000000..2be9446352260 --- /dev/null +++ b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_histogram.test.ts @@ -0,0 +1,90 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { estypes } from '@elastic/elasticsearch'; + +import type { ElasticsearchClient } from 'src/core/server'; + +import { + fetchTransactionDurationHistogram, + getTransactionDurationHistogramRequest, +} from './query_histogram'; + +const params = { index: 'apm-*' }; +const interval = 100; + +describe('query_histogram', () => { + describe('getTransactionDurationHistogramRequest', () => { + it('returns the request body for the histogram request', () => { + const req = getTransactionDurationHistogramRequest(params, interval); + + expect(req).toEqual({ + body: { + aggs: { + transaction_duration_histogram: { + histogram: { + field: 'transaction.duration.us', + interval, + }, + }, + }, + query: { + bool: { + filter: [ + { + term: { + 'processor.event': 'transaction', + }, + }, + ], + }, + }, + size: 0, + }, + index: 'apm-*', + }); + }); + }); + + describe('fetchTransactionDurationHistogram', () => { + it('returns the buckets from the histogram aggregation', async () => { + const histogramBucket = [ + { + key: 0.0, + doc_count: 1, + }, + ]; + + const esClientSearchMock = jest.fn((req: estypes.SearchRequest): { + body: estypes.SearchResponse; + } => { + return { + body: ({ + aggregations: { + transaction_duration_histogram: { + buckets: histogramBucket, + }, + }, + } as unknown) as estypes.SearchResponse, + }; + }); + + const esClientMock = ({ + search: esClientSearchMock, + } as unknown) as ElasticsearchClient; + + const resp = await fetchTransactionDurationHistogram( + esClientMock, + params, + interval + ); + + expect(resp).toEqual(histogramBucket); + expect(esClientSearchMock).toHaveBeenCalledTimes(1); + }); + }); +}); diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_histogram_interval.test.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_histogram_interval.test.ts new file mode 100644 index 0000000000000..9ed529ccabddb --- /dev/null +++ b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_histogram_interval.test.ts @@ -0,0 +1,88 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { estypes } from '@elastic/elasticsearch'; + +import type { ElasticsearchClient } from 'src/core/server'; + +import { + fetchTransactionDurationHistogramInterval, + getHistogramIntervalRequest, +} from './query_histogram_interval'; + +const params = { index: 'apm-*' }; + +describe('query_histogram_interval', () => { + describe('getHistogramIntervalRequest', () => { + it('returns the request body for the transaction duration ranges aggregation', () => { + const req = getHistogramIntervalRequest(params); + + expect(req).toEqual({ + body: { + aggs: { + transaction_duration_max: { + max: { + field: 'transaction.duration.us', + }, + }, + transaction_duration_min: { + min: { + field: 'transaction.duration.us', + }, + }, + }, + query: { + bool: { + filter: [ + { + term: { + 'processor.event': 'transaction', + }, + }, + ], + }, + }, + size: 0, + }, + index: params.index, + }); + }); + }); + + describe('fetchTransactionDurationHistogramInterval', () => { + it('fetches the interval duration for histograms', async () => { + const esClientSearchMock = jest.fn((req: estypes.SearchRequest): { + body: estypes.SearchResponse; + } => { + return { + body: ({ + aggregations: { + transaction_duration_max: { + value: 10000, + }, + transaction_duration_min: { + value: 10, + }, + }, + } as unknown) as estypes.SearchResponse, + }; + }); + + const esClientMock = ({ + search: esClientSearchMock, + } as unknown) as ElasticsearchClient; + + const resp = await fetchTransactionDurationHistogramInterval( + esClientMock, + params + ); + + expect(resp).toEqual(10); + expect(esClientSearchMock).toHaveBeenCalledTimes(1); + }); + }); +}); diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_histogram_rangesteps.test.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_histogram_rangesteps.test.ts new file mode 100644 index 0000000000000..bb366ea29fed4 --- /dev/null +++ b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_histogram_rangesteps.test.ts @@ -0,0 +1,90 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { estypes } from '@elastic/elasticsearch'; + +import type { ElasticsearchClient } from 'src/core/server'; + +import { + fetchTransactionDurationHistogramRangesteps, + getHistogramIntervalRequest, +} from './query_histogram_rangesteps'; + +const params = { index: 'apm-*' }; + +describe('query_histogram_rangesteps', () => { + describe('getHistogramIntervalRequest', () => { + it('returns the request body for the histogram interval request', () => { + const req = getHistogramIntervalRequest(params); + + expect(req).toEqual({ + body: { + aggs: { + transaction_duration_max: { + max: { + field: 'transaction.duration.us', + }, + }, + transaction_duration_min: { + min: { + field: 'transaction.duration.us', + }, + }, + }, + query: { + bool: { + filter: [ + { + term: { + 'processor.event': 'transaction', + }, + }, + ], + }, + }, + size: 0, + }, + index: params.index, + }); + }); + }); + + describe('fetchTransactionDurationHistogramRangesteps', () => { + it('fetches the range steps for the log histogram', async () => { + const esClientSearchMock = jest.fn((req: estypes.SearchRequest): { + body: estypes.SearchResponse; + } => { + return { + body: ({ + aggregations: { + transaction_duration_max: { + value: 10000, + }, + transaction_duration_min: { + value: 10, + }, + }, + } as unknown) as estypes.SearchResponse, + }; + }); + + const esClientMock = ({ + search: esClientSearchMock, + } as unknown) as ElasticsearchClient; + + const resp = await fetchTransactionDurationHistogramRangesteps( + esClientMock, + params + ); + + expect(resp.length).toEqual(100); + expect(resp[0]).toEqual(9.260965422132594); + expect(resp[99]).toEqual(18521.930844265193); + expect(esClientSearchMock).toHaveBeenCalledTimes(1); + }); + }); +}); diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_percentiles.test.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_percentiles.test.ts new file mode 100644 index 0000000000000..0c319aee0fb2b --- /dev/null +++ b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_percentiles.test.ts @@ -0,0 +1,93 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { estypes } from '@elastic/elasticsearch'; + +import type { ElasticsearchClient } from 'src/core/server'; + +import { + fetchTransactionDurationPercentiles, + getTransactionDurationPercentilesRequest, +} from './query_percentiles'; + +const params = { index: 'apm-*' }; + +describe('query_percentiles', () => { + describe('getTransactionDurationPercentilesRequest', () => { + it('returns the request body for the duration percentiles request', () => { + const req = getTransactionDurationPercentilesRequest(params); + + expect(req).toEqual({ + body: { + aggs: { + transaction_duration_percentiles: { + percentiles: { + field: 'transaction.duration.us', + hdr: { + number_of_significant_value_digits: 3, + }, + }, + }, + }, + query: { + bool: { + filter: [ + { + term: { + 'processor.event': 'transaction', + }, + }, + ], + }, + }, + size: 0, + }, + index: params.index, + }); + }); + }); + + describe('fetchTransactionDurationPercentiles', () => { + it('fetches the percentiles', async () => { + const percentilesValues = { + '1.0': 5.0, + '5.0': 25.0, + '25.0': 165.0, + '50.0': 445.0, + '75.0': 725.0, + '95.0': 945.0, + '99.0': 985.0, + }; + + const esClientSearchMock = jest.fn((req: estypes.SearchRequest): { + body: estypes.SearchResponse; + } => { + return { + body: ({ + aggregations: { + transaction_duration_percentiles: { + values: percentilesValues, + }, + }, + } as unknown) as estypes.SearchResponse, + }; + }); + + const esClientMock = ({ + search: esClientSearchMock, + } as unknown) as ElasticsearchClient; + + const resp = await fetchTransactionDurationPercentiles( + esClientMock, + params + ); + + expect(resp).toEqual(percentilesValues); + expect(esClientSearchMock).toHaveBeenCalledTimes(1); + }); + }); +}); diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_percentiles.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_percentiles.ts index 013c1ba3cbc23..18dcefb59a11a 100644 --- a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_percentiles.ts +++ b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_percentiles.ts @@ -55,7 +55,7 @@ export const getTransactionDurationPercentilesRequest = ( }; }; -export const fetchTransactionDurationPecentiles = async ( +export const fetchTransactionDurationPercentiles = async ( esClient: ElasticsearchClient, params: SearchServiceParams, percents?: number[], @@ -73,7 +73,7 @@ export const fetchTransactionDurationPecentiles = async ( if (resp.body.aggregations === undefined) { throw new Error( - 'fetchTransactionDurationPecentiles failed, did not return aggregations.' + 'fetchTransactionDurationPercentiles failed, did not return aggregations.' ); } return ( diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_ranges.test.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_ranges.test.ts new file mode 100644 index 0000000000000..9451928e47ded --- /dev/null +++ b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_ranges.test.ts @@ -0,0 +1,124 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { estypes } from '@elastic/elasticsearch'; + +import type { ElasticsearchClient } from 'src/core/server'; + +import { + fetchTransactionDurationRanges, + getTransactionDurationRangesRequest, +} from './query_ranges'; + +const params = { index: 'apm-*' }; +const rangeSteps = [1, 3, 5]; + +describe('query_ranges', () => { + describe('getTransactionDurationRangesRequest', () => { + it('returns the request body for the duration percentiles request', () => { + const req = getTransactionDurationRangesRequest(params, rangeSteps); + + expect(req).toEqual({ + body: { + aggs: { + logspace_ranges: { + range: { + field: 'transaction.duration.us', + ranges: [ + { + to: 0, + }, + { + from: 0, + to: 1, + }, + { + from: 1, + to: 3, + }, + { + from: 3, + to: 5, + }, + { + from: 5, + }, + ], + }, + }, + }, + query: { + bool: { + filter: [ + { + term: { + 'processor.event': 'transaction', + }, + }, + ], + }, + }, + size: 0, + }, + index: params.index, + }); + }); + }); + + describe('fetchTransactionDurationRanges', () => { + it('fetches the percentiles', async () => { + const logspaceRangesBuckets = [ + { + key: '*-100.0', + to: 100.0, + doc_count: 2, + }, + { + key: '100.0-200.0', + from: 100.0, + to: 200.0, + doc_count: 2, + }, + { + key: '200.0-*', + from: 200.0, + doc_count: 3, + }, + ]; + + const esClientSearchMock = jest.fn((req: estypes.SearchRequest): { + body: estypes.SearchResponse; + } => { + return { + body: ({ + aggregations: { + logspace_ranges: { + buckets: logspaceRangesBuckets, + }, + }, + } as unknown) as estypes.SearchResponse, + }; + }); + + const esClientMock = ({ + search: esClientSearchMock, + } as unknown) as ElasticsearchClient; + + const resp = await fetchTransactionDurationRanges( + esClientMock, + params, + rangeSteps + ); + + expect(resp).toEqual([ + { doc_count: 2, key: 100 }, + { doc_count: 3, key: 200 }, + ]); + expect(esClientSearchMock).toHaveBeenCalledTimes(1); + }); + }); +}); diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_ranges.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_ranges.ts index 88256f79150fc..9074e7e0809bf 100644 --- a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_ranges.ts +++ b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_ranges.ts @@ -42,7 +42,9 @@ export const getTransactionDurationRangesRequest = ( }, [{ to: 0 }] as Array<{ from?: number; to?: number }> ); - ranges.push({ from: ranges[ranges.length - 1].to }); + if (ranges.length > 0) { + ranges.push({ from: ranges[ranges.length - 1].to }); + } return { index: params.index, diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/search_strategy.test.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/search_strategy.test.ts new file mode 100644 index 0000000000000..6d4bfcdde9994 --- /dev/null +++ b/x-pack/plugins/apm/server/lib/search_strategies/correlations/search_strategy.test.ts @@ -0,0 +1,234 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { estypes } from '@elastic/elasticsearch'; + +import { SearchStrategyDependencies } from 'src/plugins/data/server'; + +import { + apmCorrelationsSearchStrategyProvider, + PartialSearchRequest, +} from './search_strategy'; + +// helper to trigger promises in the async search service +const flushPromises = () => new Promise(setImmediate); + +const clientFieldCapsMock = () => ({ body: { fields: [] } }); + +// minimal client mock to fulfill search requirements of the async search service to succeed +const clientSearchMock = ( + req: estypes.SearchRequest +): { body: estypes.SearchResponse } => { + let aggregations: + | { + transaction_duration_percentiles: estypes.AggregationsTDigestPercentilesAggregate; + } + | { + transaction_duration_min: estypes.AggregationsValueAggregate; + transaction_duration_max: estypes.AggregationsValueAggregate; + } + | { + logspace_ranges: estypes.AggregationsMultiBucketAggregate<{ + from: number; + doc_count: number; + }>; + } + | { + latency_ranges: estypes.AggregationsMultiBucketAggregate<{ + doc_count: number; + }>; + } + | undefined; + + if (req?.body?.aggs !== undefined) { + const aggs = req.body.aggs; + // fetchTransactionDurationPercentiles + if (aggs.transaction_duration_percentiles !== undefined) { + aggregations = { transaction_duration_percentiles: { values: {} } }; + } + + // fetchTransactionDurationHistogramInterval + if ( + aggs.transaction_duration_min !== undefined && + aggs.transaction_duration_max !== undefined + ) { + aggregations = { + transaction_duration_min: { value: 0 }, + transaction_duration_max: { value: 1234 }, + }; + } + + // fetchTransactionDurationCorrelation + if (aggs.logspace_ranges !== undefined) { + aggregations = { logspace_ranges: { buckets: [] } }; + } + + // fetchTransactionDurationFractions + if (aggs.latency_ranges !== undefined) { + aggregations = { latency_ranges: { buckets: [] } }; + } + } + + return { + body: { + _shards: { + failed: 0, + successful: 1, + total: 1, + }, + took: 162, + timed_out: false, + hits: { + hits: [], + total: { + value: 0, + relation: 'eq', + }, + }, + ...(aggregations !== undefined ? { aggregations } : {}), + }, + }; +}; + +describe('APM Correlations search strategy', () => { + describe('strategy interface', () => { + it('returns a custom search strategy with a `search` and `cancel` function', async () => { + const searchStrategy = await apmCorrelationsSearchStrategyProvider(); + expect(typeof searchStrategy.search).toBe('function'); + expect(typeof searchStrategy.cancel).toBe('function'); + }); + }); + + describe('search', () => { + let mockClientFieldCaps: jest.Mock; + let mockClientSearch: jest.Mock; + let mockDeps: SearchStrategyDependencies; + let params: Required['params']; + + beforeEach(() => { + mockClientFieldCaps = jest.fn(clientFieldCapsMock); + mockClientSearch = jest.fn(clientSearchMock); + mockDeps = ({ + esClient: { + asCurrentUser: { + fieldCaps: mockClientFieldCaps, + search: mockClientSearch, + }, + }, + } as unknown) as SearchStrategyDependencies; + params = { + index: 'apm-*', + }; + }); + + describe('async functionality', () => { + describe('when no params are provided', () => { + it('throws an error', async () => { + const searchStrategy = await apmCorrelationsSearchStrategyProvider(); + expect(() => searchStrategy.search({}, {}, mockDeps)).toThrow( + 'Invalid request parameters.' + ); + }); + }); + + describe('when no ID is provided', () => { + it('performs a client search with params', async () => { + const searchStrategy = await apmCorrelationsSearchStrategyProvider(); + await searchStrategy.search({ params }, {}, mockDeps).toPromise(); + const [[request]] = mockClientSearch.mock.calls; + + expect(request.index).toEqual('apm-*'); + expect(request.body).toEqual( + expect.objectContaining({ + aggs: { + transaction_duration_percentiles: { + percentiles: { + field: 'transaction.duration.us', + hdr: { number_of_significant_value_digits: 3 }, + }, + }, + }, + query: { + bool: { + filter: [{ term: { 'processor.event': 'transaction' } }], + }, + }, + size: 0, + }) + ); + }); + }); + + describe('when an ID with params is provided', () => { + it('retrieves the current request', async () => { + const searchStrategy = await apmCorrelationsSearchStrategyProvider(); + const response = await searchStrategy + .search({ id: 'my-search-id', params }, {}, mockDeps) + .toPromise(); + + expect(response).toEqual( + expect.objectContaining({ id: 'my-search-id' }) + ); + }); + }); + + describe('if the client throws', () => { + it('does not emit an error', async () => { + mockClientSearch + .mockReset() + .mockRejectedValueOnce(new Error('client error')); + const searchStrategy = await apmCorrelationsSearchStrategyProvider(); + const response = await searchStrategy + .search({ params }, {}, mockDeps) + .toPromise(); + + expect(response).toEqual( + expect.objectContaining({ isRunning: true }) + ); + }); + }); + + it('triggers the subscription only once', async () => { + expect.assertions(1); + const searchStrategy = await apmCorrelationsSearchStrategyProvider(); + searchStrategy + .search({ params }, {}, mockDeps) + .subscribe((response) => { + expect(response).toEqual( + expect.objectContaining({ loaded: 0, isRunning: true }) + ); + }); + }); + }); + + describe('response', () => { + it('sends an updated response on consecutive search calls', async () => { + const searchStrategy = await apmCorrelationsSearchStrategyProvider(); + + const response1 = await searchStrategy + .search({ params }, {}, mockDeps) + .toPromise(); + + expect(typeof response1.id).toEqual('string'); + expect(response1).toEqual( + expect.objectContaining({ loaded: 0, isRunning: true }) + ); + + await flushPromises(); + + const response2 = await searchStrategy + .search({ id: response1.id, params }, {}, mockDeps) + .toPromise(); + + expect(response2.id).toEqual(response1.id); + expect(response2).toEqual( + expect.objectContaining({ loaded: 10, isRunning: false }) + ); + }); + }); + }); +}); diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/utils/aggregation_utils.test.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/utils/aggregation_utils.test.ts new file mode 100644 index 0000000000000..63de0a59d4894 --- /dev/null +++ b/x-pack/plugins/apm/server/lib/search_strategies/correlations/utils/aggregation_utils.test.ts @@ -0,0 +1,49 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { computeExpectationsAndRanges } from './aggregation_utils'; + +describe('aggregation utils', () => { + describe('computeExpectationsAndRanges', () => { + it('returns expectations and ranges based on given percentiles #1', async () => { + const { expectations, ranges } = computeExpectationsAndRanges([0, 1]); + expect(expectations).toEqual([0, 0.5, 1]); + expect(ranges).toEqual([{ to: 0 }, { from: 0, to: 1 }, { from: 1 }]); + }); + it('returns expectations and ranges based on given percentiles #2', async () => { + const { expectations, ranges } = computeExpectationsAndRanges([1, 3, 5]); + expect(expectations).toEqual([1, 2, 4, 5]); + expect(ranges).toEqual([ + { to: 1 }, + { from: 1, to: 3 }, + { from: 3, to: 5 }, + { from: 5 }, + ]); + }); + it('returns expectations and ranges with adjusted fractions', async () => { + const { expectations, ranges } = computeExpectationsAndRanges([ + 1, + 3, + 3, + 5, + ]); + expect(expectations).toEqual([ + 1, + 2.333333333333333, + 3.666666666666667, + 5, + ]); + expect(ranges).toEqual([ + { to: 1 }, + { from: 1, to: 3 }, + { from: 3, to: 3 }, + { from: 3, to: 5 }, + { from: 5 }, + ]); + }); + }); +}); diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/utils/aggregation_utils.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/utils/aggregation_utils.ts index 34e5ae2795d58..8d83b8fc29b05 100644 --- a/x-pack/plugins/apm/server/lib/search_strategies/correlations/utils/aggregation_utils.ts +++ b/x-pack/plugins/apm/server/lib/search_strategies/correlations/utils/aggregation_utils.ts @@ -31,14 +31,16 @@ export const computeExpectationsAndRanges = ( const ranges = percentiles.reduce((p, to) => { const from = p[p.length - 1]?.to; - if (from) { + if (from !== undefined) { p.push({ from, to }); } else { p.push({ to }); } return p; }, [] as Array<{ from?: number; to?: number }>); - ranges.push({ from: ranges[ranges.length - 1].to }); + if (ranges.length > 0) { + ranges.push({ from: ranges[ranges.length - 1].to }); + } const expectations = [tempPercentiles[0]]; for (let i = 1; i < tempPercentiles.length; i++) { diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/utils/math_utils.test.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/utils/math_utils.test.ts new file mode 100644 index 0000000000000..ed4107b9d602a --- /dev/null +++ b/x-pack/plugins/apm/server/lib/search_strategies/correlations/utils/math_utils.test.ts @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { getRandomInt } from './math_utils'; + +describe('math utils', () => { + describe('getRandomInt', () => { + it('returns a random integer within the given range', () => { + const min = 0.9; + const max = 11.1; + const randomInt = getRandomInt(min, max); + expect(Number.isInteger(randomInt)).toBe(true); + expect(randomInt > min).toBe(true); + expect(randomInt < max).toBe(true); + }); + + it('returns 1 if given range only allows this integer', () => { + const randomInt = getRandomInt(0.9, 1.1); + expect(randomInt).toBe(1); + }); + }); +});