From e1fa9c42afc87d9b53bbcc516c2421530ee0319d Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Thu, 14 Oct 2021 16:46:13 +0200 Subject: [PATCH 01/39] [ML] Move data fetching for overall latency histogram to custom hook. --- .../hooks/use_latency_overall_distribution.ts | 73 +++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 x-pack/plugins/apm/public/hooks/use_latency_overall_distribution.ts diff --git a/x-pack/plugins/apm/public/hooks/use_latency_overall_distribution.ts b/x-pack/plugins/apm/public/hooks/use_latency_overall_distribution.ts new file mode 100644 index 0000000000000..eb9d234205972 --- /dev/null +++ b/x-pack/plugins/apm/public/hooks/use_latency_overall_distribution.ts @@ -0,0 +1,73 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { DEFAULT_PERCENTILE_THRESHOLD } from '../../common/search_strategies/constants'; + +import { useApmServiceContext } from '../context/apm_service/use_apm_service_context'; +import { useUrlParams } from '../context/url_params_context/use_url_params'; + +import { useApmParams } from './use_apm_params'; +import { useFetcher, FETCH_STATUS } from './use_fetcher'; +import { useTimeRange } from './use_time_range'; + +export function useLatencyOverallDistribution() { + const { serviceName, transactionType } = useApmServiceContext(); + + const { urlParams } = useUrlParams(); + const { transactionName } = urlParams; + + const { + query: { kuery, environment, rangeFrom, rangeTo }, + } = useApmParams('/services/{serviceName}/transactions/view'); + + const { start, end } = useTimeRange({ rangeFrom, rangeTo }); + + const { + data = { log: [] }, + status, + error, + } = useFetcher( + (callApmApi) => { + if (serviceName && environment && start && end) { + return callApmApi({ + endpoint: 'GET /internal/apm/latency/overall_distribution', + params: { + query: { + serviceName, + transactionName, + transactionType, + kuery, + environment, + start, + end, + percentileThreshold: DEFAULT_PERCENTILE_THRESHOLD, + }, + }, + }); + } + }, + [ + serviceName, + transactionName, + transactionType, + kuery, + environment, + start, + end, + ] + ); + + const { percentileThresholdValue } = data; + const overallHistogram = + data.overallHistogram === undefined && status !== FETCH_STATUS.LOADING + ? [] + : data.overallHistogram; + const hasData = + Array.isArray(overallHistogram) && overallHistogram.length > 0; + + return { error, hasData, overallHistogram, percentileThresholdValue, status }; +} From 46f59d6976245e59833972084cc9d0b6248d7c54 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Fri, 15 Oct 2021 09:05:38 +0200 Subject: [PATCH 02/39] [ML] Migrates field candidates to regular endpoint. --- .../app/correlations/latency_correlations.tsx | 14 +- .../correlations/use_latency_correlations.ts | 168 ++++++++++++++++++ .../plugins/apm/server/routes/correlations.ts | 50 ++++++ .../get_global_apm_server_route_repository.ts | 2 + 4 files changed, 230 insertions(+), 4 deletions(-) create mode 100644 x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts create mode 100644 x-pack/plugins/apm/server/routes/correlations.ts diff --git a/x-pack/plugins/apm/public/components/app/correlations/latency_correlations.tsx b/x-pack/plugins/apm/public/components/app/correlations/latency_correlations.tsx index db6f3ad63f00d..81908efb596e7 100644 --- a/x-pack/plugins/apm/public/components/app/correlations/latency_correlations.tsx +++ b/x-pack/plugins/apm/public/components/app/correlations/latency_correlations.tsx @@ -59,6 +59,7 @@ import { CorrelationsProgressControls } from './progress_controls'; import { useTransactionColors } from './use_transaction_colors'; import { CorrelationsContextPopover } from './context_popover'; import { OnAddFilter } from './context_popover/top_values'; +import { useLatencyCorrelations } from './use_latency_correlations'; export function LatencyCorrelations({ onFilter }: { onFilter: () => void }) { const transactionColors = useTransactionColors(); @@ -69,6 +70,8 @@ export function LatencyCorrelations({ onFilter }: { onFilter: () => void }) { const displayLog = uiSettings.get(enableInspectEsQueries); + const lt = useLatencyCorrelations(); + const { progress, response, startFetch, cancelFetch } = useSearchStrategy( APM_SEARCH_STRATEGIES.APM_LATENCY_CORRELATIONS, { @@ -78,8 +81,8 @@ export function LatencyCorrelations({ onFilter }: { onFilter: () => void }) { ); const progressNormalized = progress.loaded / progress.total; const { overallHistogram, hasData, status } = getOverallHistogram( - response, - progress.isRunning + lt.response, + lt.progress.isRunning ); const fieldStats: Record | undefined = useMemo(() => { @@ -360,7 +363,7 @@ export function LatencyCorrelations({ onFilter }: { onFilter: () => void }) { void }) { { + startFetch(); + lt.startFetch(); + }} onCancel={cancelFetch} /> diff --git a/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts b/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts new file mode 100644 index 0000000000000..91244ff1eac65 --- /dev/null +++ b/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts @@ -0,0 +1,168 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { useCallback, useEffect, useReducer } from 'react'; + +import { IHttpFetchError } from 'src/core/public'; + +import { DEFAULT_PERCENTILE_THRESHOLD } from '../../../../common/search_strategies/constants'; +import type { RawResponseBase } from '../../../../common/search_strategies/types'; +import type { LatencyCorrelationsRawResponse } from '../../../../common/search_strategies/latency_correlations/types'; + +import { useApmServiceContext } from '../../../context/apm_service/use_apm_service_context'; +import { useUrlParams } from '../../../context/url_params_context/use_url_params'; + +import { useApmParams } from '../../../hooks/use_apm_params'; +import { useTimeRange } from '../../../hooks/use_time_range'; +import { callApmApi } from '../../../services/rest/createCallApmApi'; + +type Response = LatencyCorrelationsRawResponse & RawResponseBase; + +interface SearchStrategyProgress { + error?: Error | IHttpFetchError; + isRunning: boolean; + loaded: number; + total: number; +} + +const getInitialRawResponse = (): Response => + ({ + ccsWarning: false, + took: 0, + } as Response); + +const getInitialProgress = (): SearchStrategyProgress => ({ + isRunning: false, + loaded: 0, + total: 100, +}); + +const getReducer = + () => + (prev: T, update: Partial): T => ({ + ...prev, + ...update, + }); + +export function useLatencyCorrelations() { + const { serviceName, transactionType } = useApmServiceContext(); + + const { urlParams } = useUrlParams(); + const { transactionName } = urlParams; + + const { + query: { kuery, environment, rangeFrom, rangeTo }, + } = useApmParams('/services/{serviceName}/transactions/view'); + + const { start, end } = useTimeRange({ rangeFrom, rangeTo }); + + const [rawResponse, setRawResponse] = useReducer( + getReducer(), + getInitialRawResponse() + ); + + const [fetchState, setFetchState] = useReducer( + getReducer(), + getInitialProgress() + ); + + const startFetch = useCallback(async () => { + setFetchState({ + ...getInitialProgress(), + isRunning: true, + error: undefined, + }); + + const query = { + serviceName, + transactionName, + transactionType, + kuery, + environment, + start, + end, + }; + + try { + const data = await callApmApi({ + endpoint: 'GET /internal/apm/latency/overall_distribution', + signal: null, + params: { + query: { + ...query, + percentileThreshold: DEFAULT_PERCENTILE_THRESHOLD + '', + }, + }, + }); + + setRawResponse(data); + } catch (e) { + // const err = e as Error | IHttpFetchError; + // const message = error.body?.message ?? error.response?.statusText; + setFetchState({ + error: e as Error, + }); + return; + } + + setFetchState({ + loaded: 0.05, + }); + + try { + const data = await callApmApi({ + endpoint: 'GET /internal/apm/correlations/field_candidates', + signal: null, + params: { + query, + }, + }); + } catch (e) { + // const err = e as Error | IHttpFetchError; + // const message = error.body?.message ?? error.response?.statusText; + setFetchState({ + error: e as Error, + }); + return; + } + + setFetchState({ + loaded: 0.05, + }); + + setFetchState({ + isRunning: false, + }); + }, [ + environment, + serviceName, + transactionName, + transactionType, + kuery, + start, + end, + ]); + + const cancelFetch = useCallback(() => { + setFetchState({ + isRunning: false, + }); + }, []); + + // auto-update + useEffect(() => { + startFetch(); + return cancelFetch; + }, [startFetch, cancelFetch]); + + return { + progress: fetchState, + response: rawResponse, + startFetch, + cancelFetch, + }; +} diff --git a/x-pack/plugins/apm/server/routes/correlations.ts b/x-pack/plugins/apm/server/routes/correlations.ts new file mode 100644 index 0000000000000..6929f784f3a49 --- /dev/null +++ b/x-pack/plugins/apm/server/routes/correlations.ts @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import * as t from 'io-ts'; +import { setupRequest } from '../lib/helpers/setup_request'; +import { fetchTransactionDurationFieldCandidates } from '../lib/search_strategies/queries/query_field_candidates'; +import { withApmSpan } from '../utils/with_apm_span'; + +import { createApmServerRoute } from './create_apm_server_route'; +import { createApmServerRouteRepository } from './create_apm_server_route_repository'; +import { environmentRt, kueryRt, rangeRt } from './default_api_types'; + +const params = t.type({ + query: t.intersection([ + t.partial({ + serviceName: t.string, + transactionName: t.string, + transactionType: t.string, + }), + environmentRt, + kueryRt, + rangeRt, + ]), +}); + +const fieldCandidatesRoute = createApmServerRoute({ + endpoint: 'GET /internal/apm/correlations/field_candidates', + params, + options: { tags: ['access:apm'] }, + handler: async (resources) => { + const { indices } = await setupRequest(resources); + const esClient = resources.context.core.elasticsearch.client.asCurrentUser; + + return withApmSpan( + 'get_correlations_field_candidates', + async () => + await fetchTransactionDurationFieldCandidates(esClient, { + ...resources.params.query, + index: indices.transaction, + }) + ); + }, +}); + +export const correlationsRouteRepository = + createApmServerRouteRepository().add(fieldCandidatesRoute); diff --git a/x-pack/plugins/apm/server/routes/get_global_apm_server_route_repository.ts b/x-pack/plugins/apm/server/routes/get_global_apm_server_route_repository.ts index 3fa6152d953f3..1eca835f124b2 100644 --- a/x-pack/plugins/apm/server/routes/get_global_apm_server_route_repository.ts +++ b/x-pack/plugins/apm/server/routes/get_global_apm_server_route_repository.ts @@ -12,6 +12,7 @@ import type { import { PickByValue } from 'utility-types'; import { alertsChartPreviewRouteRepository } from './alerts/chart_preview'; import { backendsRouteRepository } from './backends'; +import { correlationsRouteRepository } from './correlations'; import { createApmServerRouteRepository } from './create_apm_server_route_repository'; import { environmentsRouteRepository } from './environments'; import { errorsRouteRepository } from './errors'; @@ -60,6 +61,7 @@ const getTypedGlobalApmServerRouteRepository = () => { .merge(sourceMapsRouteRepository) .merge(apmFleetRouteRepository) .merge(backendsRouteRepository) + .merge(correlationsRouteRepository) .merge(fallbackToTransactionsRouteRepository) .merge(historicalDataRouteRepository) .merge(eventMetadataRouteRepository); From 0e37684e7364c9fbaaa6eadfef07dc9b72857f58 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Fri, 15 Oct 2021 15:36:41 +0200 Subject: [PATCH 03/39] [ML] Fetch latency correlations via regular endpoints. --- .../app/correlations/latency_correlations.tsx | 28 +- .../correlations/use_latency_correlations.ts | 97 ++++-- .../latency_correlations/index.ts | 8 - .../latency_correlations_search_service.ts | 289 ------------------ ..._correlations_search_service_state.test.ts | 62 ---- ...tency_correlations_search_service_state.ts | 121 -------- .../queries/query_field_value_pairs.ts | 29 +- .../queries/query_histograms_generator.ts | 17 +- .../register_search_strategies.ts | 10 - .../search_strategy_provider.ts | 19 -- .../plugins/apm/server/routes/correlations.ts | 215 +++++++++++-- 11 files changed, 301 insertions(+), 594 deletions(-) delete mode 100644 x-pack/plugins/apm/server/lib/search_strategies/latency_correlations/index.ts delete mode 100644 x-pack/plugins/apm/server/lib/search_strategies/latency_correlations/latency_correlations_search_service.ts delete mode 100644 x-pack/plugins/apm/server/lib/search_strategies/latency_correlations/latency_correlations_search_service_state.test.ts delete mode 100644 x-pack/plugins/apm/server/lib/search_strategies/latency_correlations/latency_correlations_search_service_state.ts diff --git a/x-pack/plugins/apm/public/components/app/correlations/latency_correlations.tsx b/x-pack/plugins/apm/public/components/app/correlations/latency_correlations.tsx index 81908efb596e7..057bf1d2e5622 100644 --- a/x-pack/plugins/apm/public/components/app/correlations/latency_correlations.tsx +++ b/x-pack/plugins/apm/public/components/app/correlations/latency_correlations.tsx @@ -31,16 +31,12 @@ import { } from '../../../../../observability/public'; import { asPreciseDecimal } from '../../../../common/utils/formatters'; -import { - APM_SEARCH_STRATEGIES, - DEFAULT_PERCENTILE_THRESHOLD, -} from '../../../../common/search_strategies/constants'; +import { DEFAULT_PERCENTILE_THRESHOLD } from '../../../../common/search_strategies/constants'; import { LatencyCorrelation } from '../../../../common/search_strategies/latency_correlations/types'; import { FieldStats } from '../../../../common/search_strategies/field_stats_types'; import { useApmPluginContext } from '../../../context/apm_plugin/use_apm_plugin_context'; import { FETCH_STATUS } from '../../../hooks/use_fetcher'; -import { useSearchStrategy } from '../../../hooks/use_search_strategy'; import { TransactionDistributionChart, @@ -70,19 +66,12 @@ export function LatencyCorrelations({ onFilter }: { onFilter: () => void }) { const displayLog = uiSettings.get(enableInspectEsQueries); - const lt = useLatencyCorrelations(); - - const { progress, response, startFetch, cancelFetch } = useSearchStrategy( - APM_SEARCH_STRATEGIES.APM_LATENCY_CORRELATIONS, - { - percentileThreshold: DEFAULT_PERCENTILE_THRESHOLD, - analyzeCorrelations: true, - } - ); + const { progress, response, startFetch, cancelFetch } = + useLatencyCorrelations(); const progressNormalized = progress.loaded / progress.total; const { overallHistogram, hasData, status } = getOverallHistogram( - lt.response, - lt.progress.isRunning + response, + progress.isRunning ); const fieldStats: Record | undefined = useMemo(() => { @@ -363,7 +352,7 @@ export function LatencyCorrelations({ onFilter }: { onFilter: () => void }) { void }) { { - startFetch(); - lt.startFetch(); - }} + onRefresh={startFetch} onCancel={cancelFetch} /> diff --git a/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts b/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts index 91244ff1eac65..00b97537c810e 100644 --- a/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts +++ b/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts @@ -11,7 +11,10 @@ import { IHttpFetchError } from 'src/core/public'; import { DEFAULT_PERCENTILE_THRESHOLD } from '../../../../common/search_strategies/constants'; import type { RawResponseBase } from '../../../../common/search_strategies/types'; -import type { LatencyCorrelationsRawResponse } from '../../../../common/search_strategies/latency_correlations/types'; +import type { + LatencyCorrelation, + LatencyCorrelationsRawResponse, +} from '../../../../common/search_strategies/latency_correlations/types'; import { useApmServiceContext } from '../../../context/apm_service/use_apm_service_context'; import { useUrlParams } from '../../../context/url_params_context/use_url_params'; @@ -71,6 +74,7 @@ export function useLatencyCorrelations() { ); const startFetch = useCallback(async () => { + // TODO re-implemented cancelling setFetchState({ ...getInitialProgress(), isRunning: true, @@ -88,7 +92,7 @@ export function useLatencyCorrelations() { }; try { - const data = await callApmApi({ + const rawResponseUpdate = (await callApmApi({ endpoint: 'GET /internal/apm/latency/overall_distribution', signal: null, params: { @@ -97,30 +101,87 @@ export function useLatencyCorrelations() { percentileThreshold: DEFAULT_PERCENTILE_THRESHOLD + '', }, }, - }); + })) as Response; - setRawResponse(data); - } catch (e) { - // const err = e as Error | IHttpFetchError; - // const message = error.body?.message ?? error.response?.statusText; + setRawResponse(rawResponseUpdate); setFetchState({ - error: e as Error, + loaded: 5, }); - return; - } - - setFetchState({ - loaded: 0.05, - }); - try { - const data = await callApmApi({ + const { fieldCandidates } = await callApmApi({ endpoint: 'GET /internal/apm/correlations/field_candidates', signal: null, params: { query, }, }); + + setFetchState({ + loaded: 10, + }); + + const { fieldValuePairs } = await callApmApi({ + endpoint: 'GET /internal/apm/correlations/field_value_pairs', + signal: null, + params: { + query: { + ...query, + fieldCandidates, + }, + }, + }); + setFetchState({ + loaded: 20, + }); + + const fieldsToSample = new Set(); + const latencyCorrelations: LatencyCorrelation[] = []; + let loadCounter = 0; + for (const { fieldName, fieldValue } of fieldValuePairs) { + const significantCorrelations = await callApmApi({ + endpoint: 'GET /internal/apm/correlations/significant_correlations', + signal: null, + params: { + query: { + ...query, + fieldName, + fieldValue: fieldValue + '', + }, + }, + }); + + if (significantCorrelations.latencyCorrelations.length > 0) { + fieldsToSample.add(fieldName); + latencyCorrelations.push( + ...significantCorrelations.latencyCorrelations + ); + rawResponseUpdate.latencyCorrelations = latencyCorrelations; + setRawResponse(rawResponseUpdate); + } + + loadCounter++; + setFetchState({ + loaded: 20 + Math.round((loadCounter / fieldValuePairs.length) * 80), + }); + } + + const fieldStats = await callApmApi({ + endpoint: 'POST /internal/apm/correlations/field_stats', + signal: null, + params: { + body: { + ...query, + fieldsToSample: [...fieldsToSample], + }, + }, + }); + + rawResponseUpdate.fieldStats = fieldStats.stats; + setRawResponse(rawResponseUpdate); + + setFetchState({ + loaded: 100, + }); } catch (e) { // const err = e as Error | IHttpFetchError; // const message = error.body?.message ?? error.response?.statusText; @@ -130,10 +191,6 @@ export function useLatencyCorrelations() { return; } - setFetchState({ - loaded: 0.05, - }); - setFetchState({ isRunning: false, }); diff --git a/x-pack/plugins/apm/server/lib/search_strategies/latency_correlations/index.ts b/x-pack/plugins/apm/server/lib/search_strategies/latency_correlations/index.ts deleted file mode 100644 index 040aa5a7e424e..0000000000000 --- a/x-pack/plugins/apm/server/lib/search_strategies/latency_correlations/index.ts +++ /dev/null @@ -1,8 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -export { latencyCorrelationsSearchServiceProvider } from './latency_correlations_search_service'; diff --git a/x-pack/plugins/apm/server/lib/search_strategies/latency_correlations/latency_correlations_search_service.ts b/x-pack/plugins/apm/server/lib/search_strategies/latency_correlations/latency_correlations_search_service.ts deleted file mode 100644 index f170818d018d4..0000000000000 --- a/x-pack/plugins/apm/server/lib/search_strategies/latency_correlations/latency_correlations_search_service.ts +++ /dev/null @@ -1,289 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { range } from 'lodash'; -import type { ElasticsearchClient } from 'src/core/server'; - -import type { - RawResponseBase, - SearchStrategyClientParams, - SearchStrategyServerParams, -} from '../../../../common/search_strategies/types'; -import type { - LatencyCorrelationsParams, - LatencyCorrelationsRawResponse, -} from '../../../../common/search_strategies/latency_correlations/types'; - -import type { ApmIndicesConfig } from '../../settings/apm_indices/get_apm_indices'; - -import { - fetchTransactionDurationFieldCandidates, - fetchTransactionDurationFieldValuePairs, - fetchTransactionDurationFractions, - fetchTransactionDurationPercentiles, - fetchTransactionDurationHistograms, - fetchTransactionDurationHistogramRangeSteps, - fetchTransactionDurationRanges, -} from '../queries'; -import { computeExpectationsAndRanges } from '../utils'; -import { searchServiceLogProvider } from '../search_service_log'; -import type { SearchServiceProvider } from '../search_strategy_provider'; - -import { latencyCorrelationsSearchServiceStateProvider } from './latency_correlations_search_service_state'; -import { fetchFieldsStats } from '../queries/field_stats/get_fields_stats'; - -type LatencyCorrelationsSearchServiceProvider = SearchServiceProvider< - LatencyCorrelationsParams & SearchStrategyClientParams, - LatencyCorrelationsRawResponse & RawResponseBase ->; - -export const latencyCorrelationsSearchServiceProvider: LatencyCorrelationsSearchServiceProvider = - ( - esClient: ElasticsearchClient, - getApmIndices: () => Promise, - searchServiceParams: LatencyCorrelationsParams & SearchStrategyClientParams, - includeFrozen: boolean - ) => { - const { addLogMessage, getLogMessages } = searchServiceLogProvider(); - - const state = latencyCorrelationsSearchServiceStateProvider(); - - async function fetchCorrelations() { - let params: - | (LatencyCorrelationsParams & - SearchStrategyClientParams & - SearchStrategyServerParams) - | undefined; - - try { - const indices = await getApmIndices(); - params = { - ...searchServiceParams, - index: indices.transaction, - includeFrozen, - }; - - // 95th percentile to be displayed as a marker in the log log chart - const { totalDocs, percentiles: percentilesResponseThresholds } = - await fetchTransactionDurationPercentiles( - esClient, - params, - params.percentileThreshold - ? [params.percentileThreshold] - : undefined - ); - const percentileThresholdValue = - percentilesResponseThresholds[`${params.percentileThreshold}.0`]; - state.setPercentileThresholdValue(percentileThresholdValue); - - addLogMessage( - `Fetched ${params.percentileThreshold}th percentile value of ${percentileThresholdValue} based on ${totalDocs} documents.` - ); - - // finish early if we weren't able to identify the percentileThresholdValue. - if (percentileThresholdValue === undefined) { - addLogMessage( - `Abort service since percentileThresholdValue could not be determined.` - ); - state.setProgress({ - loadedHistogramStepsize: 1, - loadedOverallHistogram: 1, - loadedFieldCandidates: 1, - loadedFieldValuePairs: 1, - loadedHistograms: 1, - }); - state.setIsRunning(false); - return; - } - - const histogramRangeSteps = - await fetchTransactionDurationHistogramRangeSteps(esClient, params); - state.setProgress({ loadedHistogramStepsize: 1 }); - - addLogMessage(`Loaded histogram range steps.`); - - if (state.getIsCancelled()) { - state.setIsRunning(false); - return; - } - - const overallLogHistogramChartData = - await fetchTransactionDurationRanges( - esClient, - params, - histogramRangeSteps - ); - state.setProgress({ loadedOverallHistogram: 1 }); - state.setOverallHistogram(overallLogHistogramChartData); - - addLogMessage(`Loaded overall histogram chart data.`); - - if (state.getIsCancelled()) { - state.setIsRunning(false); - return; - } - - // finish early if correlation analysis is not required. - if (params.analyzeCorrelations === false) { - addLogMessage( - `Finish service since correlation analysis wasn't requested.` - ); - state.setProgress({ - loadedHistogramStepsize: 1, - loadedOverallHistogram: 1, - loadedFieldCandidates: 1, - loadedFieldValuePairs: 1, - loadedHistograms: 1, - }); - state.setIsRunning(false); - return; - } - - // Create an array of ranges [2, 4, 6, ..., 98] - const percentileAggregationPercents = range(2, 100, 2); - const { percentiles: percentilesRecords } = - await fetchTransactionDurationPercentiles( - esClient, - params, - percentileAggregationPercents - ); - const percentiles = Object.values(percentilesRecords); - - addLogMessage(`Loaded percentiles.`); - - if (state.getIsCancelled()) { - state.setIsRunning(false); - return; - } - - const { fieldCandidates } = - await fetchTransactionDurationFieldCandidates(esClient, params); - - addLogMessage(`Identified ${fieldCandidates.length} fieldCandidates.`); - - state.setProgress({ loadedFieldCandidates: 1 }); - - const fieldValuePairs = await fetchTransactionDurationFieldValuePairs( - esClient, - params, - fieldCandidates, - state, - addLogMessage - ); - - addLogMessage(`Identified ${fieldValuePairs.length} fieldValuePairs.`); - - if (state.getIsCancelled()) { - state.setIsRunning(false); - return; - } - - const { expectations, ranges } = - computeExpectationsAndRanges(percentiles); - - const { fractions, totalDocCount } = - await fetchTransactionDurationFractions(esClient, params, ranges); - - addLogMessage( - `Loaded fractions and totalDocCount of ${totalDocCount}.` - ); - - const fieldsToSample = new Set(); - let loadedHistograms = 0; - for await (const item of fetchTransactionDurationHistograms( - esClient, - addLogMessage, - params, - state, - expectations, - ranges, - fractions, - histogramRangeSteps, - totalDocCount, - fieldValuePairs - )) { - if (item !== undefined) { - state.addLatencyCorrelation(item); - fieldsToSample.add(item.fieldName); - } - loadedHistograms++; - state.setProgress({ - loadedHistograms: loadedHistograms / fieldValuePairs.length, - }); - } - - addLogMessage( - `Identified ${ - state.getState().latencyCorrelations.length - } significant correlations out of ${ - fieldValuePairs.length - } field/value pairs.` - ); - - addLogMessage( - `Identified ${fieldsToSample.size} fields to sample for field statistics.` - ); - - const { stats: fieldStats } = await fetchFieldsStats(esClient, params, [ - ...fieldsToSample, - ]); - - addLogMessage( - `Retrieved field statistics for ${fieldStats.length} fields out of ${fieldsToSample.size} fields.` - ); - state.addFieldStats(fieldStats); - } catch (e) { - state.setError(e); - } - - if (state.getState().error !== undefined && params?.index.includes(':')) { - state.setCcsWarning(true); - } - - state.setIsRunning(false); - } - - function cancel() { - addLogMessage(`Service cancelled.`); - state.setIsCancelled(true); - } - - fetchCorrelations(); - - return () => { - const { - ccsWarning, - error, - isRunning, - overallHistogram, - percentileThresholdValue, - progress, - fieldStats, - } = state.getState(); - - return { - cancel, - error, - meta: { - loaded: Math.round(state.getOverallProgress() * 100), - total: 100, - isRunning, - isPartial: isRunning, - }, - rawResponse: { - ccsWarning, - log: getLogMessages(), - took: Date.now() - progress.started, - latencyCorrelations: - state.getLatencyCorrelationsSortedByCorrelation(), - percentileThresholdValue, - overallHistogram, - fieldStats, - }, - }; - }; - }; diff --git a/x-pack/plugins/apm/server/lib/search_strategies/latency_correlations/latency_correlations_search_service_state.test.ts b/x-pack/plugins/apm/server/lib/search_strategies/latency_correlations/latency_correlations_search_service_state.test.ts deleted file mode 100644 index ce9014004f4b0..0000000000000 --- a/x-pack/plugins/apm/server/lib/search_strategies/latency_correlations/latency_correlations_search_service_state.test.ts +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { latencyCorrelationsSearchServiceStateProvider } from './latency_correlations_search_service_state'; - -describe('search service', () => { - describe('latencyCorrelationsSearchServiceStateProvider', () => { - it('initializes with default state', () => { - const state = latencyCorrelationsSearchServiceStateProvider(); - const defaultState = state.getState(); - const defaultProgress = state.getOverallProgress(); - - expect(defaultState.ccsWarning).toBe(false); - expect(defaultState.error).toBe(undefined); - expect(defaultState.isCancelled).toBe(false); - expect(defaultState.isRunning).toBe(true); - expect(defaultState.overallHistogram).toBe(undefined); - expect(defaultState.progress.loadedFieldCandidates).toBe(0); - expect(defaultState.progress.loadedFieldValuePairs).toBe(0); - expect(defaultState.progress.loadedHistogramStepsize).toBe(0); - expect(defaultState.progress.loadedHistograms).toBe(0); - expect(defaultState.progress.loadedOverallHistogram).toBe(0); - expect(defaultState.progress.started > 0).toBe(true); - - expect(defaultProgress).toBe(0); - }); - - it('returns updated state', () => { - const state = latencyCorrelationsSearchServiceStateProvider(); - - state.setCcsWarning(true); - state.setError(new Error('the-error-message')); - state.setIsCancelled(true); - state.setIsRunning(false); - state.setOverallHistogram([{ key: 1392202800000, doc_count: 1234 }]); - state.setProgress({ loadedHistograms: 0.5 }); - - const updatedState = state.getState(); - const updatedProgress = state.getOverallProgress(); - - expect(updatedState.ccsWarning).toBe(true); - expect(updatedState.error?.message).toBe('the-error-message'); - expect(updatedState.isCancelled).toBe(true); - expect(updatedState.isRunning).toBe(false); - expect(updatedState.overallHistogram).toEqual([ - { key: 1392202800000, doc_count: 1234 }, - ]); - expect(updatedState.progress.loadedFieldCandidates).toBe(0); - expect(updatedState.progress.loadedFieldValuePairs).toBe(0); - expect(updatedState.progress.loadedHistogramStepsize).toBe(0); - expect(updatedState.progress.loadedHistograms).toBe(0.5); - expect(updatedState.progress.loadedOverallHistogram).toBe(0); - expect(updatedState.progress.started > 0).toBe(true); - - expect(updatedProgress).toBe(0.45); - }); - }); -}); diff --git a/x-pack/plugins/apm/server/lib/search_strategies/latency_correlations/latency_correlations_search_service_state.ts b/x-pack/plugins/apm/server/lib/search_strategies/latency_correlations/latency_correlations_search_service_state.ts deleted file mode 100644 index 186099e4c307a..0000000000000 --- a/x-pack/plugins/apm/server/lib/search_strategies/latency_correlations/latency_correlations_search_service_state.ts +++ /dev/null @@ -1,121 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { HistogramItem } from '../../../../common/search_strategies/types'; -import type { - LatencyCorrelationSearchServiceProgress, - LatencyCorrelation, -} from '../../../../common/search_strategies/latency_correlations/types'; -import { FieldStats } from '../../../../common/search_strategies/field_stats_types'; - -export const latencyCorrelationsSearchServiceStateProvider = () => { - let ccsWarning = false; - function setCcsWarning(d: boolean) { - ccsWarning = d; - } - - let error: Error; - function setError(d: Error) { - error = d; - } - - let isCancelled = false; - function getIsCancelled() { - return isCancelled; - } - function setIsCancelled(d: boolean) { - isCancelled = d; - } - - let isRunning = true; - function setIsRunning(d: boolean) { - isRunning = d; - } - - let overallHistogram: HistogramItem[] | undefined; - function setOverallHistogram(d: HistogramItem[]) { - overallHistogram = d; - } - - let percentileThresholdValue: number; - function setPercentileThresholdValue(d: number) { - percentileThresholdValue = d; - } - - let progress: LatencyCorrelationSearchServiceProgress = { - started: Date.now(), - loadedHistogramStepsize: 0, - loadedOverallHistogram: 0, - loadedFieldCandidates: 0, - loadedFieldValuePairs: 0, - loadedHistograms: 0, - }; - function getOverallProgress() { - return ( - progress.loadedHistogramStepsize * 0.025 + - progress.loadedOverallHistogram * 0.025 + - progress.loadedFieldCandidates * 0.025 + - progress.loadedFieldValuePairs * 0.025 + - progress.loadedHistograms * 0.9 - ); - } - function setProgress( - d: Partial> - ) { - progress = { - ...progress, - ...d, - }; - } - - const latencyCorrelations: LatencyCorrelation[] = []; - function addLatencyCorrelation(d: LatencyCorrelation) { - latencyCorrelations.push(d); - } - - function getLatencyCorrelationsSortedByCorrelation() { - return latencyCorrelations.sort((a, b) => b.correlation - a.correlation); - } - const fieldStats: FieldStats[] = []; - function addFieldStats(stats: FieldStats[]) { - fieldStats.push(...stats); - } - - function getState() { - return { - ccsWarning, - error, - isCancelled, - isRunning, - overallHistogram, - percentileThresholdValue, - progress, - latencyCorrelations, - fieldStats, - }; - } - - return { - addLatencyCorrelation, - getIsCancelled, - getOverallProgress, - getState, - getLatencyCorrelationsSortedByCorrelation, - setCcsWarning, - setError, - setIsCancelled, - setIsRunning, - setOverallHistogram, - setPercentileThresholdValue, - setProgress, - addFieldStats, - }; -}; - -export type LatencyCorrelationsSearchServiceState = ReturnType< - typeof latencyCorrelationsSearchServiceStateProvider ->; diff --git a/x-pack/plugins/apm/server/lib/search_strategies/queries/query_field_value_pairs.ts b/x-pack/plugins/apm/server/lib/search_strategies/queries/query_field_value_pairs.ts index 296abfd2d8653..97584133c964f 100644 --- a/x-pack/plugins/apm/server/lib/search_strategies/queries/query_field_value_pairs.ts +++ b/x-pack/plugins/apm/server/lib/search_strategies/queries/query_field_value_pairs.ts @@ -15,7 +15,6 @@ import type { } from '../../../../common/search_strategies/types'; import type { SearchServiceLog } from '../search_service_log'; -import type { LatencyCorrelationsSearchServiceState } from '../latency_correlations/latency_correlations_search_service_state'; import { TERMS_SIZE } from '../constants'; import { getQueryWithParams } from './get_query_with_params'; @@ -44,16 +43,18 @@ const fetchTransactionDurationFieldTerms = async ( esClient: ElasticsearchClient, params: SearchStrategyParams, fieldName: string, - addLogMessage: SearchServiceLog['addLogMessage'] + addLogMessage?: SearchServiceLog['addLogMessage'] ): Promise => { try { const resp = await esClient.search(getTermsAggRequest(params, fieldName)); if (resp.body.aggregations === undefined) { - addLogMessage( - `Failed to fetch terms for field candidate ${fieldName} fieldValuePairs, no aggregations returned.`, - JSON.stringify(resp) - ); + if (addLogMessage) { + addLogMessage( + `Failed to fetch terms for field candidate ${fieldName} fieldValuePairs, no aggregations returned.`, + JSON.stringify(resp) + ); + } return []; } const buckets = ( @@ -69,10 +70,12 @@ const fetchTransactionDurationFieldTerms = async ( })); } } catch (e) { - addLogMessage( - `Failed to fetch terms for field candidate ${fieldName} fieldValuePairs.`, - JSON.stringify(e) - ); + if (addLogMessage) { + addLogMessage( + `Failed to fetch terms for field candidate ${fieldName} fieldValuePairs.`, + JSON.stringify(e) + ); + } } return []; @@ -95,8 +98,7 @@ export const fetchTransactionDurationFieldValuePairs = async ( esClient: ElasticsearchClient, params: SearchStrategyParams, fieldCandidates: string[], - state: LatencyCorrelationsSearchServiceState, - addLogMessage: SearchServiceLog['addLogMessage'] + addLogMessage?: SearchServiceLog['addLogMessage'] ): Promise => { let fieldValuePairsProgress = 1; @@ -110,9 +112,6 @@ export const fetchTransactionDurationFieldValuePairs = async ( addLogMessage ); - state.setProgress({ - loadedFieldValuePairs: fieldValuePairsProgress / fieldCandidates.length, - }); fieldValuePairsProgress++; return fieldTerms; diff --git a/x-pack/plugins/apm/server/lib/search_strategies/queries/query_histograms_generator.ts b/x-pack/plugins/apm/server/lib/search_strategies/queries/query_histograms_generator.ts index d526c63c7de34..f9c3bb4358a66 100644 --- a/x-pack/plugins/apm/server/lib/search_strategies/queries/query_histograms_generator.ts +++ b/x-pack/plugins/apm/server/lib/search_strategies/queries/query_histograms_generator.ts @@ -15,7 +15,6 @@ import type { } from '../../../../common/search_strategies/types'; import type { SearchServiceLog } from '../search_service_log'; -import type { LatencyCorrelationsSearchServiceState } from '../latency_correlations/latency_correlations_search_service_state'; import { CORRELATION_THRESHOLD, KS_TEST_THRESHOLD } from '../constants'; import { getPrioritizedFieldValuePairs } from './get_prioritized_field_value_pairs'; @@ -26,7 +25,6 @@ export async function* fetchTransactionDurationHistograms( esClient: ElasticsearchClient, addLogMessage: SearchServiceLog['addLogMessage'], params: SearchStrategyParams, - state: LatencyCorrelationsSearchServiceState, expectations: number[], ranges: estypes.AggregationsAggregationRange[], fractions: number[], @@ -35,8 +33,7 @@ export async function* fetchTransactionDurationHistograms( fieldValuePairs: FieldValuePair[] ) { for (const item of getPrioritizedFieldValuePairs(fieldValuePairs)) { - if (params === undefined || item === undefined || state.getIsCancelled()) { - state.setIsRunning(false); + if (params === undefined || item === undefined) { return; } @@ -53,11 +50,6 @@ export async function* fetchTransactionDurationHistograms( [item] ); - if (state.getIsCancelled()) { - state.setIsRunning(false); - return; - } - if ( correlation !== null && correlation > CORRELATION_THRESHOLD && @@ -87,9 +79,10 @@ export async function* fetchTransactionDurationHistograms( `Failed to fetch correlation/kstest for '${item.fieldName}/${item.fieldValue}'`, JSON.stringify(e) ); - if (params?.index.includes(':')) { - state.setCcsWarning(true); - } + // TODO return CCS warning + // if (params?.index.includes(':')) { + // state.setCcsWarning(true); + // } yield undefined; } } diff --git a/x-pack/plugins/apm/server/lib/search_strategies/register_search_strategies.ts b/x-pack/plugins/apm/server/lib/search_strategies/register_search_strategies.ts index 713c5e390ca8b..128fd32b8a377 100644 --- a/x-pack/plugins/apm/server/lib/search_strategies/register_search_strategies.ts +++ b/x-pack/plugins/apm/server/lib/search_strategies/register_search_strategies.ts @@ -12,7 +12,6 @@ import { APM_SEARCH_STRATEGIES } from '../../../common/search_strategies/constan import type { ApmIndicesConfig } from '../settings/apm_indices/get_apm_indices'; import { failedTransactionsCorrelationsSearchServiceProvider } from './failed_transactions_correlations'; -import { latencyCorrelationsSearchServiceProvider } from './latency_correlations'; import { searchStrategyProvider } from './search_strategy_provider'; export const registerSearchStrategies = ( @@ -20,15 +19,6 @@ export const registerSearchStrategies = ( getApmIndices: () => Promise, includeFrozen: boolean ) => { - registerSearchStrategy( - APM_SEARCH_STRATEGIES.APM_LATENCY_CORRELATIONS, - searchStrategyProvider( - latencyCorrelationsSearchServiceProvider, - getApmIndices, - includeFrozen - ) - ); - registerSearchStrategy( APM_SEARCH_STRATEGIES.APM_FAILED_TRANSACTIONS_CORRELATIONS, searchStrategyProvider( diff --git a/x-pack/plugins/apm/server/lib/search_strategies/search_strategy_provider.ts b/x-pack/plugins/apm/server/lib/search_strategies/search_strategy_provider.ts index 8035e9e4d97ca..9f332cc0b969d 100644 --- a/x-pack/plugins/apm/server/lib/search_strategies/search_strategy_provider.ts +++ b/x-pack/plugins/apm/server/lib/search_strategies/search_strategy_provider.ts @@ -25,10 +25,6 @@ import type { RawSearchStrategyClientParams, SearchStrategyClientParams, } from '../../../common/search_strategies/types'; -import type { - LatencyCorrelationsParams, - LatencyCorrelationsRawResponse, -} from '../../../common/search_strategies/latency_correlations/types'; import type { FailedTransactionsCorrelationsParams, FailedTransactionsCorrelationsRawResponse, @@ -78,21 +74,6 @@ export function searchStrategyProvider( > >; -// Latency Correlations function overload -export function searchStrategyProvider( - searchServiceProvider: SearchServiceProvider< - LatencyCorrelationsParams & SearchStrategyClientParams, - LatencyCorrelationsRawResponse & RawResponseBase - >, - getApmIndices: () => Promise, - includeFrozen: boolean -): ISearchStrategy< - IKibanaSearchRequest< - LatencyCorrelationsParams & RawSearchStrategyClientParams - >, - IKibanaSearchResponse ->; - export function searchStrategyProvider( searchServiceProvider: SearchServiceProvider< TRequestParams & SearchStrategyClientParams, diff --git a/x-pack/plugins/apm/server/routes/correlations.ts b/x-pack/plugins/apm/server/routes/correlations.ts index 6929f784f3a49..21d32a8a49aee 100644 --- a/x-pack/plugins/apm/server/routes/correlations.ts +++ b/x-pack/plugins/apm/server/routes/correlations.ts @@ -6,30 +6,45 @@ */ import * as t from 'io-ts'; +import { range } from 'lodash'; + +// import { toNumberRt } from '@kbn/io-ts-utils'; + +import type { FieldValuePair } from '../../common/search_strategies/types'; +import type { LatencyCorrelation } from '../../common/search_strategies/latency_correlations/types'; + import { setupRequest } from '../lib/helpers/setup_request'; -import { fetchTransactionDurationFieldCandidates } from '../lib/search_strategies/queries/query_field_candidates'; +import { + fetchTransactionDurationFieldCandidates, + fetchTransactionDurationFieldValuePairs, + fetchTransactionDurationFractions, + fetchTransactionDurationHistogramRangeSteps, + fetchTransactionDurationHistograms, + fetchTransactionDurationPercentiles, +} from '../lib/search_strategies/queries'; +import { fetchFieldsStats } from '../lib/search_strategies/queries/field_stats/get_fields_stats'; +import { computeExpectationsAndRanges } from '../lib/search_strategies/utils'; + import { withApmSpan } from '../utils/with_apm_span'; import { createApmServerRoute } from './create_apm_server_route'; import { createApmServerRouteRepository } from './create_apm_server_route_repository'; import { environmentRt, kueryRt, rangeRt } from './default_api_types'; -const params = t.type({ - query: t.intersection([ - t.partial({ - serviceName: t.string, - transactionName: t.string, - transactionType: t.string, - }), - environmentRt, - kueryRt, - rangeRt, - ]), -}); - const fieldCandidatesRoute = createApmServerRoute({ endpoint: 'GET /internal/apm/correlations/field_candidates', - params, + params: t.type({ + query: t.intersection([ + t.partial({ + serviceName: t.string, + transactionName: t.string, + transactionType: t.string, + }), + environmentRt, + kueryRt, + rangeRt, + ]), + }), options: { tags: ['access:apm'] }, handler: async (resources) => { const { indices } = await setupRequest(resources); @@ -46,5 +61,171 @@ const fieldCandidatesRoute = createApmServerRoute({ }, }); -export const correlationsRouteRepository = - createApmServerRouteRepository().add(fieldCandidatesRoute); +const fieldStatsRoute = createApmServerRoute({ + endpoint: 'POST /internal/apm/correlations/field_stats', + params: t.type({ + body: t.intersection([ + t.partial({ + serviceName: t.string, + transactionName: t.string, + transactionType: t.string, + }), + environmentRt, + kueryRt, + rangeRt, + t.type({ + fieldsToSample: t.array(t.string), + }), + ]), + }), + options: { tags: ['access:apm'] }, + handler: async (resources) => { + const { indices } = await setupRequest(resources); + const esClient = resources.context.core.elasticsearch.client.asCurrentUser; + + const { fieldsToSample, ...params } = resources.params.body; + + return withApmSpan( + 'get_correlations_field_stats', + async () => + await fetchFieldsStats( + esClient, + { + ...params, + index: indices.transaction, + }, + fieldsToSample + ) + ); + }, +}); + +const fieldValuePairsRoute = createApmServerRoute({ + endpoint: 'GET /internal/apm/correlations/field_value_pairs', + params: t.type({ + query: t.intersection([ + t.partial({ + serviceName: t.string, + transactionName: t.string, + transactionType: t.string, + }), + environmentRt, + kueryRt, + rangeRt, + t.type({ + fieldCandidates: t.array(t.string), + }), + ]), + }), + options: { tags: ['access:apm'] }, + handler: async (resources) => { + const { indices } = await setupRequest(resources); + const esClient = resources.context.core.elasticsearch.client.asCurrentUser; + + const { fieldCandidates, ...params } = resources.params.query; + + return withApmSpan('get_correlations_field_value_pairs', async () => ({ + fieldValuePairs: await fetchTransactionDurationFieldValuePairs( + esClient, + { + ...params, + index: indices.transaction, + }, + fieldCandidates + ), + })); + }, +}); + +const significantCorrelationsRoute = createApmServerRoute({ + endpoint: 'GET /internal/apm/correlations/significant_correlations', + params: t.type({ + query: t.intersection([ + t.partial({ + serviceName: t.string, + transactionName: t.string, + transactionType: t.string, + }), + environmentRt, + kueryRt, + rangeRt, + t.type({ + fieldName: t.string, // t.array(t.string), + fieldValue: t.string, // t.array(t.union([t.string, toNumberRt])), + }), + ]), + }), + options: { tags: ['access:apm'] }, + handler: async (resources) => { + const { indices } = await setupRequest(resources); + const esClient = resources.context.core.elasticsearch.client.asCurrentUser; + + const { fieldName, fieldValue, ...params } = resources.params.query; + const fieldValuePairs: FieldValuePair[] = [ + { + fieldName, + fieldValue, + }, + ]; + + const paramsWithIndex = { + ...params, + index: indices.transaction, + }; + + return withApmSpan('get_significant_correlations', async () => { + // Create an array of ranges [2, 4, 6, ..., 98] + const percentileAggregationPercents = range(2, 100, 2); + const { percentiles: percentilesRecords } = + await fetchTransactionDurationPercentiles( + esClient, + paramsWithIndex, + percentileAggregationPercents + ); + const percentiles = Object.values(percentilesRecords); + + const { expectations, ranges } = + computeExpectationsAndRanges(percentiles); + + const { fractions, totalDocCount } = + await fetchTransactionDurationFractions( + esClient, + paramsWithIndex, + ranges + ); + + const histogramRangeSteps = + await fetchTransactionDurationHistogramRangeSteps( + esClient, + paramsWithIndex + ); + + const latencyCorrelations: LatencyCorrelation[] = []; + + for await (const item of fetchTransactionDurationHistograms( + esClient, + () => {}, + paramsWithIndex, + expectations, + ranges, + fractions, + histogramRangeSteps, + totalDocCount, + fieldValuePairs + )) { + if (item !== undefined) { + latencyCorrelations.push(item); + } + } + + // TODO Fix CCS warning + return { latencyCorrelations, ccsWarning: false }; + }); + }, +}); + +export const correlationsRouteRepository = createApmServerRouteRepository() + .add(fieldCandidatesRoute) + .add(fieldStatsRoute) + .add(fieldValuePairsRoute) + .add(significantCorrelationsRoute); From 7d75d5f2f9cbec5b7a7023b6a3bcd2896ad95826 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Sat, 16 Oct 2021 23:25:40 +0200 Subject: [PATCH 04/39] [ML] Fetch failed transaction correlations via regular endpoints. --- .../failed_transactions_correlations.tsx | 16 +- .../use_failed_transactions_correlations.ts | 252 +++++++++++++++ .../correlations/use_latency_correlations.ts | 51 ++- .../hooks/use_latency_overall_distribution.ts | 4 +- .../apm/public/hooks/use_search_strategy.ts | 218 ------------- ...ransactions_correlations_search_service.ts | 259 --------------- ...tions_correlations_search_service_state.ts | 131 -------- .../failed_transactions_correlations/index.ts | 8 - .../apm/server/lib/search_strategies/index.ts | 8 - .../lib/search_strategies/queries/index.ts | 2 + .../queries/query_failure_correlation.ts | 3 +- .../queries/query_p_values.ts | 63 ++++ .../queries/query_significant_correlations.ts | 74 +++++ .../register_search_strategies.ts | 30 -- .../search_service_log.test.ts | 47 --- .../search_strategies/search_service_log.ts | 34 -- .../search_strategy_provider.test.ts | 302 ------------------ .../search_strategy_provider.ts | 185 ----------- x-pack/plugins/apm/server/plugin.ts | 20 -- .../plugins/apm/server/routes/correlations.ts | 115 +++---- 20 files changed, 489 insertions(+), 1333 deletions(-) create mode 100644 x-pack/plugins/apm/public/components/app/correlations/use_failed_transactions_correlations.ts delete mode 100644 x-pack/plugins/apm/public/hooks/use_search_strategy.ts delete mode 100644 x-pack/plugins/apm/server/lib/search_strategies/failed_transactions_correlations/failed_transactions_correlations_search_service.ts delete mode 100644 x-pack/plugins/apm/server/lib/search_strategies/failed_transactions_correlations/failed_transactions_correlations_search_service_state.ts delete mode 100644 x-pack/plugins/apm/server/lib/search_strategies/failed_transactions_correlations/index.ts delete mode 100644 x-pack/plugins/apm/server/lib/search_strategies/index.ts create mode 100644 x-pack/plugins/apm/server/lib/search_strategies/queries/query_p_values.ts create mode 100644 x-pack/plugins/apm/server/lib/search_strategies/queries/query_significant_correlations.ts delete mode 100644 x-pack/plugins/apm/server/lib/search_strategies/register_search_strategies.ts delete mode 100644 x-pack/plugins/apm/server/lib/search_strategies/search_service_log.test.ts delete mode 100644 x-pack/plugins/apm/server/lib/search_strategies/search_service_log.ts delete mode 100644 x-pack/plugins/apm/server/lib/search_strategies/search_strategy_provider.test.ts delete mode 100644 x-pack/plugins/apm/server/lib/search_strategies/search_strategy_provider.ts diff --git a/x-pack/plugins/apm/public/components/app/correlations/failed_transactions_correlations.tsx b/x-pack/plugins/apm/public/components/app/correlations/failed_transactions_correlations.tsx index 838671cbae7d9..661bae27acea9 100644 --- a/x-pack/plugins/apm/public/components/app/correlations/failed_transactions_correlations.tsx +++ b/x-pack/plugins/apm/public/components/app/correlations/failed_transactions_correlations.tsx @@ -36,16 +36,12 @@ import { import { asPercent } from '../../../../common/utils/formatters'; import { FailedTransactionsCorrelation } from '../../../../common/search_strategies/failed_transactions_correlations/types'; -import { - APM_SEARCH_STRATEGIES, - DEFAULT_PERCENTILE_THRESHOLD, -} from '../../../../common/search_strategies/constants'; +import { DEFAULT_PERCENTILE_THRESHOLD } from '../../../../common/search_strategies/constants'; import { FieldStats } from '../../../../common/search_strategies/field_stats_types'; import { useApmPluginContext } from '../../../context/apm_plugin/use_apm_plugin_context'; import { useLocalStorage } from '../../../hooks/useLocalStorage'; import { FETCH_STATUS } from '../../../hooks/use_fetcher'; -import { useSearchStrategy } from '../../../hooks/use_search_strategy'; import { useTheme } from '../../../hooks/use_theme'; import { ImpactBar } from '../../shared/ImpactBar'; @@ -68,6 +64,8 @@ import { useTransactionColors } from './use_transaction_colors'; import { CorrelationsContextPopover } from './context_popover'; import { OnAddFilter } from './context_popover/top_values'; +import { useFailedTransactionsCorrelations } from './use_failed_transactions_correlations'; + export function FailedTransactionsCorrelations({ onFilter, }: { @@ -83,12 +81,8 @@ export function FailedTransactionsCorrelations({ const inspectEnabled = uiSettings.get(enableInspectEsQueries); - const { progress, response, startFetch, cancelFetch } = useSearchStrategy( - APM_SEARCH_STRATEGIES.APM_FAILED_TRANSACTIONS_CORRELATIONS, - { - percentileThreshold: DEFAULT_PERCENTILE_THRESHOLD, - } - ); + const { progress, response, startFetch, cancelFetch } = + useFailedTransactionsCorrelations(); const fieldStats: Record | undefined = useMemo(() => { return response.fieldStats?.reduce((obj, field) => { diff --git a/x-pack/plugins/apm/public/components/app/correlations/use_failed_transactions_correlations.ts b/x-pack/plugins/apm/public/components/app/correlations/use_failed_transactions_correlations.ts new file mode 100644 index 0000000000000..bc5e994fabd73 --- /dev/null +++ b/x-pack/plugins/apm/public/components/app/correlations/use_failed_transactions_correlations.ts @@ -0,0 +1,252 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { useCallback, useEffect, useReducer, useRef } from 'react'; +import { chunk } from 'lodash'; + +import { IHttpFetchError } from 'src/core/public'; + +import { EVENT_OUTCOME } from '../../../../common/elasticsearch_fieldnames'; +import { EventOutcome } from '../../../../common/event_outcome'; +import { DEFAULT_PERCENTILE_THRESHOLD } from '../../../../common/search_strategies/constants'; +import type { RawResponseBase } from '../../../../common/search_strategies/types'; +import type { + FailedTransactionsCorrelation, + FailedTransactionsCorrelationsRawResponse, +} from '../../../../common/search_strategies/failed_transactions_correlations/types'; + +import { useApmServiceContext } from '../../../context/apm_service/use_apm_service_context'; +import { useUrlParams } from '../../../context/url_params_context/use_url_params'; + +import { useApmParams } from '../../../hooks/use_apm_params'; +import { useTimeRange } from '../../../hooks/use_time_range'; +import { callApmApi } from '../../../services/rest/createCallApmApi'; + +type Response = FailedTransactionsCorrelationsRawResponse & RawResponseBase; + +interface SearchStrategyProgress { + error?: Error | IHttpFetchError; + isRunning: boolean; + loaded: number; + total: number; +} + +const getInitialRawResponse = (): Response => + ({ + ccsWarning: false, + took: 0, + } as Response); + +const getInitialProgress = (): SearchStrategyProgress => ({ + isRunning: false, + loaded: 0, + total: 100, +}); + +const getReducer = + () => + (prev: T, update: Partial): T => ({ + ...prev, + ...update, + }); + +export function useFailedTransactionsCorrelations() { + const { serviceName, transactionType } = useApmServiceContext(); + + const { urlParams } = useUrlParams(); + const { transactionName } = urlParams; + + const { + query: { kuery, environment, rangeFrom, rangeTo }, + } = useApmParams('/services/{serviceName}/transactions/view'); + + const { start, end } = useTimeRange({ rangeFrom, rangeTo }); + + const [rawResponse, setRawResponse] = useReducer( + getReducer(), + getInitialRawResponse() + ); + + const [fetchState, setFetchState] = useReducer( + getReducer(), + getInitialProgress() + ); + + const isCancelledRef = useRef(false); + + const startFetch = useCallback(async () => { + isCancelledRef.current = false; + + setFetchState({ + ...getInitialProgress(), + isRunning: true, + error: undefined, + }); + + const query = { + serviceName, + transactionName, + transactionType, + kuery, + environment, + start, + end, + }; + + try { + const rawResponseUpdate = (await callApmApi({ + endpoint: 'POST /internal/apm/latency/overall_distribution', + signal: null, + params: { + body: { + ...query, + percentileThreshold: DEFAULT_PERCENTILE_THRESHOLD, + }, + }, + })) as Response; + + const { overallHistogram: errorHistogram } = (await callApmApi({ + endpoint: 'POST /internal/apm/latency/overall_distribution', + signal: null, + params: { + body: { + ...query, + percentileThreshold: DEFAULT_PERCENTILE_THRESHOLD, + termFilters: [ + { fieldName: EVENT_OUTCOME, fieldValue: EventOutcome.failure }, + ], + }, + }, + })) as Response; + + if (isCancelledRef.current) { + return; + } + + setRawResponse({ + ...rawResponseUpdate, + errorHistogram, + }); + setFetchState({ + loaded: 5, + }); + + const { fieldCandidates: candidates } = await callApmApi({ + endpoint: 'GET /internal/apm/correlations/field_candidates', + signal: null, + params: { + query, + }, + }); + + if (isCancelledRef.current) { + return; + } + + const fieldCandidates = candidates.filter((t) => !(t === EVENT_OUTCOME)); + + setFetchState({ + loaded: 10, + }); + + const failedTransactionsCorrelations: FailedTransactionsCorrelation[] = + []; + const fieldsToSample = new Set(); + const chunkSize = 10; + let loadCounter = 0; + + const fieldCandidatesChunks = chunk(fieldCandidates, chunkSize); + + for (const fieldCandidatesChunk of fieldCandidatesChunks) { + const pValues = await callApmApi({ + endpoint: 'POST /internal/apm/correlations/p_values', + signal: null, + params: { + body: { ...query, fieldCandidates: fieldCandidatesChunk }, + }, + }); + + if (pValues.failedTransactionsCorrelations.length > 0) { + pValues.failedTransactionsCorrelations.forEach((d) => { + fieldsToSample.add(d.fieldName); + }); + failedTransactionsCorrelations.push( + ...pValues.failedTransactionsCorrelations + ); + rawResponseUpdate.failedTransactionsCorrelations = + failedTransactionsCorrelations; + setRawResponse(rawResponseUpdate); + } + + if (isCancelledRef.current) { + return; + } + + loadCounter += chunkSize; + setFetchState({ + loaded: 20 + Math.round((loadCounter / fieldCandidates.length) * 80), + }); + } + + const fieldStats = await callApmApi({ + endpoint: 'POST /internal/apm/correlations/field_stats', + signal: null, + params: { + body: { + ...query, + fieldsToSample: [...fieldsToSample], + }, + }, + }); + + rawResponseUpdate.fieldStats = fieldStats.stats; + setRawResponse(rawResponseUpdate); + + setFetchState({ + loaded: 100, + }); + } catch (e) { + // const err = e as Error | IHttpFetchError; + // const message = error.body?.message ?? error.response?.statusText; + setFetchState({ + error: e as Error, + }); + } + + setFetchState({ + isRunning: false, + }); + }, [ + environment, + serviceName, + transactionName, + transactionType, + kuery, + start, + end, + ]); + + const cancelFetch = useCallback(() => { + isCancelledRef.current = true; + setFetchState({ + isRunning: false, + }); + }, []); + + // auto-update + useEffect(() => { + startFetch(); + return cancelFetch; + }, [startFetch, cancelFetch]); + + return { + progress: fetchState, + response: rawResponse, + startFetch, + cancelFetch, + }; +} diff --git a/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts b/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts index 00b97537c810e..a4a8d18ed28ff 100644 --- a/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts +++ b/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts @@ -5,7 +5,8 @@ * 2.0. */ -import { useCallback, useEffect, useReducer } from 'react'; +import { useCallback, useEffect, useReducer, useRef } from 'react'; +import { chunk } from 'lodash'; import { IHttpFetchError } from 'src/core/public'; @@ -73,8 +74,11 @@ export function useLatencyCorrelations() { getInitialProgress() ); + const isCancelledRef = useRef(false); + const startFetch = useCallback(async () => { - // TODO re-implemented cancelling + isCancelledRef.current = false; + setFetchState({ ...getInitialProgress(), isRunning: true, @@ -93,16 +97,20 @@ export function useLatencyCorrelations() { try { const rawResponseUpdate = (await callApmApi({ - endpoint: 'GET /internal/apm/latency/overall_distribution', + endpoint: 'POST /internal/apm/latency/overall_distribution', signal: null, params: { - query: { + body: { ...query, percentileThreshold: DEFAULT_PERCENTILE_THRESHOLD + '', }, }, })) as Response; + if (isCancelledRef.current) { + return; + } + setRawResponse(rawResponseUpdate); setFetchState({ loaded: 5, @@ -116,6 +124,10 @@ export function useLatencyCorrelations() { }, }); + if (isCancelledRef.current) { + return; + } + setFetchState({ loaded: 10, }); @@ -130,28 +142,35 @@ export function useLatencyCorrelations() { }, }, }); + + if (isCancelledRef.current) { + return; + } + setFetchState({ loaded: 20, }); const fieldsToSample = new Set(); const latencyCorrelations: LatencyCorrelation[] = []; + const chunkSize = 10; let loadCounter = 0; - for (const { fieldName, fieldValue } of fieldValuePairs) { + + const fieldValuePairChunks = chunk(fieldValuePairs, chunkSize); + + for (const fieldValuePairChunk of fieldValuePairChunks) { const significantCorrelations = await callApmApi({ - endpoint: 'GET /internal/apm/correlations/significant_correlations', + endpoint: 'POST /internal/apm/correlations/significant_correlations', signal: null, params: { - query: { - ...query, - fieldName, - fieldValue: fieldValue + '', - }, + body: { ...query, fieldValuePairs: fieldValuePairChunk }, }, }); if (significantCorrelations.latencyCorrelations.length > 0) { - fieldsToSample.add(fieldName); + significantCorrelations.latencyCorrelations.forEach((d) => { + fieldsToSample.add(d.fieldName); + }); latencyCorrelations.push( ...significantCorrelations.latencyCorrelations ); @@ -159,7 +178,11 @@ export function useLatencyCorrelations() { setRawResponse(rawResponseUpdate); } - loadCounter++; + if (isCancelledRef.current) { + return; + } + + loadCounter += chunkSize; setFetchState({ loaded: 20 + Math.round((loadCounter / fieldValuePairs.length) * 80), }); @@ -188,7 +211,6 @@ export function useLatencyCorrelations() { setFetchState({ error: e as Error, }); - return; } setFetchState({ @@ -205,6 +227,7 @@ export function useLatencyCorrelations() { ]); const cancelFetch = useCallback(() => { + isCancelledRef.current = true; setFetchState({ isRunning: false, }); diff --git a/x-pack/plugins/apm/public/hooks/use_latency_overall_distribution.ts b/x-pack/plugins/apm/public/hooks/use_latency_overall_distribution.ts index eb9d234205972..490545344552c 100644 --- a/x-pack/plugins/apm/public/hooks/use_latency_overall_distribution.ts +++ b/x-pack/plugins/apm/public/hooks/use_latency_overall_distribution.ts @@ -34,9 +34,9 @@ export function useLatencyOverallDistribution() { (callApmApi) => { if (serviceName && environment && start && end) { return callApmApi({ - endpoint: 'GET /internal/apm/latency/overall_distribution', + endpoint: 'POST /internal/apm/latency/overall_distribution', params: { - query: { + body: { serviceName, transactionName, transactionType, diff --git a/x-pack/plugins/apm/public/hooks/use_search_strategy.ts b/x-pack/plugins/apm/public/hooks/use_search_strategy.ts deleted file mode 100644 index 275eddb68ae00..0000000000000 --- a/x-pack/plugins/apm/public/hooks/use_search_strategy.ts +++ /dev/null @@ -1,218 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { useCallback, useEffect, useReducer, useRef } from 'react'; -import type { Subscription } from 'rxjs'; - -import { - IKibanaSearchRequest, - IKibanaSearchResponse, - isCompleteResponse, - isErrorResponse, -} from '../../../../../src/plugins/data/public'; -import { useKibana } from '../../../../../src/plugins/kibana_react/public'; - -import type { RawSearchStrategyClientParams } from '../../common/search_strategies/types'; -import type { RawResponseBase } from '../../common/search_strategies/types'; -import type { - LatencyCorrelationsParams, - LatencyCorrelationsRawResponse, -} from '../../common/search_strategies/latency_correlations/types'; -import type { - FailedTransactionsCorrelationsParams, - FailedTransactionsCorrelationsRawResponse, -} from '../../common/search_strategies/failed_transactions_correlations/types'; -import { - ApmSearchStrategies, - APM_SEARCH_STRATEGIES, -} from '../../common/search_strategies/constants'; -import { useApmServiceContext } from '../context/apm_service/use_apm_service_context'; -import { useUrlParams } from '../context/url_params_context/use_url_params'; - -import { ApmPluginStartDeps } from '../plugin'; - -import { useApmParams } from './use_apm_params'; -import { useTimeRange } from './use_time_range'; - -interface SearchStrategyProgress { - error?: Error; - isRunning: boolean; - loaded: number; - total: number; -} - -const getInitialRawResponse = < - TRawResponse extends RawResponseBase ->(): TRawResponse => - ({ - ccsWarning: false, - took: 0, - } as TRawResponse); - -const getInitialProgress = (): SearchStrategyProgress => ({ - isRunning: false, - loaded: 0, - total: 100, -}); - -const getReducer = - () => - (prev: T, update: Partial): T => ({ - ...prev, - ...update, - }); - -interface SearchStrategyReturnBase { - progress: SearchStrategyProgress; - response: TRawResponse; - startFetch: () => void; - cancelFetch: () => void; -} - -// Function overload for Latency Correlations -export function useSearchStrategy( - searchStrategyName: typeof APM_SEARCH_STRATEGIES.APM_LATENCY_CORRELATIONS, - searchStrategyParams: LatencyCorrelationsParams -): SearchStrategyReturnBase; - -// Function overload for Failed Transactions Correlations -export function useSearchStrategy( - searchStrategyName: typeof APM_SEARCH_STRATEGIES.APM_FAILED_TRANSACTIONS_CORRELATIONS, - searchStrategyParams: FailedTransactionsCorrelationsParams -): SearchStrategyReturnBase< - FailedTransactionsCorrelationsRawResponse & RawResponseBase ->; - -export function useSearchStrategy< - TRawResponse extends RawResponseBase, - TParams = unknown ->( - searchStrategyName: ApmSearchStrategies, - searchStrategyParams?: TParams -): SearchStrategyReturnBase { - const { - services: { data }, - } = useKibana(); - - const { serviceName, transactionType } = useApmServiceContext(); - const { - query: { kuery, environment, rangeFrom, rangeTo }, - } = useApmParams('/services/{serviceName}/transactions/view'); - const { start, end } = useTimeRange({ rangeFrom, rangeTo }); - const { urlParams } = useUrlParams(); - const { transactionName } = urlParams; - - const [rawResponse, setRawResponse] = useReducer( - getReducer(), - getInitialRawResponse() - ); - - const [fetchState, setFetchState] = useReducer( - getReducer(), - getInitialProgress() - ); - - const abortCtrl = useRef(new AbortController()); - const searchSubscription$ = useRef(); - const searchStrategyParamsRef = useRef(searchStrategyParams); - - const startFetch = useCallback(() => { - searchSubscription$.current?.unsubscribe(); - abortCtrl.current.abort(); - abortCtrl.current = new AbortController(); - setFetchState({ - ...getInitialProgress(), - error: undefined, - }); - - const request = { - params: { - environment, - serviceName, - transactionName, - transactionType, - kuery, - start, - end, - ...(searchStrategyParamsRef.current - ? { ...searchStrategyParamsRef.current } - : {}), - }, - }; - - // Submit the search request using the `data.search` service. - searchSubscription$.current = data.search - .search< - IKibanaSearchRequest, - IKibanaSearchResponse - >(request, { - strategy: searchStrategyName, - abortSignal: abortCtrl.current.signal, - }) - .subscribe({ - next: (response: IKibanaSearchResponse) => { - setRawResponse(response.rawResponse); - setFetchState({ - isRunning: response.isRunning || false, - ...(response.loaded ? { loaded: response.loaded } : {}), - ...(response.total ? { total: response.total } : {}), - }); - - if (isCompleteResponse(response)) { - searchSubscription$.current?.unsubscribe(); - setFetchState({ - isRunning: false, - }); - } else if (isErrorResponse(response)) { - searchSubscription$.current?.unsubscribe(); - setFetchState({ - error: response as unknown as Error, - isRunning: false, - }); - } - }, - error: (error: Error) => { - setFetchState({ - error, - isRunning: false, - }); - }, - }); - }, [ - searchStrategyName, - data.search, - environment, - serviceName, - transactionName, - transactionType, - kuery, - start, - end, - ]); - - const cancelFetch = useCallback(() => { - searchSubscription$.current?.unsubscribe(); - searchSubscription$.current = undefined; - abortCtrl.current.abort(); - setFetchState({ - isRunning: false, - }); - }, []); - - // auto-update - useEffect(() => { - startFetch(); - return cancelFetch; - }, [startFetch, cancelFetch]); - - return { - progress: fetchState, - response: rawResponse, - startFetch, - cancelFetch, - }; -} diff --git a/x-pack/plugins/apm/server/lib/search_strategies/failed_transactions_correlations/failed_transactions_correlations_search_service.ts b/x-pack/plugins/apm/server/lib/search_strategies/failed_transactions_correlations/failed_transactions_correlations_search_service.ts deleted file mode 100644 index efc28ce98e5e0..0000000000000 --- a/x-pack/plugins/apm/server/lib/search_strategies/failed_transactions_correlations/failed_transactions_correlations_search_service.ts +++ /dev/null @@ -1,259 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { chunk } from 'lodash'; - -import type { ElasticsearchClient } from 'src/core/server'; - -import { EVENT_OUTCOME } from '../../../../common/elasticsearch_fieldnames'; -import { EventOutcome } from '../../../../common/event_outcome'; -import type { - SearchStrategyClientParams, - SearchStrategyServerParams, - RawResponseBase, -} from '../../../../common/search_strategies/types'; -import type { - FailedTransactionsCorrelationsParams, - FailedTransactionsCorrelationsRawResponse, -} from '../../../../common/search_strategies/failed_transactions_correlations/types'; -import type { ApmIndicesConfig } from '../../settings/apm_indices/get_apm_indices'; -import { searchServiceLogProvider } from '../search_service_log'; -import { - fetchFailedTransactionsCorrelationPValues, - fetchTransactionDurationFieldCandidates, - fetchTransactionDurationPercentiles, - fetchTransactionDurationRanges, - fetchTransactionDurationHistogramRangeSteps, -} from '../queries'; -import type { SearchServiceProvider } from '../search_strategy_provider'; - -import { failedTransactionsCorrelationsSearchServiceStateProvider } from './failed_transactions_correlations_search_service_state'; - -import { ERROR_CORRELATION_THRESHOLD } from '../constants'; -import { fetchFieldsStats } from '../queries/field_stats/get_fields_stats'; - -type FailedTransactionsCorrelationsSearchServiceProvider = - SearchServiceProvider< - FailedTransactionsCorrelationsParams & SearchStrategyClientParams, - FailedTransactionsCorrelationsRawResponse & RawResponseBase - >; - -export const failedTransactionsCorrelationsSearchServiceProvider: FailedTransactionsCorrelationsSearchServiceProvider = - ( - esClient: ElasticsearchClient, - getApmIndices: () => Promise, - searchServiceParams: FailedTransactionsCorrelationsParams & - SearchStrategyClientParams, - includeFrozen: boolean - ) => { - const { addLogMessage, getLogMessages } = searchServiceLogProvider(); - - const state = failedTransactionsCorrelationsSearchServiceStateProvider(); - - async function fetchErrorCorrelations() { - try { - const indices = await getApmIndices(); - const params: FailedTransactionsCorrelationsParams & - SearchStrategyClientParams & - SearchStrategyServerParams = { - ...searchServiceParams, - index: indices.transaction, - includeFrozen, - }; - - // 95th percentile to be displayed as a marker in the log log chart - const { totalDocs, percentiles: percentilesResponseThresholds } = - await fetchTransactionDurationPercentiles( - esClient, - params, - params.percentileThreshold - ? [params.percentileThreshold] - : undefined - ); - const percentileThresholdValue = - percentilesResponseThresholds[`${params.percentileThreshold}.0`]; - state.setPercentileThresholdValue(percentileThresholdValue); - - addLogMessage( - `Fetched ${params.percentileThreshold}th percentile value of ${percentileThresholdValue} based on ${totalDocs} documents.` - ); - - // finish early if we weren't able to identify the percentileThresholdValue. - if (percentileThresholdValue === undefined) { - addLogMessage( - `Abort service since percentileThresholdValue could not be determined.` - ); - state.setProgress({ - loadedFieldCandidates: 1, - loadedErrorCorrelations: 1, - loadedOverallHistogram: 1, - loadedFailedTransactionsCorrelations: 1, - }); - state.setIsRunning(false); - return; - } - - const histogramRangeSteps = - await fetchTransactionDurationHistogramRangeSteps(esClient, params); - - const overallLogHistogramChartData = - await fetchTransactionDurationRanges( - esClient, - params, - histogramRangeSteps - ); - const errorLogHistogramChartData = await fetchTransactionDurationRanges( - esClient, - params, - histogramRangeSteps, - [{ fieldName: EVENT_OUTCOME, fieldValue: EventOutcome.failure }] - ); - - state.setProgress({ loadedOverallHistogram: 1 }); - state.setErrorHistogram(errorLogHistogramChartData); - state.setOverallHistogram(overallLogHistogramChartData); - - const { fieldCandidates: candidates } = - await fetchTransactionDurationFieldCandidates(esClient, params); - - const fieldCandidates = candidates.filter( - (t) => !(t === EVENT_OUTCOME) - ); - - addLogMessage(`Identified ${fieldCandidates.length} fieldCandidates.`); - - state.setProgress({ loadedFieldCandidates: 1 }); - - let fieldCandidatesFetchedCount = 0; - const fieldsToSample = new Set(); - if (params !== undefined && fieldCandidates.length > 0) { - const batches = chunk(fieldCandidates, 10); - for (let i = 0; i < batches.length; i++) { - try { - const results = await Promise.allSettled( - batches[i].map((fieldName) => - fetchFailedTransactionsCorrelationPValues( - esClient, - params, - histogramRangeSteps, - fieldName - ) - ) - ); - - results.forEach((result, idx) => { - if (result.status === 'fulfilled') { - const significantCorrelations = result.value.filter( - (record) => - record && - record.pValue !== undefined && - record.pValue < ERROR_CORRELATION_THRESHOLD - ); - - significantCorrelations.forEach((r) => { - fieldsToSample.add(r.fieldName); - }); - - state.addFailedTransactionsCorrelations( - significantCorrelations - ); - } else { - // If one of the fields in the batch had an error - addLogMessage( - `Error getting error correlation for field ${batches[i][idx]}: ${result.reason}.` - ); - } - }); - } catch (e) { - state.setError(e); - - if (params?.index.includes(':')) { - state.setCcsWarning(true); - } - } finally { - fieldCandidatesFetchedCount += batches[i].length; - state.setProgress({ - loadedFailedTransactionsCorrelations: - fieldCandidatesFetchedCount / fieldCandidates.length, - }); - } - } - - addLogMessage( - `Identified correlations for ${fieldCandidatesFetchedCount} fields out of ${fieldCandidates.length} candidates.` - ); - } - - addLogMessage( - `Identified ${fieldsToSample.size} fields to sample for field statistics.` - ); - - const { stats: fieldStats } = await fetchFieldsStats( - esClient, - params, - [...fieldsToSample], - [{ fieldName: EVENT_OUTCOME, fieldValue: EventOutcome.failure }] - ); - - addLogMessage( - `Retrieved field statistics for ${fieldStats.length} fields out of ${fieldsToSample.size} fields.` - ); - - state.addFieldStats(fieldStats); - } catch (e) { - state.setError(e); - } - - addLogMessage( - `Identified ${ - state.getState().failedTransactionsCorrelations.length - } significant correlations relating to failed transactions.` - ); - - state.setIsRunning(false); - } - - fetchErrorCorrelations(); - - return () => { - const { - ccsWarning, - error, - isRunning, - overallHistogram, - errorHistogram, - percentileThresholdValue, - progress, - fieldStats, - } = state.getState(); - - return { - cancel: () => { - addLogMessage(`Service cancelled.`); - state.setIsCancelled(true); - }, - error, - meta: { - loaded: Math.round(state.getOverallProgress() * 100), - total: 100, - isRunning, - isPartial: isRunning, - }, - rawResponse: { - ccsWarning, - log: getLogMessages(), - took: Date.now() - progress.started, - failedTransactionsCorrelations: - state.getFailedTransactionsCorrelationsSortedByScore(), - overallHistogram, - errorHistogram, - percentileThresholdValue, - fieldStats, - }, - }; - }; - }; diff --git a/x-pack/plugins/apm/server/lib/search_strategies/failed_transactions_correlations/failed_transactions_correlations_search_service_state.ts b/x-pack/plugins/apm/server/lib/search_strategies/failed_transactions_correlations/failed_transactions_correlations_search_service_state.ts deleted file mode 100644 index ed0fe5d6e178b..0000000000000 --- a/x-pack/plugins/apm/server/lib/search_strategies/failed_transactions_correlations/failed_transactions_correlations_search_service_state.ts +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { FailedTransactionsCorrelation } from '../../../../common/search_strategies/failed_transactions_correlations/types'; - -import type { HistogramItem } from '../../../../common/search_strategies/types'; -import { FieldStats } from '../../../../common/search_strategies/field_stats_types'; - -interface Progress { - started: number; - loadedFieldCandidates: number; - loadedErrorCorrelations: number; - loadedOverallHistogram: number; - loadedFailedTransactionsCorrelations: number; -} - -export const failedTransactionsCorrelationsSearchServiceStateProvider = () => { - let ccsWarning = false; - function setCcsWarning(d: boolean) { - ccsWarning = d; - } - - let error: Error; - function setError(d: Error) { - error = d; - } - - let isCancelled = false; - function setIsCancelled(d: boolean) { - isCancelled = d; - } - - let isRunning = true; - function setIsRunning(d: boolean) { - isRunning = d; - } - - let errorHistogram: HistogramItem[] | undefined; - function setErrorHistogram(d: HistogramItem[]) { - errorHistogram = d; - } - - let overallHistogram: HistogramItem[] | undefined; - function setOverallHistogram(d: HistogramItem[]) { - overallHistogram = d; - } - - let percentileThresholdValue: number; - function setPercentileThresholdValue(d: number) { - percentileThresholdValue = d; - } - - let progress: Progress = { - started: Date.now(), - loadedFieldCandidates: 0, - loadedErrorCorrelations: 0, - loadedOverallHistogram: 0, - loadedFailedTransactionsCorrelations: 0, - }; - function getOverallProgress() { - return ( - progress.loadedFieldCandidates * 0.025 + - progress.loadedFailedTransactionsCorrelations * (1 - 0.025) - ); - } - function setProgress(d: Partial>) { - progress = { - ...progress, - ...d, - }; - } - - const fieldStats: FieldStats[] = []; - function addFieldStats(stats: FieldStats[]) { - fieldStats.push(...stats); - } - - const failedTransactionsCorrelations: FailedTransactionsCorrelation[] = []; - function addFailedTransactionsCorrelation(d: FailedTransactionsCorrelation) { - failedTransactionsCorrelations.push(d); - } - function addFailedTransactionsCorrelations( - d: FailedTransactionsCorrelation[] - ) { - failedTransactionsCorrelations.push(...d); - } - - function getFailedTransactionsCorrelationsSortedByScore() { - return failedTransactionsCorrelations.sort((a, b) => b.score - a.score); - } - - function getState() { - return { - ccsWarning, - error, - isCancelled, - isRunning, - overallHistogram, - errorHistogram, - percentileThresholdValue, - progress, - failedTransactionsCorrelations, - fieldStats, - }; - } - - return { - addFailedTransactionsCorrelation, - addFailedTransactionsCorrelations, - getOverallProgress, - getState, - getFailedTransactionsCorrelationsSortedByScore, - setCcsWarning, - setError, - setIsCancelled, - setIsRunning, - setOverallHistogram, - setErrorHistogram, - setPercentileThresholdValue, - setProgress, - addFieldStats, - }; -}; - -export type FailedTransactionsCorrelationsSearchServiceState = ReturnType< - typeof failedTransactionsCorrelationsSearchServiceStateProvider ->; diff --git a/x-pack/plugins/apm/server/lib/search_strategies/failed_transactions_correlations/index.ts b/x-pack/plugins/apm/server/lib/search_strategies/failed_transactions_correlations/index.ts deleted file mode 100644 index 4763cd994d309..0000000000000 --- a/x-pack/plugins/apm/server/lib/search_strategies/failed_transactions_correlations/index.ts +++ /dev/null @@ -1,8 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -export { failedTransactionsCorrelationsSearchServiceProvider } from './failed_transactions_correlations_search_service'; diff --git a/x-pack/plugins/apm/server/lib/search_strategies/index.ts b/x-pack/plugins/apm/server/lib/search_strategies/index.ts deleted file mode 100644 index b4668138eefab..0000000000000 --- a/x-pack/plugins/apm/server/lib/search_strategies/index.ts +++ /dev/null @@ -1,8 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -export { registerSearchStrategies } from './register_search_strategies'; diff --git a/x-pack/plugins/apm/server/lib/search_strategies/queries/index.ts b/x-pack/plugins/apm/server/lib/search_strategies/queries/index.ts index e691b81e4adcf..f0c5a6a79a375 100644 --- a/x-pack/plugins/apm/server/lib/search_strategies/queries/index.ts +++ b/x-pack/plugins/apm/server/lib/search_strategies/queries/index.ts @@ -6,6 +6,8 @@ */ export { fetchFailedTransactionsCorrelationPValues } from './query_failure_correlation'; +export { fetchPValues } from './query_p_values'; +export { fetchSignificantCorrelations } from './query_significant_correlations'; export { fetchTransactionDurationFieldCandidates } from './query_field_candidates'; export { fetchTransactionDurationFieldValuePairs } from './query_field_value_pairs'; export { fetchTransactionDurationFractions } from './query_fractions'; diff --git a/x-pack/plugins/apm/server/lib/search_strategies/queries/query_failure_correlation.ts b/x-pack/plugins/apm/server/lib/search_strategies/queries/query_failure_correlation.ts index 64249a0f3547e..57339d73227d0 100644 --- a/x-pack/plugins/apm/server/lib/search_strategies/queries/query_failure_correlation.ts +++ b/x-pack/plugins/apm/server/lib/search_strategies/queries/query_failure_correlation.ts @@ -7,6 +7,7 @@ import { estypes } from '@elastic/elasticsearch'; import { ElasticsearchClient } from 'kibana/server'; import { SearchStrategyParams } from '../../../../common/search_strategies/types'; +import { FailedTransactionsCorrelation } from '../../../../common/search_strategies/failed_transactions_correlations/types'; import { EVENT_OUTCOME } from '../../../../common/elasticsearch_fieldnames'; import { EventOutcome } from '../../../../common/event_outcome'; import { fetchTransactionDurationRanges } from './query_ranges'; @@ -88,7 +89,7 @@ export const fetchFailedTransactionsCorrelationPValues = async ( }>; // Using for of to sequentially augment the results with histogram data. - const result = []; + const result: FailedTransactionsCorrelation[] = []; for (const bucket of overallResult.buckets) { // Scale the score into a value from 0 - 1 // using a concave piecewise linear function in -log(p-value) diff --git a/x-pack/plugins/apm/server/lib/search_strategies/queries/query_p_values.ts b/x-pack/plugins/apm/server/lib/search_strategies/queries/query_p_values.ts new file mode 100644 index 0000000000000..8897edc43daef --- /dev/null +++ b/x-pack/plugins/apm/server/lib/search_strategies/queries/query_p_values.ts @@ -0,0 +1,63 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ElasticsearchClient } from 'src/core/server'; + +import type { SearchStrategyParams } from '../../../../common/search_strategies/types'; +import type { FailedTransactionsCorrelation } from '../../../../common/search_strategies/failed_transactions_correlations/types'; + +import { ERROR_CORRELATION_THRESHOLD } from '../constants'; + +import { + fetchFailedTransactionsCorrelationPValues, + fetchTransactionDurationHistogramRangeSteps, +} from './index'; + +export const fetchPValues = async ( + esClient: ElasticsearchClient, + paramsWithIndex: SearchStrategyParams, + fieldCandidates: string[] +) => { + const failedTransactionsCorrelations: FailedTransactionsCorrelation[] = []; + + const histogramRangeSteps = await fetchTransactionDurationHistogramRangeSteps( + esClient, + paramsWithIndex + ); + + const results = await Promise.allSettled( + fieldCandidates.map((fieldName) => + fetchFailedTransactionsCorrelationPValues( + esClient, + paramsWithIndex, + histogramRangeSteps, + fieldName + ) + ) + ); + + results.forEach((result, idx) => { + if (result.status === 'fulfilled') { + failedTransactionsCorrelations.push( + ...result.value.filter( + (record) => + record && + typeof record.pValue === 'number' && + record.pValue < ERROR_CORRELATION_THRESHOLD + ) + ); + } else { + // If one of the fields in the batch had an error + // addLogMessage( + // `Error getting error correlation for field ${fieldCandidates[idx]}: ${result.reason}.` + // ); + } + }); + + // TODO Fix CCS warning + return { failedTransactionsCorrelations, ccsWarning: false }; +}; diff --git a/x-pack/plugins/apm/server/lib/search_strategies/queries/query_significant_correlations.ts b/x-pack/plugins/apm/server/lib/search_strategies/queries/query_significant_correlations.ts new file mode 100644 index 0000000000000..558c130f07de2 --- /dev/null +++ b/x-pack/plugins/apm/server/lib/search_strategies/queries/query_significant_correlations.ts @@ -0,0 +1,74 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { range } from 'lodash'; + +import type { ElasticsearchClient } from 'src/core/server'; + +import type { + FieldValuePair, + SearchStrategyParams, +} from '../../../../common/search_strategies/types'; +import type { LatencyCorrelation } from '../../../../common/search_strategies/latency_correlations/types'; + +import { + fetchTransactionDurationFractions, + fetchTransactionDurationHistogramRangeSteps, + fetchTransactionDurationHistograms, + fetchTransactionDurationPercentiles, +} from './index'; +import { computeExpectationsAndRanges } from '../utils'; + +export const fetchSignificantCorrelations = async ( + esClient: ElasticsearchClient, + paramsWithIndex: SearchStrategyParams, + fieldValuePairs: FieldValuePair[] +) => { + // Create an array of ranges [2, 4, 6, ..., 98] + const percentileAggregationPercents = range(2, 100, 2); + const { percentiles: percentilesRecords } = + await fetchTransactionDurationPercentiles( + esClient, + paramsWithIndex, + percentileAggregationPercents + ); + const percentiles = Object.values(percentilesRecords); + + const { expectations, ranges } = computeExpectationsAndRanges(percentiles); + + const { fractions, totalDocCount } = await fetchTransactionDurationFractions( + esClient, + paramsWithIndex, + ranges + ); + + const histogramRangeSteps = await fetchTransactionDurationHistogramRangeSteps( + esClient, + paramsWithIndex + ); + + const latencyCorrelations: LatencyCorrelation[] = []; + + for await (const item of fetchTransactionDurationHistograms( + esClient, + () => {}, + paramsWithIndex, + expectations, + ranges, + fractions, + histogramRangeSteps, + totalDocCount, + fieldValuePairs + )) { + if (item !== undefined) { + latencyCorrelations.push(item); + } + } + + // TODO Fix CCS warning + return { latencyCorrelations, ccsWarning: false }; +}; diff --git a/x-pack/plugins/apm/server/lib/search_strategies/register_search_strategies.ts b/x-pack/plugins/apm/server/lib/search_strategies/register_search_strategies.ts deleted file mode 100644 index 128fd32b8a377..0000000000000 --- a/x-pack/plugins/apm/server/lib/search_strategies/register_search_strategies.ts +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { PluginSetup as DataPluginSetup } from 'src/plugins/data/server'; - -import { APM_SEARCH_STRATEGIES } from '../../../common/search_strategies/constants'; - -import type { ApmIndicesConfig } from '../settings/apm_indices/get_apm_indices'; - -import { failedTransactionsCorrelationsSearchServiceProvider } from './failed_transactions_correlations'; -import { searchStrategyProvider } from './search_strategy_provider'; - -export const registerSearchStrategies = ( - registerSearchStrategy: DataPluginSetup['search']['registerSearchStrategy'], - getApmIndices: () => Promise, - includeFrozen: boolean -) => { - registerSearchStrategy( - APM_SEARCH_STRATEGIES.APM_FAILED_TRANSACTIONS_CORRELATIONS, - searchStrategyProvider( - failedTransactionsCorrelationsSearchServiceProvider, - getApmIndices, - includeFrozen - ) - ); -}; diff --git a/x-pack/plugins/apm/server/lib/search_strategies/search_service_log.test.ts b/x-pack/plugins/apm/server/lib/search_strategies/search_service_log.test.ts deleted file mode 100644 index 5b887f15a584e..0000000000000 --- a/x-pack/plugins/apm/server/lib/search_strategies/search_service_log.test.ts +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { - searchServiceLogProvider, - currentTimeAsString, -} from './search_service_log'; - -describe('search service', () => { - describe('currentTimeAsString', () => { - it('returns the current time as a string', () => { - const mockDate = new Date(1392202800000); - // @ts-ignore ignore the mockImplementation callback error - const spy = jest.spyOn(global, 'Date').mockReturnValue(mockDate); - - const timeString = currentTimeAsString(); - - expect(timeString).toEqual('2014-02-12T11:00:00.000Z'); - - spy.mockRestore(); - }); - }); - - describe('searchServiceLogProvider', () => { - it('adds and retrieves messages from the log', async () => { - const { addLogMessage, getLogMessages } = searchServiceLogProvider(); - - const mockDate = new Date(1392202800000); - // @ts-ignore ignore the mockImplementation callback error - const spy = jest.spyOn(global, 'Date').mockReturnValue(mockDate); - - addLogMessage('the first message'); - addLogMessage('the second message'); - - expect(getLogMessages()).toEqual([ - '2014-02-12T11:00:00.000Z: the first message', - '2014-02-12T11:00:00.000Z: the second message', - ]); - - spy.mockRestore(); - }); - }); -}); diff --git a/x-pack/plugins/apm/server/lib/search_strategies/search_service_log.ts b/x-pack/plugins/apm/server/lib/search_strategies/search_service_log.ts deleted file mode 100644 index 73a59021b01ed..0000000000000 --- a/x-pack/plugins/apm/server/lib/search_strategies/search_service_log.ts +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -interface LogMessage { - timestamp: string; - message: string; - error?: string; -} - -export const currentTimeAsString = () => new Date().toISOString(); - -export const searchServiceLogProvider = () => { - const log: LogMessage[] = []; - - function addLogMessage(message: string, error?: string) { - log.push({ - timestamp: currentTimeAsString(), - message, - ...(error !== undefined ? { error } : {}), - }); - } - - function getLogMessages() { - return log.map((l) => `${l.timestamp}: ${l.message}`); - } - - return { addLogMessage, getLogMessages }; -}; - -export type SearchServiceLog = ReturnType; diff --git a/x-pack/plugins/apm/server/lib/search_strategies/search_strategy_provider.test.ts b/x-pack/plugins/apm/server/lib/search_strategies/search_strategy_provider.test.ts deleted file mode 100644 index 034bd2a60ad19..0000000000000 --- a/x-pack/plugins/apm/server/lib/search_strategies/search_strategy_provider.test.ts +++ /dev/null @@ -1,302 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { estypes } from '@elastic/elasticsearch'; - -import { SearchStrategyDependencies } from 'src/plugins/data/server'; - -import { IKibanaSearchRequest } from '../../../../../../src/plugins/data/common'; - -import { ENVIRONMENT_ALL } from '../../../common/environment_filter_values'; -import type { LatencyCorrelationsParams } from '../../../common/search_strategies/latency_correlations/types'; -import type { RawSearchStrategyClientParams } from '../../../common/search_strategies/types'; - -import type { ApmIndicesConfig } from '../settings/apm_indices/get_apm_indices'; - -import { latencyCorrelationsSearchServiceProvider } from './latency_correlations'; -import { searchStrategyProvider } from './search_strategy_provider'; - -// helper to trigger promises in the async search service -const flushPromises = () => new Promise(setImmediate); - -const clientFieldCapsMock = () => ({ body: { fields: [] } }); - -// minimal client mock to fulfill search requirements of the async search service to succeed -const clientSearchMock = ( - req: estypes.SearchRequest -): { body: estypes.SearchResponse } => { - let aggregations: - | { - transaction_duration_percentiles: estypes.AggregationsTDigestPercentilesAggregate; - } - | { - transaction_duration_min: estypes.AggregationsValueAggregate; - transaction_duration_max: estypes.AggregationsValueAggregate; - } - | { - logspace_ranges: estypes.AggregationsMultiBucketAggregate<{ - from: number; - doc_count: number; - }>; - } - | { - latency_ranges: estypes.AggregationsMultiBucketAggregate<{ - doc_count: number; - }>; - } - | undefined; - - if (req?.body?.aggs !== undefined) { - const aggs = req.body.aggs; - // fetchTransactionDurationPercentiles - if (aggs.transaction_duration_percentiles !== undefined) { - aggregations = { transaction_duration_percentiles: { values: {} } }; - } - - // fetchTransactionDurationCorrelation - if (aggs.logspace_ranges !== undefined) { - aggregations = { logspace_ranges: { buckets: [] } }; - } - - // fetchTransactionDurationFractions - if (aggs.latency_ranges !== undefined) { - aggregations = { latency_ranges: { buckets: [] } }; - } - } - - return { - body: { - _shards: { - failed: 0, - successful: 1, - total: 1, - }, - took: 162, - timed_out: false, - hits: { - hits: [], - total: { - value: 0, - relation: 'eq', - }, - }, - ...(aggregations !== undefined ? { aggregations } : {}), - }, - }; -}; - -const getApmIndicesMock = async () => - ({ transaction: 'apm-*' } as ApmIndicesConfig); - -describe('APM Correlations search strategy', () => { - describe('strategy interface', () => { - it('returns a custom search strategy with a `search` and `cancel` function', async () => { - const searchStrategy = await searchStrategyProvider( - latencyCorrelationsSearchServiceProvider, - getApmIndicesMock, - false - ); - expect(typeof searchStrategy.search).toBe('function'); - expect(typeof searchStrategy.cancel).toBe('function'); - }); - }); - - describe('search', () => { - let mockClientFieldCaps: jest.Mock; - let mockClientSearch: jest.Mock; - let mockGetApmIndicesMock: jest.Mock; - let mockDeps: SearchStrategyDependencies; - let params: Required< - IKibanaSearchRequest< - LatencyCorrelationsParams & RawSearchStrategyClientParams - > - >['params']; - - beforeEach(() => { - mockClientFieldCaps = jest.fn(clientFieldCapsMock); - mockClientSearch = jest.fn(clientSearchMock); - mockGetApmIndicesMock = jest.fn(getApmIndicesMock); - mockDeps = { - esClient: { - asCurrentUser: { - fieldCaps: mockClientFieldCaps, - search: mockClientSearch, - }, - }, - } as unknown as SearchStrategyDependencies; - params = { - start: '2020', - end: '2021', - environment: ENVIRONMENT_ALL.value, - kuery: '', - percentileThreshold: 95, - analyzeCorrelations: true, - }; - }); - - describe('async functionality', () => { - describe('when no params are provided', () => { - it('throws an error', async () => { - const searchStrategy = await searchStrategyProvider( - latencyCorrelationsSearchServiceProvider, - mockGetApmIndicesMock, - false - ); - - expect(mockGetApmIndicesMock).toHaveBeenCalledTimes(0); - - expect(() => searchStrategy.search({}, {}, mockDeps)).toThrow( - 'Invalid request parameters.' - ); - }); - }); - - describe('when no ID is provided', () => { - it('performs a client search with params', async () => { - const searchStrategy = await searchStrategyProvider( - latencyCorrelationsSearchServiceProvider, - mockGetApmIndicesMock, - false - ); - await searchStrategy.search({ params }, {}, mockDeps).toPromise(); - - expect(mockGetApmIndicesMock).toHaveBeenCalledTimes(1); - - const [[request]] = mockClientSearch.mock.calls; - - expect(request.index).toEqual('apm-*'); - expect(request.body).toEqual( - expect.objectContaining({ - aggs: { - transaction_duration_percentiles: { - percentiles: { - field: 'transaction.duration.us', - hdr: { number_of_significant_value_digits: 3 }, - percents: [95], - }, - }, - }, - query: { - bool: { - filter: [ - { term: { 'processor.event': 'transaction' } }, - { - range: { - '@timestamp': { - format: 'epoch_millis', - gte: 1577836800000, - lte: 1609459200000, - }, - }, - }, - ], - }, - }, - size: 0, - track_total_hits: true, - }) - ); - }); - }); - - describe('when an ID with params is provided', () => { - it('retrieves the current request', async () => { - const searchStrategy = await searchStrategyProvider( - latencyCorrelationsSearchServiceProvider, - mockGetApmIndicesMock, - false - ); - const response = await searchStrategy - .search({ params }, {}, mockDeps) - .toPromise(); - - const searchStrategyId = response.id; - - const response2 = await searchStrategy - .search({ id: searchStrategyId, params }, {}, mockDeps) - .toPromise(); - - expect(mockGetApmIndicesMock).toHaveBeenCalledTimes(1); - expect(response2).toEqual( - expect.objectContaining({ id: searchStrategyId }) - ); - }); - }); - - describe('if the client throws', () => { - it('does not emit an error', async () => { - mockClientSearch - .mockReset() - .mockRejectedValueOnce(new Error('client error')); - const searchStrategy = await searchStrategyProvider( - latencyCorrelationsSearchServiceProvider, - mockGetApmIndicesMock, - false - ); - const response = await searchStrategy - .search({ params }, {}, mockDeps) - .toPromise(); - - expect(mockGetApmIndicesMock).toHaveBeenCalledTimes(1); - - expect(response).toEqual( - expect.objectContaining({ isRunning: true }) - ); - }); - }); - - it('triggers the subscription only once', async () => { - expect.assertions(2); - const searchStrategy = await searchStrategyProvider( - latencyCorrelationsSearchServiceProvider, - mockGetApmIndicesMock, - false - ); - searchStrategy - .search({ params }, {}, mockDeps) - .subscribe((response) => { - expect(mockGetApmIndicesMock).toHaveBeenCalledTimes(1); - expect(response).toEqual( - expect.objectContaining({ loaded: 0, isRunning: true }) - ); - }); - }); - }); - - describe('response', () => { - it('sends an updated response on consecutive search calls', async () => { - const searchStrategy = await searchStrategyProvider( - latencyCorrelationsSearchServiceProvider, - mockGetApmIndicesMock, - false - ); - - const response1 = await searchStrategy - .search({ params }, {}, mockDeps) - .toPromise(); - - expect(mockGetApmIndicesMock).toHaveBeenCalledTimes(1); - expect(typeof response1.id).toEqual('string'); - expect(response1).toEqual( - expect.objectContaining({ loaded: 0, isRunning: true }) - ); - - await flushPromises(); - - const response2 = await searchStrategy - .search({ id: response1.id, params }, {}, mockDeps) - .toPromise(); - - expect(mockGetApmIndicesMock).toHaveBeenCalledTimes(1); - expect(response2.id).toEqual(response1.id); - expect(response2).toEqual( - expect.objectContaining({ loaded: 100, isRunning: false }) - ); - }); - }); - }); -}); diff --git a/x-pack/plugins/apm/server/lib/search_strategies/search_strategy_provider.ts b/x-pack/plugins/apm/server/lib/search_strategies/search_strategy_provider.ts deleted file mode 100644 index 9f332cc0b969d..0000000000000 --- a/x-pack/plugins/apm/server/lib/search_strategies/search_strategy_provider.ts +++ /dev/null @@ -1,185 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import uuid from 'uuid'; -import { of } from 'rxjs'; -import { getOrElse } from 'fp-ts/lib/Either'; -import { pipe } from 'fp-ts/lib/pipeable'; -import * as t from 'io-ts'; -import { failure } from 'io-ts/lib/PathReporter'; - -import type { ElasticsearchClient } from 'src/core/server'; - -import type { ISearchStrategy } from '../../../../../../src/plugins/data/server'; -import { - IKibanaSearchRequest, - IKibanaSearchResponse, -} from '../../../../../../src/plugins/data/common'; - -import type { - RawResponseBase, - RawSearchStrategyClientParams, - SearchStrategyClientParams, -} from '../../../common/search_strategies/types'; -import type { - FailedTransactionsCorrelationsParams, - FailedTransactionsCorrelationsRawResponse, -} from '../../../common/search_strategies/failed_transactions_correlations/types'; -import { rangeRt } from '../../routes/default_api_types'; -import type { ApmIndicesConfig } from '../settings/apm_indices/get_apm_indices'; - -interface SearchServiceState { - cancel: () => void; - error: Error; - meta: { - loaded: number; - total: number; - isRunning: boolean; - isPartial: boolean; - }; - rawResponse: TRawResponse; -} - -type GetSearchServiceState = - () => SearchServiceState; - -export type SearchServiceProvider< - TSearchStrategyClientParams extends SearchStrategyClientParams, - TRawResponse extends RawResponseBase -> = ( - esClient: ElasticsearchClient, - getApmIndices: () => Promise, - searchServiceParams: TSearchStrategyClientParams, - includeFrozen: boolean -) => GetSearchServiceState; - -// Failed Transactions Correlations function overload -export function searchStrategyProvider( - searchServiceProvider: SearchServiceProvider< - FailedTransactionsCorrelationsParams & SearchStrategyClientParams, - FailedTransactionsCorrelationsRawResponse & RawResponseBase - >, - getApmIndices: () => Promise, - includeFrozen: boolean -): ISearchStrategy< - IKibanaSearchRequest< - FailedTransactionsCorrelationsParams & RawSearchStrategyClientParams - >, - IKibanaSearchResponse< - FailedTransactionsCorrelationsRawResponse & RawResponseBase - > ->; - -export function searchStrategyProvider( - searchServiceProvider: SearchServiceProvider< - TRequestParams & SearchStrategyClientParams, - TResponseParams & RawResponseBase - >, - getApmIndices: () => Promise, - includeFrozen: boolean -): ISearchStrategy< - IKibanaSearchRequest, - IKibanaSearchResponse -> { - const searchServiceMap = new Map< - string, - GetSearchServiceState - >(); - - return { - search: (request, options, deps) => { - if (request.params === undefined) { - throw new Error('Invalid request parameters.'); - } - - const { start: startString, end: endString } = request.params; - - // converts string based start/end to epochmillis - const decodedRange = pipe( - rangeRt.decode({ start: startString, end: endString }), - getOrElse((errors) => { - throw new Error(failure(errors).join('\n')); - }) - ); - - // The function to fetch the current state of the search service. - // This will be either an existing service for a follow up fetch or a new one for new requests. - let getSearchServiceState: GetSearchServiceState< - TResponseParams & RawResponseBase - >; - - // If the request includes an ID, we require that the search service already exists - // otherwise we throw an error. The client should never poll a service that's been cancelled or finished. - // This also avoids instantiating search services when the service gets called with random IDs. - if (typeof request.id === 'string') { - const existingGetSearchServiceState = searchServiceMap.get(request.id); - - if (typeof existingGetSearchServiceState === 'undefined') { - throw new Error( - `SearchService with ID '${request.id}' does not exist.` - ); - } - - getSearchServiceState = existingGetSearchServiceState; - } else { - const { - start, - end, - environment, - kuery, - serviceName, - transactionName, - transactionType, - ...requestParams - } = request.params; - - getSearchServiceState = searchServiceProvider( - deps.esClient.asCurrentUser, - getApmIndices, - { - environment, - kuery, - serviceName, - transactionName, - transactionType, - start: decodedRange.start, - end: decodedRange.end, - ...(requestParams as unknown as TRequestParams), - }, - includeFrozen - ); - } - - // Reuse the request's id or create a new one. - const id = request.id ?? uuid(); - - const { error, meta, rawResponse } = getSearchServiceState(); - - if (error instanceof Error) { - searchServiceMap.delete(id); - throw error; - } else if (meta.isRunning) { - searchServiceMap.set(id, getSearchServiceState); - } else { - searchServiceMap.delete(id); - } - - return of({ - id, - ...meta, - rawResponse, - }); - }, - cancel: async (id, options, deps) => { - const getSearchServiceState = searchServiceMap.get(id); - if (getSearchServiceState !== undefined) { - getSearchServiceState().cancel(); - searchServiceMap.delete(id); - } - }, - }; -} diff --git a/x-pack/plugins/apm/server/plugin.ts b/x-pack/plugins/apm/server/plugin.ts index 72a1bc483015e..ca55486b70acb 100644 --- a/x-pack/plugins/apm/server/plugin.ts +++ b/x-pack/plugins/apm/server/plugin.ts @@ -26,7 +26,6 @@ import { registerFleetPolicyCallbacks } from './lib/fleet/register_fleet_policy_ import { createApmTelemetry } from './lib/apm_telemetry'; import { createApmEventClient } from './lib/helpers/create_es_client/create_apm_event_client'; import { getInternalSavedObjectsClient } from './lib/helpers/get_internal_saved_objects_client'; -import { registerSearchStrategies } from './lib/search_strategies'; import { createApmAgentConfigurationIndex } from './lib/settings/agent_configuration/create_agent_config_index'; import { getApmIndices } from './lib/settings/apm_indices/get_apm_indices'; import { createApmCustomLinkIndex } from './lib/settings/custom_link/create_custom_link_index'; @@ -197,25 +196,6 @@ export class APMPlugin logger: this.logger, }); - // search strategies for async partial search results - core.getStartServices().then(([coreStart]) => { - (async () => { - const savedObjectsClient = new SavedObjectsClient( - coreStart.savedObjects.createInternalRepository() - ); - - const includeFrozen = await coreStart.uiSettings - .asScopedToClient(savedObjectsClient) - .get(UI_SETTINGS.SEARCH_INCLUDE_FROZEN); - - registerSearchStrategies( - plugins.data.search.registerSearchStrategy, - boundGetApmIndices, - includeFrozen - ); - })(); - }); - core.deprecations.registerDeprecations({ getDeprecations: getDeprecations({ cloudSetup: plugins.cloud, diff --git a/x-pack/plugins/apm/server/routes/correlations.ts b/x-pack/plugins/apm/server/routes/correlations.ts index 21d32a8a49aee..a4d0e4979a5be 100644 --- a/x-pack/plugins/apm/server/routes/correlations.ts +++ b/x-pack/plugins/apm/server/routes/correlations.ts @@ -6,24 +6,17 @@ */ import * as t from 'io-ts'; -import { range } from 'lodash'; -// import { toNumberRt } from '@kbn/io-ts-utils'; - -import type { FieldValuePair } from '../../common/search_strategies/types'; -import type { LatencyCorrelation } from '../../common/search_strategies/latency_correlations/types'; +import { toNumberRt } from '@kbn/io-ts-utils'; import { setupRequest } from '../lib/helpers/setup_request'; import { + fetchPValues, + fetchSignificantCorrelations, fetchTransactionDurationFieldCandidates, fetchTransactionDurationFieldValuePairs, - fetchTransactionDurationFractions, - fetchTransactionDurationHistogramRangeSteps, - fetchTransactionDurationHistograms, - fetchTransactionDurationPercentiles, } from '../lib/search_strategies/queries'; import { fetchFieldsStats } from '../lib/search_strategies/queries/field_stats/get_fields_stats'; -import { computeExpectationsAndRanges } from '../lib/search_strategies/utils'; import { withApmSpan } from '../utils/with_apm_span'; @@ -138,9 +131,9 @@ const fieldValuePairsRoute = createApmServerRoute({ }); const significantCorrelationsRoute = createApmServerRoute({ - endpoint: 'GET /internal/apm/correlations/significant_correlations', + endpoint: 'POST /internal/apm/correlations/significant_correlations', params: t.type({ - query: t.intersection([ + body: t.intersection([ t.partial({ serviceName: t.string, transactionName: t.string, @@ -150,8 +143,12 @@ const significantCorrelationsRoute = createApmServerRoute({ kueryRt, rangeRt, t.type({ - fieldName: t.string, // t.array(t.string), - fieldValue: t.string, // t.array(t.union([t.string, toNumberRt])), + fieldValuePairs: t.array( + t.type({ + fieldName: t.string, + fieldValue: t.union([t.string, toNumberRt]), + }) + ), }), ]), }), @@ -160,71 +157,63 @@ const significantCorrelationsRoute = createApmServerRoute({ const { indices } = await setupRequest(resources); const esClient = resources.context.core.elasticsearch.client.asCurrentUser; - const { fieldName, fieldValue, ...params } = resources.params.query; - const fieldValuePairs: FieldValuePair[] = [ - { - fieldName, - fieldValue, - }, - ]; + const { fieldValuePairs, ...params } = resources.params.body; const paramsWithIndex = { ...params, index: indices.transaction, }; - return withApmSpan('get_significant_correlations', async () => { - // Create an array of ranges [2, 4, 6, ..., 98] - const percentileAggregationPercents = range(2, 100, 2); - const { percentiles: percentilesRecords } = - await fetchTransactionDurationPercentiles( + return withApmSpan( + 'get_significant_correlations', + async () => + await fetchSignificantCorrelations( esClient, paramsWithIndex, - percentileAggregationPercents - ); - const percentiles = Object.values(percentilesRecords); - - const { expectations, ranges } = - computeExpectationsAndRanges(percentiles); + fieldValuePairs + ) + ); + }, +}); - const { fractions, totalDocCount } = - await fetchTransactionDurationFractions( - esClient, - paramsWithIndex, - ranges - ); +const pValuesRoute = createApmServerRoute({ + endpoint: 'POST /internal/apm/correlations/p_values', + params: t.type({ + body: t.intersection([ + t.partial({ + serviceName: t.string, + transactionName: t.string, + transactionType: t.string, + }), + environmentRt, + kueryRt, + rangeRt, + t.type({ + fieldCandidates: t.array(t.string), + }), + ]), + }), + options: { tags: ['access:apm'] }, + handler: async (resources) => { + const { indices } = await setupRequest(resources); + const esClient = resources.context.core.elasticsearch.client.asCurrentUser; - const histogramRangeSteps = - await fetchTransactionDurationHistogramRangeSteps( - esClient, - paramsWithIndex - ); + const { fieldCandidates, ...params } = resources.params.body; - const latencyCorrelations: LatencyCorrelation[] = []; + const paramsWithIndex = { + ...params, + index: indices.transaction, + }; - for await (const item of fetchTransactionDurationHistograms( - esClient, - () => {}, - paramsWithIndex, - expectations, - ranges, - fractions, - histogramRangeSteps, - totalDocCount, - fieldValuePairs - )) { - if (item !== undefined) { - latencyCorrelations.push(item); - } - } - - // TODO Fix CCS warning - return { latencyCorrelations, ccsWarning: false }; - }); + return withApmSpan( + 'get_p_values', + async () => await fetchPValues(esClient, paramsWithIndex, fieldCandidates) + ); }, }); export const correlationsRouteRepository = createApmServerRouteRepository() + .add(pValuesRoute) .add(fieldCandidatesRoute) .add(fieldStatsRoute) .add(fieldValuePairsRoute) From 37aa29b6ab42f09f4e1eaec7c07c84a4bf231fff Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Sun, 17 Oct 2021 13:25:37 +0200 Subject: [PATCH 05/39] [ML] Fix types. --- .../queries/query_field_value_pairs.test.ts | 18 +++----- .../queries/query_field_value_pairs.ts | 44 ++++++++----------- .../query_histograms_generator.test.ts | 25 +++-------- .../queries/query_histograms_generator.ts | 10 ++--- .../queries/query_significant_correlations.ts | 1 - x-pack/plugins/apm/server/plugin.ts | 1 - 6 files changed, 34 insertions(+), 65 deletions(-) diff --git a/x-pack/plugins/apm/server/lib/search_strategies/queries/query_field_value_pairs.test.ts b/x-pack/plugins/apm/server/lib/search_strategies/queries/query_field_value_pairs.test.ts index ab7a0b4e02072..64441d1421821 100644 --- a/x-pack/plugins/apm/server/lib/search_strategies/queries/query_field_value_pairs.test.ts +++ b/x-pack/plugins/apm/server/lib/search_strategies/queries/query_field_value_pairs.test.ts @@ -10,9 +10,6 @@ import type { estypes } from '@elastic/elasticsearch'; import type { ElasticsearchClient } from 'src/core/server'; import { ENVIRONMENT_ALL } from '../../../../common/environment_filter_values'; -import { searchServiceLogProvider } from '../search_service_log'; -import { latencyCorrelationsSearchServiceStateProvider } from '../latency_correlations/latency_correlations_search_service_state'; - import { fetchTransactionDurationFieldValuePairs, getTermsAggRequest, @@ -66,20 +63,16 @@ describe('query_field_value_pairs', () => { search: esClientSearchMock, } as unknown as ElasticsearchClient; - const { addLogMessage, getLogMessages } = searchServiceLogProvider(); - const state = latencyCorrelationsSearchServiceStateProvider(); - const resp = await fetchTransactionDurationFieldValuePairs( esClientMock, params, - fieldCandidates, - state, - addLogMessage + fieldCandidates ); - const { progress } = state.getState(); + // TODO PROGRESS + // const { progress } = state.getState(); - expect(progress.loadedFieldValuePairs).toBe(1); + // expect(progress.loadedFieldValuePairs).toBe(1); expect(resp).toEqual([ { fieldName: 'myFieldCandidate1', fieldValue: 'myValue1' }, { fieldName: 'myFieldCandidate1', fieldValue: 'myValue2' }, @@ -89,7 +82,8 @@ describe('query_field_value_pairs', () => { { fieldName: 'myFieldCandidate3', fieldValue: 'myValue2' }, ]); expect(esClientSearchMock).toHaveBeenCalledTimes(3); - expect(getLogMessages()).toEqual([]); + // TODO Log + // expect(getLogMessages()).toEqual([]); }); }); }); diff --git a/x-pack/plugins/apm/server/lib/search_strategies/queries/query_field_value_pairs.ts b/x-pack/plugins/apm/server/lib/search_strategies/queries/query_field_value_pairs.ts index 97584133c964f..1219089c1594d 100644 --- a/x-pack/plugins/apm/server/lib/search_strategies/queries/query_field_value_pairs.ts +++ b/x-pack/plugins/apm/server/lib/search_strategies/queries/query_field_value_pairs.ts @@ -14,7 +14,6 @@ import type { SearchStrategyParams, } from '../../../../common/search_strategies/types'; -import type { SearchServiceLog } from '../search_service_log'; import { TERMS_SIZE } from '../constants'; import { getQueryWithParams } from './get_query_with_params'; @@ -42,19 +41,19 @@ export const getTermsAggRequest = ( const fetchTransactionDurationFieldTerms = async ( esClient: ElasticsearchClient, params: SearchStrategyParams, - fieldName: string, - addLogMessage?: SearchServiceLog['addLogMessage'] + fieldName: string ): Promise => { try { const resp = await esClient.search(getTermsAggRequest(params, fieldName)); if (resp.body.aggregations === undefined) { - if (addLogMessage) { - addLogMessage( - `Failed to fetch terms for field candidate ${fieldName} fieldValuePairs, no aggregations returned.`, - JSON.stringify(resp) - ); - } + // TODO LOG + // if (addLogMessage) { + // addLogMessage( + // `Failed to fetch terms for field candidate ${fieldName} fieldValuePairs, no aggregations returned.`, + // JSON.stringify(resp) + // ); + // } return []; } const buckets = ( @@ -70,12 +69,13 @@ const fetchTransactionDurationFieldTerms = async ( })); } } catch (e) { - if (addLogMessage) { - addLogMessage( - `Failed to fetch terms for field candidate ${fieldName} fieldValuePairs.`, - JSON.stringify(e) - ); - } + // TODO LOG + // if (addLogMessage) { + // addLogMessage( + // `Failed to fetch terms for field candidate ${fieldName} fieldValuePairs.`, + // JSON.stringify(e) + // ); + // } } return []; @@ -97,24 +97,16 @@ async function fetchInSequence( export const fetchTransactionDurationFieldValuePairs = async ( esClient: ElasticsearchClient, params: SearchStrategyParams, - fieldCandidates: string[], - addLogMessage?: SearchServiceLog['addLogMessage'] + fieldCandidates: string[] ): Promise => { - let fieldValuePairsProgress = 1; - return await fetchInSequence( fieldCandidates, async function (fieldCandidate: string) { - const fieldTerms = await fetchTransactionDurationFieldTerms( + return await fetchTransactionDurationFieldTerms( esClient, params, - fieldCandidate, - addLogMessage + fieldCandidate ); - - fieldValuePairsProgress++; - - return fieldTerms; } ); }; diff --git a/x-pack/plugins/apm/server/lib/search_strategies/queries/query_histograms_generator.test.ts b/x-pack/plugins/apm/server/lib/search_strategies/queries/query_histograms_generator.test.ts index 00e8c26497eb2..4dc71a09191e0 100644 --- a/x-pack/plugins/apm/server/lib/search_strategies/queries/query_histograms_generator.test.ts +++ b/x-pack/plugins/apm/server/lib/search_strategies/queries/query_histograms_generator.test.ts @@ -10,9 +10,6 @@ import type { estypes } from '@elastic/elasticsearch'; import type { ElasticsearchClient } from 'src/core/server'; import { ENVIRONMENT_ALL } from '../../../../common/environment_filter_values'; -import { searchServiceLogProvider } from '../search_service_log'; -import { latencyCorrelationsSearchServiceStateProvider } from '../latency_correlations/latency_correlations_search_service_state'; - import { fetchTransactionDurationHistograms } from './query_histograms_generator'; const params = { @@ -54,17 +51,12 @@ describe('query_histograms_generator', () => { search: esClientSearchMock, } as unknown as ElasticsearchClient; - const state = latencyCorrelationsSearchServiceStateProvider(); - const { addLogMessage, getLogMessages } = searchServiceLogProvider(); - let loadedHistograms = 0; const items = []; for await (const item of fetchTransactionDurationHistograms( esClientMock, - addLogMessage, params, - state, expectations, ranges, fractions, @@ -81,11 +73,11 @@ describe('query_histograms_generator', () => { expect(items.length).toEqual(0); expect(loadedHistograms).toEqual(3); expect(esClientSearchMock).toHaveBeenCalledTimes(3); - expect(getLogMessages().map((d) => d.split(': ')[1])).toEqual([ - "Failed to fetch correlation/kstest for 'the-field-name-1/the-field-value-1'", - "Failed to fetch correlation/kstest for 'the-field-name-2/the-field-value-2'", - "Failed to fetch correlation/kstest for 'the-field-name-2/the-field-value-3'", - ]); + // expect(getLogMessages().map((d) => d.split(': ')[1])).toEqual([ + // "Failed to fetch correlation/kstest for 'the-field-name-1/the-field-value-1'", + // "Failed to fetch correlation/kstest for 'the-field-name-2/the-field-value-2'", + // "Failed to fetch correlation/kstest for 'the-field-name-2/the-field-value-3'", + // ]); }); it('returns items with correlation and ks-test value', async () => { @@ -112,17 +104,12 @@ describe('query_histograms_generator', () => { search: esClientSearchMock, } as unknown as ElasticsearchClient; - const state = latencyCorrelationsSearchServiceStateProvider(); - const { addLogMessage, getLogMessages } = searchServiceLogProvider(); - let loadedHistograms = 0; const items = []; for await (const item of fetchTransactionDurationHistograms( esClientMock, - addLogMessage, params, - state, expectations, ranges, fractions, @@ -139,7 +126,7 @@ describe('query_histograms_generator', () => { expect(items.length).toEqual(3); expect(loadedHistograms).toEqual(3); expect(esClientSearchMock).toHaveBeenCalledTimes(6); - expect(getLogMessages().length).toEqual(0); + // expect(getLogMessages().length).toEqual(0); }); }); }); diff --git a/x-pack/plugins/apm/server/lib/search_strategies/queries/query_histograms_generator.ts b/x-pack/plugins/apm/server/lib/search_strategies/queries/query_histograms_generator.ts index f9c3bb4358a66..d11d143724a1c 100644 --- a/x-pack/plugins/apm/server/lib/search_strategies/queries/query_histograms_generator.ts +++ b/x-pack/plugins/apm/server/lib/search_strategies/queries/query_histograms_generator.ts @@ -14,7 +14,6 @@ import type { SearchStrategyParams, } from '../../../../common/search_strategies/types'; -import type { SearchServiceLog } from '../search_service_log'; import { CORRELATION_THRESHOLD, KS_TEST_THRESHOLD } from '../constants'; import { getPrioritizedFieldValuePairs } from './get_prioritized_field_value_pairs'; @@ -23,7 +22,6 @@ import { fetchTransactionDurationRanges } from './query_ranges'; export async function* fetchTransactionDurationHistograms( esClient: ElasticsearchClient, - addLogMessage: SearchServiceLog['addLogMessage'], params: SearchStrategyParams, expectations: number[], ranges: estypes.AggregationsAggregationRange[], @@ -75,10 +73,10 @@ export async function* fetchTransactionDurationHistograms( // don't fail the whole process for individual correlation queries, // just add the error to the internal log and check if we'd want to set the // cross-cluster search compatibility warning to true. - addLogMessage( - `Failed to fetch correlation/kstest for '${item.fieldName}/${item.fieldValue}'`, - JSON.stringify(e) - ); + // addLogMessage( + // `Failed to fetch correlation/kstest for '${item.fieldName}/${item.fieldValue}'`, + // JSON.stringify(e) + // ); // TODO return CCS warning // if (params?.index.includes(':')) { // state.setCcsWarning(true); diff --git a/x-pack/plugins/apm/server/lib/search_strategies/queries/query_significant_correlations.ts b/x-pack/plugins/apm/server/lib/search_strategies/queries/query_significant_correlations.ts index 558c130f07de2..b88374adc2dcd 100644 --- a/x-pack/plugins/apm/server/lib/search_strategies/queries/query_significant_correlations.ts +++ b/x-pack/plugins/apm/server/lib/search_strategies/queries/query_significant_correlations.ts @@ -55,7 +55,6 @@ export const fetchSignificantCorrelations = async ( for await (const item of fetchTransactionDurationHistograms( esClient, - () => {}, paramsWithIndex, expectations, ranges, diff --git a/x-pack/plugins/apm/server/plugin.ts b/x-pack/plugins/apm/server/plugin.ts index ca55486b70acb..4e2ee4f37a8e6 100644 --- a/x-pack/plugins/apm/server/plugin.ts +++ b/x-pack/plugins/apm/server/plugin.ts @@ -15,7 +15,6 @@ import { PluginInitializerContext, } from 'src/core/server'; import { isEmpty, mapValues } from 'lodash'; -import { SavedObjectsClient } from '../../../../src/core/server'; import { mappingFromFieldMap } from '../../rule_registry/common/mapping_from_field_map'; import { Dataset } from '../../rule_registry/server'; import { APMConfig, APM_SERVER_FEATURE_ID } from '.'; From 414a992b2fadd88abd4adde815c64c897941a855 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Sun, 17 Oct 2021 13:40:50 +0200 Subject: [PATCH 06/39] [ML] Rename common/search_strategies to common/correlations. --- .../{search_strategies => correlations}/constants.ts | 7 ------- .../failed_transactions_correlations/constants.ts | 0 .../failed_transactions_correlations/types.ts | 4 ---- .../field_stats_types.ts | 0 .../latency_correlations/types.ts | 0 .../common/{search_strategies => correlations}/types.ts | 0 .../app/correlations/context_popover/context_popover.tsx | 2 +- .../app/correlations/context_popover/top_values.tsx | 2 +- .../components/app/correlations/correlations_table.tsx | 2 +- .../app/correlations/failed_transactions_correlations.tsx | 6 +++--- .../app/correlations/latency_correlations.test.tsx | 4 ++-- .../components/app/correlations/latency_correlations.tsx | 6 +++--- .../correlations/use_failed_transactions_correlations.ts | 6 +++--- .../app/correlations/use_latency_correlations.ts | 6 +++--- ...et_failed_transactions_correlation_impact_label.test.ts | 2 +- .../get_failed_transactions_correlation_impact_label.ts | 4 ++-- .../app/correlations/utils/get_overall_histogram.test.ts | 2 +- .../app/correlations/utils/get_overall_histogram.ts | 2 +- .../app/transaction_details/distribution/index.tsx | 2 +- .../use_transaction_distribution_chart_data.ts | 4 ++-- .../charts/transaction_distribution_chart/index.test.tsx | 2 +- .../shared/charts/transaction_distribution_chart/index.tsx | 2 +- .../apm/public/hooks/use_latency_overall_distribution.ts | 2 +- x-pack/plugins/apm/server/lib/latency/types.ts | 2 +- .../queries/field_stats/get_boolean_field_stats.ts | 4 ++-- .../queries/field_stats/get_fields_stats.ts | 4 ++-- .../queries/field_stats/get_keyword_field_stats.ts | 4 ++-- .../queries/field_stats/get_numeric_field_stats.ts | 4 ++-- .../server/lib/search_strategies/queries/get_filters.ts | 2 +- .../queries/get_prioritized_field_value_pairs.ts | 2 +- .../lib/search_strategies/queries/get_query_with_params.ts | 2 +- .../lib/search_strategies/queries/get_request_base.ts | 2 +- .../lib/search_strategies/queries/query_correlation.ts | 2 +- .../search_strategies/queries/query_failure_correlation.ts | 4 ++-- .../search_strategies/queries/query_field_candidates.ts | 2 +- .../search_strategies/queries/query_field_value_pairs.ts | 2 +- .../lib/search_strategies/queries/query_fractions.ts | 2 +- .../lib/search_strategies/queries/query_histogram.ts | 2 +- .../queries/query_histogram_range_steps.ts | 2 +- .../queries/query_histograms_generator.ts | 2 +- .../server/lib/search_strategies/queries/query_p_values.ts | 4 ++-- .../lib/search_strategies/queries/query_percentiles.ts | 2 +- .../server/lib/search_strategies/queries/query_ranges.ts | 2 +- .../queries/query_significant_correlations.ts | 4 ++-- 44 files changed, 56 insertions(+), 67 deletions(-) rename x-pack/plugins/apm/common/{search_strategies => correlations}/constants.ts (51%) rename x-pack/plugins/apm/common/{search_strategies => correlations}/failed_transactions_correlations/constants.ts (100%) rename x-pack/plugins/apm/common/{search_strategies => correlations}/failed_transactions_correlations/types.ts (92%) rename x-pack/plugins/apm/common/{search_strategies => correlations}/field_stats_types.ts (100%) rename x-pack/plugins/apm/common/{search_strategies => correlations}/latency_correlations/types.ts (100%) rename x-pack/plugins/apm/common/{search_strategies => correlations}/types.ts (100%) diff --git a/x-pack/plugins/apm/common/search_strategies/constants.ts b/x-pack/plugins/apm/common/correlations/constants.ts similarity index 51% rename from x-pack/plugins/apm/common/search_strategies/constants.ts rename to x-pack/plugins/apm/common/correlations/constants.ts index 58203c93e5a42..4732f66e59c91 100644 --- a/x-pack/plugins/apm/common/search_strategies/constants.ts +++ b/x-pack/plugins/apm/common/correlations/constants.ts @@ -5,11 +5,4 @@ * 2.0. */ -export const APM_SEARCH_STRATEGIES = { - APM_FAILED_TRANSACTIONS_CORRELATIONS: 'apmFailedTransactionsCorrelations', - APM_LATENCY_CORRELATIONS: 'apmLatencyCorrelations', -} as const; -export type ApmSearchStrategies = - typeof APM_SEARCH_STRATEGIES[keyof typeof APM_SEARCH_STRATEGIES]; - export const DEFAULT_PERCENTILE_THRESHOLD = 95; diff --git a/x-pack/plugins/apm/common/search_strategies/failed_transactions_correlations/constants.ts b/x-pack/plugins/apm/common/correlations/failed_transactions_correlations/constants.ts similarity index 100% rename from x-pack/plugins/apm/common/search_strategies/failed_transactions_correlations/constants.ts rename to x-pack/plugins/apm/common/correlations/failed_transactions_correlations/constants.ts diff --git a/x-pack/plugins/apm/common/search_strategies/failed_transactions_correlations/types.ts b/x-pack/plugins/apm/common/correlations/failed_transactions_correlations/types.ts similarity index 92% rename from x-pack/plugins/apm/common/search_strategies/failed_transactions_correlations/types.ts rename to x-pack/plugins/apm/common/correlations/failed_transactions_correlations/types.ts index 28ce2ff24b961..94081f8b9d2b6 100644 --- a/x-pack/plugins/apm/common/search_strategies/failed_transactions_correlations/types.ts +++ b/x-pack/plugins/apm/common/correlations/failed_transactions_correlations/types.ts @@ -24,10 +24,6 @@ export interface FailedTransactionsCorrelation extends FieldValuePair { export type FailedTransactionsCorrelationsImpactThreshold = typeof FAILED_TRANSACTIONS_IMPACT_THRESHOLD[keyof typeof FAILED_TRANSACTIONS_IMPACT_THRESHOLD]; -export interface FailedTransactionsCorrelationsParams { - percentileThreshold: number; -} - export interface FailedTransactionsCorrelationsRawResponse { log: string[]; failedTransactionsCorrelations?: FailedTransactionsCorrelation[]; diff --git a/x-pack/plugins/apm/common/search_strategies/field_stats_types.ts b/x-pack/plugins/apm/common/correlations/field_stats_types.ts similarity index 100% rename from x-pack/plugins/apm/common/search_strategies/field_stats_types.ts rename to x-pack/plugins/apm/common/correlations/field_stats_types.ts diff --git a/x-pack/plugins/apm/common/search_strategies/latency_correlations/types.ts b/x-pack/plugins/apm/common/correlations/latency_correlations/types.ts similarity index 100% rename from x-pack/plugins/apm/common/search_strategies/latency_correlations/types.ts rename to x-pack/plugins/apm/common/correlations/latency_correlations/types.ts diff --git a/x-pack/plugins/apm/common/search_strategies/types.ts b/x-pack/plugins/apm/common/correlations/types.ts similarity index 100% rename from x-pack/plugins/apm/common/search_strategies/types.ts rename to x-pack/plugins/apm/common/correlations/types.ts diff --git a/x-pack/plugins/apm/public/components/app/correlations/context_popover/context_popover.tsx b/x-pack/plugins/apm/public/components/app/correlations/context_popover/context_popover.tsx index 4a0f7d81e24dc..7165aa67a5e5a 100644 --- a/x-pack/plugins/apm/public/components/app/correlations/context_popover/context_popover.tsx +++ b/x-pack/plugins/apm/public/components/app/correlations/context_popover/context_popover.tsx @@ -19,7 +19,7 @@ import { import React, { Fragment, useState } from 'react'; import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n/react'; -import { FieldStats } from '../../../../../common/search_strategies/field_stats_types'; +import { FieldStats } from '../../../../../common/correlations/field_stats_types'; import { OnAddFilter, TopValues } from './top_values'; import { useTheme } from '../../../../hooks/use_theme'; diff --git a/x-pack/plugins/apm/public/components/app/correlations/context_popover/top_values.tsx b/x-pack/plugins/apm/public/components/app/correlations/context_popover/top_values.tsx index 803b474fe7754..05b4f6d56fa45 100644 --- a/x-pack/plugins/apm/public/components/app/correlations/context_popover/top_values.tsx +++ b/x-pack/plugins/apm/public/components/app/correlations/context_popover/top_values.tsx @@ -14,7 +14,7 @@ import { EuiToolTip, } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; -import { FieldStats } from '../../../../../common/search_strategies/field_stats_types'; +import { FieldStats } from '../../../../../common/correlations/field_stats_types'; import { asPercent } from '../../../../../common/utils/formatters'; import { useTheme } from '../../../../hooks/use_theme'; diff --git a/x-pack/plugins/apm/public/components/app/correlations/correlations_table.tsx b/x-pack/plugins/apm/public/components/app/correlations/correlations_table.tsx index eda3b64c309cc..a2026b0a8abea 100644 --- a/x-pack/plugins/apm/public/components/app/correlations/correlations_table.tsx +++ b/x-pack/plugins/apm/public/components/app/correlations/correlations_table.tsx @@ -14,7 +14,7 @@ import type { Criteria } from '@elastic/eui/src/components/basic_table/basic_tab import { FETCH_STATUS } from '../../../hooks/use_fetcher'; import { useUiTracker } from '../../../../../observability/public'; import { useTheme } from '../../../hooks/use_theme'; -import type { FieldValuePair } from '../../../../common/search_strategies/types'; +import type { FieldValuePair } from '../../../../common/correlations/types'; const PAGINATION_SIZE_OPTIONS = [5, 10, 20, 50]; diff --git a/x-pack/plugins/apm/public/components/app/correlations/failed_transactions_correlations.tsx b/x-pack/plugins/apm/public/components/app/correlations/failed_transactions_correlations.tsx index 661bae27acea9..55cb2551f2ebb 100644 --- a/x-pack/plugins/apm/public/components/app/correlations/failed_transactions_correlations.tsx +++ b/x-pack/plugins/apm/public/components/app/correlations/failed_transactions_correlations.tsx @@ -35,9 +35,9 @@ import { } from '../../../../../observability/public'; import { asPercent } from '../../../../common/utils/formatters'; -import { FailedTransactionsCorrelation } from '../../../../common/search_strategies/failed_transactions_correlations/types'; -import { DEFAULT_PERCENTILE_THRESHOLD } from '../../../../common/search_strategies/constants'; -import { FieldStats } from '../../../../common/search_strategies/field_stats_types'; +import { FailedTransactionsCorrelation } from '../../../../common/correlations/failed_transactions_correlations/types'; +import { DEFAULT_PERCENTILE_THRESHOLD } from '../../../../common/correlations/constants'; +import { FieldStats } from '../../../../common/correlations/field_stats_types'; import { useApmPluginContext } from '../../../context/apm_plugin/use_apm_plugin_context'; import { useLocalStorage } from '../../../hooks/useLocalStorage'; diff --git a/x-pack/plugins/apm/public/components/app/correlations/latency_correlations.test.tsx b/x-pack/plugins/apm/public/components/app/correlations/latency_correlations.test.tsx index 918f94e64ef09..b2170c99bb6fe 100644 --- a/x-pack/plugins/apm/public/components/app/correlations/latency_correlations.test.tsx +++ b/x-pack/plugins/apm/public/components/app/correlations/latency_correlations.test.tsx @@ -18,8 +18,8 @@ import { dataPluginMock } from 'src/plugins/data/public/mocks'; import type { IKibanaSearchResponse } from 'src/plugins/data/public'; import { EuiThemeProvider } from 'src/plugins/kibana_react/common'; import { createKibanaReactContext } from 'src/plugins/kibana_react/public'; -import type { LatencyCorrelationsRawResponse } from '../../../../common/search_strategies/latency_correlations/types'; -import type { RawResponseBase } from '../../../../common/search_strategies/types'; +import type { LatencyCorrelationsRawResponse } from '../../../../common/correlations/latency_correlations/types'; +import type { RawResponseBase } from '../../../../common/correlations/types'; import { MockUrlParamsContextProvider } from '../../../context/url_params_context/mock_url_params_context_provider'; import { ApmPluginContextValue } from '../../../context/apm_plugin/apm_plugin_context'; import { diff --git a/x-pack/plugins/apm/public/components/app/correlations/latency_correlations.tsx b/x-pack/plugins/apm/public/components/app/correlations/latency_correlations.tsx index 057bf1d2e5622..dad44baabac4c 100644 --- a/x-pack/plugins/apm/public/components/app/correlations/latency_correlations.tsx +++ b/x-pack/plugins/apm/public/components/app/correlations/latency_correlations.tsx @@ -31,9 +31,9 @@ import { } from '../../../../../observability/public'; import { asPreciseDecimal } from '../../../../common/utils/formatters'; -import { DEFAULT_PERCENTILE_THRESHOLD } from '../../../../common/search_strategies/constants'; -import { LatencyCorrelation } from '../../../../common/search_strategies/latency_correlations/types'; -import { FieldStats } from '../../../../common/search_strategies/field_stats_types'; +import { DEFAULT_PERCENTILE_THRESHOLD } from '../../../../common/correlations/constants'; +import { LatencyCorrelation } from '../../../../common/correlations/latency_correlations/types'; +import { FieldStats } from '../../../../common/correlations/field_stats_types'; import { useApmPluginContext } from '../../../context/apm_plugin/use_apm_plugin_context'; import { FETCH_STATUS } from '../../../hooks/use_fetcher'; diff --git a/x-pack/plugins/apm/public/components/app/correlations/use_failed_transactions_correlations.ts b/x-pack/plugins/apm/public/components/app/correlations/use_failed_transactions_correlations.ts index bc5e994fabd73..ee13d5967d720 100644 --- a/x-pack/plugins/apm/public/components/app/correlations/use_failed_transactions_correlations.ts +++ b/x-pack/plugins/apm/public/components/app/correlations/use_failed_transactions_correlations.ts @@ -12,12 +12,12 @@ import { IHttpFetchError } from 'src/core/public'; import { EVENT_OUTCOME } from '../../../../common/elasticsearch_fieldnames'; import { EventOutcome } from '../../../../common/event_outcome'; -import { DEFAULT_PERCENTILE_THRESHOLD } from '../../../../common/search_strategies/constants'; -import type { RawResponseBase } from '../../../../common/search_strategies/types'; +import { DEFAULT_PERCENTILE_THRESHOLD } from '../../../../common/correlations/constants'; +import type { RawResponseBase } from '../../../../common/correlations/types'; import type { FailedTransactionsCorrelation, FailedTransactionsCorrelationsRawResponse, -} from '../../../../common/search_strategies/failed_transactions_correlations/types'; +} from '../../../../common/correlations/failed_transactions_correlations/types'; import { useApmServiceContext } from '../../../context/apm_service/use_apm_service_context'; import { useUrlParams } from '../../../context/url_params_context/use_url_params'; diff --git a/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts b/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts index a4a8d18ed28ff..cd7f5e6abd3dc 100644 --- a/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts +++ b/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts @@ -10,12 +10,12 @@ import { chunk } from 'lodash'; import { IHttpFetchError } from 'src/core/public'; -import { DEFAULT_PERCENTILE_THRESHOLD } from '../../../../common/search_strategies/constants'; -import type { RawResponseBase } from '../../../../common/search_strategies/types'; +import { DEFAULT_PERCENTILE_THRESHOLD } from '../../../../common/correlations/constants'; +import type { RawResponseBase } from '../../../../common/correlations/types'; import type { LatencyCorrelation, LatencyCorrelationsRawResponse, -} from '../../../../common/search_strategies/latency_correlations/types'; +} from '../../../../common/correlations/latency_correlations/types'; import { useApmServiceContext } from '../../../context/apm_service/use_apm_service_context'; import { useUrlParams } from '../../../context/url_params_context/use_url_params'; diff --git a/x-pack/plugins/apm/public/components/app/correlations/utils/get_failed_transactions_correlation_impact_label.test.ts b/x-pack/plugins/apm/public/components/app/correlations/utils/get_failed_transactions_correlation_impact_label.test.ts index e4c08b42b2420..d35833295703f 100644 --- a/x-pack/plugins/apm/public/components/app/correlations/utils/get_failed_transactions_correlation_impact_label.test.ts +++ b/x-pack/plugins/apm/public/components/app/correlations/utils/get_failed_transactions_correlation_impact_label.test.ts @@ -6,7 +6,7 @@ */ import { getFailedTransactionsCorrelationImpactLabel } from './get_failed_transactions_correlation_impact_label'; -import { FAILED_TRANSACTIONS_IMPACT_THRESHOLD } from '../../../../../common/search_strategies/failed_transactions_correlations/constants'; +import { FAILED_TRANSACTIONS_IMPACT_THRESHOLD } from '../../../../../common/correlations/failed_transactions_correlations/constants'; const EXPECTED_RESULT = { HIGH: { diff --git a/x-pack/plugins/apm/public/components/app/correlations/utils/get_failed_transactions_correlation_impact_label.ts b/x-pack/plugins/apm/public/components/app/correlations/utils/get_failed_transactions_correlation_impact_label.ts index cbfaee88ff6f4..d5d0fd4dcae51 100644 --- a/x-pack/plugins/apm/public/components/app/correlations/utils/get_failed_transactions_correlation_impact_label.ts +++ b/x-pack/plugins/apm/public/components/app/correlations/utils/get_failed_transactions_correlation_impact_label.ts @@ -8,8 +8,8 @@ import { FailedTransactionsCorrelation, FailedTransactionsCorrelationsImpactThreshold, -} from '../../../../../common/search_strategies/failed_transactions_correlations/types'; -import { FAILED_TRANSACTIONS_IMPACT_THRESHOLD } from '../../../../../common/search_strategies/failed_transactions_correlations/constants'; +} from '../../../../../common/correlations/failed_transactions_correlations/types'; +import { FAILED_TRANSACTIONS_IMPACT_THRESHOLD } from '../../../../../common/correlations/failed_transactions_correlations/constants'; export function getFailedTransactionsCorrelationImpactLabel( pValue: FailedTransactionsCorrelation['pValue'] diff --git a/x-pack/plugins/apm/public/components/app/correlations/utils/get_overall_histogram.test.ts b/x-pack/plugins/apm/public/components/app/correlations/utils/get_overall_histogram.test.ts index c323b69594013..abf2bd9bec563 100644 --- a/x-pack/plugins/apm/public/components/app/correlations/utils/get_overall_histogram.test.ts +++ b/x-pack/plugins/apm/public/components/app/correlations/utils/get_overall_histogram.test.ts @@ -5,7 +5,7 @@ * 2.0. */ -import type { LatencyCorrelationsRawResponse } from '../../../../../common/search_strategies/latency_correlations/types'; +import type { LatencyCorrelationsRawResponse } from '../../../../../common/correlations/latency_correlations/types'; import { getOverallHistogram } from './get_overall_histogram'; diff --git a/x-pack/plugins/apm/public/components/app/correlations/utils/get_overall_histogram.ts b/x-pack/plugins/apm/public/components/app/correlations/utils/get_overall_histogram.ts index 3a90eb4b89123..36b9f2e1188c3 100644 --- a/x-pack/plugins/apm/public/components/app/correlations/utils/get_overall_histogram.ts +++ b/x-pack/plugins/apm/public/components/app/correlations/utils/get_overall_histogram.ts @@ -5,7 +5,7 @@ * 2.0. */ -import type { LatencyCorrelationsRawResponse } from '../../../../../common/search_strategies/latency_correlations/types'; +import type { LatencyCorrelationsRawResponse } from '../../../../../common/correlations/latency_correlations/types'; import { FETCH_STATUS } from '../../../../hooks/use_fetcher'; diff --git a/x-pack/plugins/apm/public/components/app/transaction_details/distribution/index.tsx b/x-pack/plugins/apm/public/components/app/transaction_details/distribution/index.tsx index 8862596fd6d2a..0b58db492b472 100644 --- a/x-pack/plugins/apm/public/components/app/transaction_details/distribution/index.tsx +++ b/x-pack/plugins/apm/public/components/app/transaction_details/distribution/index.tsx @@ -23,7 +23,7 @@ import { FormattedMessage } from '@kbn/i18n/react'; import { useUiTracker } from '../../../../../../observability/public'; import { getDurationFormatter } from '../../../../../common/utils/formatters'; -import { DEFAULT_PERCENTILE_THRESHOLD } from '../../../../../common/search_strategies/constants'; +import { DEFAULT_PERCENTILE_THRESHOLD } from '../../../../../common/correlations/constants'; import { useUrlParams } from '../../../../context/url_params_context/use_url_params'; import { FETCH_STATUS } from '../../../../hooks/use_fetcher'; diff --git a/x-pack/plugins/apm/public/components/app/transaction_details/distribution/use_transaction_distribution_chart_data.ts b/x-pack/plugins/apm/public/components/app/transaction_details/distribution/use_transaction_distribution_chart_data.ts index 0edf5f648a980..650756b3fc538 100644 --- a/x-pack/plugins/apm/public/components/app/transaction_details/distribution/use_transaction_distribution_chart_data.ts +++ b/x-pack/plugins/apm/public/components/app/transaction_details/distribution/use_transaction_distribution_chart_data.ts @@ -9,8 +9,8 @@ import { useEffect, useMemo } from 'react'; import { i18n } from '@kbn/i18n'; -import { DEFAULT_PERCENTILE_THRESHOLD } from '../../../../../common/search_strategies/constants'; -import { RawSearchStrategyClientParams } from '../../../../../common/search_strategies/types'; +import { DEFAULT_PERCENTILE_THRESHOLD } from '../../../../../common/correlations/constants'; +import { RawSearchStrategyClientParams } from '../../../../../common/correlations/types'; import { EVENT_OUTCOME } from '../../../../../common/elasticsearch_fieldnames'; import { EventOutcome } from '../../../../../common/event_outcome'; diff --git a/x-pack/plugins/apm/public/components/shared/charts/transaction_distribution_chart/index.test.tsx b/x-pack/plugins/apm/public/components/shared/charts/transaction_distribution_chart/index.test.tsx index 8a57063ac4d45..b8d070c64ca9f 100644 --- a/x-pack/plugins/apm/public/components/shared/charts/transaction_distribution_chart/index.test.tsx +++ b/x-pack/plugins/apm/public/components/shared/charts/transaction_distribution_chart/index.test.tsx @@ -5,7 +5,7 @@ * 2.0. */ -import type { HistogramItem } from '../../../../../common/search_strategies/types'; +import type { HistogramItem } from '../../../../../common/correlations/types'; import { replaceHistogramDotsWithBars } from './index'; diff --git a/x-pack/plugins/apm/public/components/shared/charts/transaction_distribution_chart/index.tsx b/x-pack/plugins/apm/public/components/shared/charts/transaction_distribution_chart/index.tsx index 1a1ea13fa45ec..d7f8849decd78 100644 --- a/x-pack/plugins/apm/public/components/shared/charts/transaction_distribution_chart/index.tsx +++ b/x-pack/plugins/apm/public/components/shared/charts/transaction_distribution_chart/index.tsx @@ -32,7 +32,7 @@ import { i18n } from '@kbn/i18n'; import { useChartTheme } from '../../../../../../observability/public'; import { getDurationFormatter } from '../../../../../common/utils/formatters'; -import type { HistogramItem } from '../../../../../common/search_strategies/types'; +import type { HistogramItem } from '../../../../../common/correlations/types'; import { FETCH_STATUS } from '../../../../hooks/use_fetcher'; import { useTheme } from '../../../../hooks/use_theme'; diff --git a/x-pack/plugins/apm/public/hooks/use_latency_overall_distribution.ts b/x-pack/plugins/apm/public/hooks/use_latency_overall_distribution.ts index 490545344552c..2669179d91991 100644 --- a/x-pack/plugins/apm/public/hooks/use_latency_overall_distribution.ts +++ b/x-pack/plugins/apm/public/hooks/use_latency_overall_distribution.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { DEFAULT_PERCENTILE_THRESHOLD } from '../../common/search_strategies/constants'; +import { DEFAULT_PERCENTILE_THRESHOLD } from '../../common/correlations/constants'; import { useApmServiceContext } from '../context/apm_service/use_apm_service_context'; import { useUrlParams } from '../context/url_params_context/use_url_params'; diff --git a/x-pack/plugins/apm/server/lib/latency/types.ts b/x-pack/plugins/apm/server/lib/latency/types.ts index ed7408c297ad7..ba735db2acebb 100644 --- a/x-pack/plugins/apm/server/lib/latency/types.ts +++ b/x-pack/plugins/apm/server/lib/latency/types.ts @@ -8,7 +8,7 @@ import type { FieldValuePair, SearchStrategyClientParams, -} from '../../../common/search_strategies/types'; +} from '../../../common/correlations/types'; import { Setup } from '../helpers/setup_request'; diff --git a/x-pack/plugins/apm/server/lib/search_strategies/queries/field_stats/get_boolean_field_stats.ts b/x-pack/plugins/apm/server/lib/search_strategies/queries/field_stats/get_boolean_field_stats.ts index 551ecfe3cd4ea..aa11ae1a4a211 100644 --- a/x-pack/plugins/apm/server/lib/search_strategies/queries/field_stats/get_boolean_field_stats.ts +++ b/x-pack/plugins/apm/server/lib/search_strategies/queries/field_stats/get_boolean_field_stats.ts @@ -9,13 +9,13 @@ import { ElasticsearchClient } from 'kibana/server'; import { SearchRequest } from '@elastic/elasticsearch/api/types'; import { estypes } from '@elastic/elasticsearch'; import { buildSamplerAggregation } from '../../utils/field_stats_utils'; -import { FieldValuePair } from '../../../../../common/search_strategies/types'; +import { FieldValuePair } from '../../../../../common/correlations/types'; import { FieldStatsCommonRequestParams, BooleanFieldStats, Aggs, TopValueBucket, -} from '../../../../../common/search_strategies/field_stats_types'; +} from '../../../../../common/correlations/field_stats_types'; import { getQueryWithParams } from '../get_query_with_params'; export const getBooleanFieldStatsRequest = ( diff --git a/x-pack/plugins/apm/server/lib/search_strategies/queries/field_stats/get_fields_stats.ts b/x-pack/plugins/apm/server/lib/search_strategies/queries/field_stats/get_fields_stats.ts index 2e1441ccbd6a1..7b5e53e0dfee7 100644 --- a/x-pack/plugins/apm/server/lib/search_strategies/queries/field_stats/get_fields_stats.ts +++ b/x-pack/plugins/apm/server/lib/search_strategies/queries/field_stats/get_fields_stats.ts @@ -11,14 +11,14 @@ import { ES_FIELD_TYPES } from '@kbn/field-types'; import { FieldValuePair, SearchStrategyParams, -} from '../../../../../common/search_strategies/types'; +} from '../../../../../common/correlations/types'; import { getRequestBase } from '../get_request_base'; import { fetchKeywordFieldStats } from './get_keyword_field_stats'; import { fetchNumericFieldStats } from './get_numeric_field_stats'; import { FieldStats, FieldStatsCommonRequestParams, -} from '../../../../../common/search_strategies/field_stats_types'; +} from '../../../../../common/correlations/field_stats_types'; import { fetchBooleanFieldStats } from './get_boolean_field_stats'; export const fetchFieldsStats = async ( diff --git a/x-pack/plugins/apm/server/lib/search_strategies/queries/field_stats/get_keyword_field_stats.ts b/x-pack/plugins/apm/server/lib/search_strategies/queries/field_stats/get_keyword_field_stats.ts index b15449657cba5..0b2bd797b83d8 100644 --- a/x-pack/plugins/apm/server/lib/search_strategies/queries/field_stats/get_keyword_field_stats.ts +++ b/x-pack/plugins/apm/server/lib/search_strategies/queries/field_stats/get_keyword_field_stats.ts @@ -8,7 +8,7 @@ import { ElasticsearchClient } from 'kibana/server'; import { SearchRequest } from '@elastic/elasticsearch/api/types'; import { estypes } from '@elastic/elasticsearch'; -import { FieldValuePair } from '../../../../../common/search_strategies/types'; +import { FieldValuePair } from '../../../../../common/correlations/types'; import { getQueryWithParams } from '../get_query_with_params'; import { buildSamplerAggregation } from '../../utils/field_stats_utils'; import { @@ -16,7 +16,7 @@ import { KeywordFieldStats, Aggs, TopValueBucket, -} from '../../../../../common/search_strategies/field_stats_types'; +} from '../../../../../common/correlations/field_stats_types'; export const getKeywordFieldStatsRequest = ( params: FieldStatsCommonRequestParams, diff --git a/x-pack/plugins/apm/server/lib/search_strategies/queries/field_stats/get_numeric_field_stats.ts b/x-pack/plugins/apm/server/lib/search_strategies/queries/field_stats/get_numeric_field_stats.ts index bab4a1af29b65..aa423429e2c33 100644 --- a/x-pack/plugins/apm/server/lib/search_strategies/queries/field_stats/get_numeric_field_stats.ts +++ b/x-pack/plugins/apm/server/lib/search_strategies/queries/field_stats/get_numeric_field_stats.ts @@ -14,8 +14,8 @@ import { FieldStatsCommonRequestParams, TopValueBucket, Aggs, -} from '../../../../../common/search_strategies/field_stats_types'; -import { FieldValuePair } from '../../../../../common/search_strategies/types'; +} from '../../../../../common/correlations/field_stats_types'; +import { FieldValuePair } from '../../../../../common/correlations/types'; import { getQueryWithParams } from '../get_query_with_params'; import { buildSamplerAggregation } from '../../utils/field_stats_utils'; diff --git a/x-pack/plugins/apm/server/lib/search_strategies/queries/get_filters.ts b/x-pack/plugins/apm/server/lib/search_strategies/queries/get_filters.ts index 4c91f2ca987b5..b9ce529b1a05b 100644 --- a/x-pack/plugins/apm/server/lib/search_strategies/queries/get_filters.ts +++ b/x-pack/plugins/apm/server/lib/search_strategies/queries/get_filters.ts @@ -15,7 +15,7 @@ import { PROCESSOR_EVENT, } from '../../../../common/elasticsearch_fieldnames'; import { ProcessorEvent } from '../../../../common/processor_event'; -import { SearchStrategyClientParams } from '../../../../common/search_strategies/types'; +import { SearchStrategyClientParams } from '../../../../common/correlations/types'; export function getCorrelationsFilters({ environment, diff --git a/x-pack/plugins/apm/server/lib/search_strategies/queries/get_prioritized_field_value_pairs.ts b/x-pack/plugins/apm/server/lib/search_strategies/queries/get_prioritized_field_value_pairs.ts index 6338422b022da..6e8fba9738730 100644 --- a/x-pack/plugins/apm/server/lib/search_strategies/queries/get_prioritized_field_value_pairs.ts +++ b/x-pack/plugins/apm/server/lib/search_strategies/queries/get_prioritized_field_value_pairs.ts @@ -8,7 +8,7 @@ import { FIELDS_TO_ADD_AS_CANDIDATE } from '../constants'; import { hasPrefixToInclude } from '../utils'; -import type { FieldValuePair } from '../../../../common/search_strategies/types'; +import type { FieldValuePair } from '../../../../common/correlations/types'; export const getPrioritizedFieldValuePairs = ( fieldValuePairs: FieldValuePair[] diff --git a/x-pack/plugins/apm/server/lib/search_strategies/queries/get_query_with_params.ts b/x-pack/plugins/apm/server/lib/search_strategies/queries/get_query_with_params.ts index 31a98b0a6bb18..22b8ce2353747 100644 --- a/x-pack/plugins/apm/server/lib/search_strategies/queries/get_query_with_params.ts +++ b/x-pack/plugins/apm/server/lib/search_strategies/queries/get_query_with_params.ts @@ -9,7 +9,7 @@ import type { estypes } from '@elastic/elasticsearch'; import type { FieldValuePair, SearchStrategyParams, -} from '../../../../common/search_strategies/types'; +} from '../../../../common/correlations/types'; import { getCorrelationsFilters } from './get_filters'; export const getTermsQuery = ({ fieldName, fieldValue }: FieldValuePair) => { diff --git a/x-pack/plugins/apm/server/lib/search_strategies/queries/get_request_base.ts b/x-pack/plugins/apm/server/lib/search_strategies/queries/get_request_base.ts index fb1639b5d5f4a..1323a2c088404 100644 --- a/x-pack/plugins/apm/server/lib/search_strategies/queries/get_request_base.ts +++ b/x-pack/plugins/apm/server/lib/search_strategies/queries/get_request_base.ts @@ -5,7 +5,7 @@ * 2.0. */ -import type { SearchStrategyParams } from '../../../../common/search_strategies/types'; +import type { SearchStrategyParams } from '../../../../common/correlations/types'; export const getRequestBase = ({ index, diff --git a/x-pack/plugins/apm/server/lib/search_strategies/queries/query_correlation.ts b/x-pack/plugins/apm/server/lib/search_strategies/queries/query_correlation.ts index 24db25f8afd89..9ada1b47e574d 100644 --- a/x-pack/plugins/apm/server/lib/search_strategies/queries/query_correlation.ts +++ b/x-pack/plugins/apm/server/lib/search_strategies/queries/query_correlation.ts @@ -14,7 +14,7 @@ import type { FieldValuePair, ResponseHit, SearchStrategyParams, -} from '../../../../common/search_strategies/types'; +} from '../../../../common/correlations/types'; import { getQueryWithParams } from './get_query_with_params'; import { getRequestBase } from './get_request_base'; diff --git a/x-pack/plugins/apm/server/lib/search_strategies/queries/query_failure_correlation.ts b/x-pack/plugins/apm/server/lib/search_strategies/queries/query_failure_correlation.ts index 57339d73227d0..f763b1a9abdb0 100644 --- a/x-pack/plugins/apm/server/lib/search_strategies/queries/query_failure_correlation.ts +++ b/x-pack/plugins/apm/server/lib/search_strategies/queries/query_failure_correlation.ts @@ -6,8 +6,8 @@ */ import { estypes } from '@elastic/elasticsearch'; import { ElasticsearchClient } from 'kibana/server'; -import { SearchStrategyParams } from '../../../../common/search_strategies/types'; -import { FailedTransactionsCorrelation } from '../../../../common/search_strategies/failed_transactions_correlations/types'; +import { SearchStrategyParams } from '../../../../common/correlations/types'; +import { FailedTransactionsCorrelation } from '../../../../common/correlations/failed_transactions_correlations/types'; import { EVENT_OUTCOME } from '../../../../common/elasticsearch_fieldnames'; import { EventOutcome } from '../../../../common/event_outcome'; import { fetchTransactionDurationRanges } from './query_ranges'; diff --git a/x-pack/plugins/apm/server/lib/search_strategies/queries/query_field_candidates.ts b/x-pack/plugins/apm/server/lib/search_strategies/queries/query_field_candidates.ts index 390243295c4f0..676a2f82a9185 100644 --- a/x-pack/plugins/apm/server/lib/search_strategies/queries/query_field_candidates.ts +++ b/x-pack/plugins/apm/server/lib/search_strategies/queries/query_field_candidates.ts @@ -9,7 +9,7 @@ import type { estypes } from '@elastic/elasticsearch'; import type { ElasticsearchClient } from 'src/core/server'; -import type { SearchStrategyParams } from '../../../../common/search_strategies/types'; +import type { SearchStrategyParams } from '../../../../common/correlations/types'; import { FIELD_PREFIX_TO_EXCLUDE_AS_CANDIDATE, diff --git a/x-pack/plugins/apm/server/lib/search_strategies/queries/query_field_value_pairs.ts b/x-pack/plugins/apm/server/lib/search_strategies/queries/query_field_value_pairs.ts index 1219089c1594d..9fd73b22fd39c 100644 --- a/x-pack/plugins/apm/server/lib/search_strategies/queries/query_field_value_pairs.ts +++ b/x-pack/plugins/apm/server/lib/search_strategies/queries/query_field_value_pairs.ts @@ -12,7 +12,7 @@ import type { estypes } from '@elastic/elasticsearch'; import type { FieldValuePair, SearchStrategyParams, -} from '../../../../common/search_strategies/types'; +} from '../../../../common/correlations/types'; import { TERMS_SIZE } from '../constants'; diff --git a/x-pack/plugins/apm/server/lib/search_strategies/queries/query_fractions.ts b/x-pack/plugins/apm/server/lib/search_strategies/queries/query_fractions.ts index ccea480052c9b..0f59c5e18ee0f 100644 --- a/x-pack/plugins/apm/server/lib/search_strategies/queries/query_fractions.ts +++ b/x-pack/plugins/apm/server/lib/search_strategies/queries/query_fractions.ts @@ -8,7 +8,7 @@ import { ElasticsearchClient } from 'kibana/server'; import { estypes } from '@elastic/elasticsearch'; -import { SearchStrategyParams } from '../../../../common/search_strategies/types'; +import { SearchStrategyParams } from '../../../../common/correlations/types'; import { TRANSACTION_DURATION } from '../../../../common/elasticsearch_fieldnames'; import { getQueryWithParams } from './get_query_with_params'; diff --git a/x-pack/plugins/apm/server/lib/search_strategies/queries/query_histogram.ts b/x-pack/plugins/apm/server/lib/search_strategies/queries/query_histogram.ts index 5fb7ef76fc728..f4c4e05a248a1 100644 --- a/x-pack/plugins/apm/server/lib/search_strategies/queries/query_histogram.ts +++ b/x-pack/plugins/apm/server/lib/search_strategies/queries/query_histogram.ts @@ -15,7 +15,7 @@ import type { HistogramItem, ResponseHit, SearchStrategyParams, -} from '../../../../common/search_strategies/types'; +} from '../../../../common/correlations/types'; import { getQueryWithParams } from './get_query_with_params'; import { getRequestBase } from './get_request_base'; diff --git a/x-pack/plugins/apm/server/lib/search_strategies/queries/query_histogram_range_steps.ts b/x-pack/plugins/apm/server/lib/search_strategies/queries/query_histogram_range_steps.ts index 439bb9e4b9cd6..a2ce649b9a99a 100644 --- a/x-pack/plugins/apm/server/lib/search_strategies/queries/query_histogram_range_steps.ts +++ b/x-pack/plugins/apm/server/lib/search_strategies/queries/query_histogram_range_steps.ts @@ -12,7 +12,7 @@ import type { estypes } from '@elastic/elasticsearch'; import type { ElasticsearchClient } from 'src/core/server'; import { TRANSACTION_DURATION } from '../../../../common/elasticsearch_fieldnames'; -import type { SearchStrategyParams } from '../../../../common/search_strategies/types'; +import type { SearchStrategyParams } from '../../../../common/correlations/types'; import { getQueryWithParams } from './get_query_with_params'; import { getRequestBase } from './get_request_base'; diff --git a/x-pack/plugins/apm/server/lib/search_strategies/queries/query_histograms_generator.ts b/x-pack/plugins/apm/server/lib/search_strategies/queries/query_histograms_generator.ts index d11d143724a1c..6afeb79cea9e6 100644 --- a/x-pack/plugins/apm/server/lib/search_strategies/queries/query_histograms_generator.ts +++ b/x-pack/plugins/apm/server/lib/search_strategies/queries/query_histograms_generator.ts @@ -12,7 +12,7 @@ import type { ElasticsearchClient } from 'src/core/server'; import type { FieldValuePair, SearchStrategyParams, -} from '../../../../common/search_strategies/types'; +} from '../../../../common/correlations/types'; import { CORRELATION_THRESHOLD, KS_TEST_THRESHOLD } from '../constants'; diff --git a/x-pack/plugins/apm/server/lib/search_strategies/queries/query_p_values.ts b/x-pack/plugins/apm/server/lib/search_strategies/queries/query_p_values.ts index 8897edc43daef..4cfd3fec4482b 100644 --- a/x-pack/plugins/apm/server/lib/search_strategies/queries/query_p_values.ts +++ b/x-pack/plugins/apm/server/lib/search_strategies/queries/query_p_values.ts @@ -7,8 +7,8 @@ import type { ElasticsearchClient } from 'src/core/server'; -import type { SearchStrategyParams } from '../../../../common/search_strategies/types'; -import type { FailedTransactionsCorrelation } from '../../../../common/search_strategies/failed_transactions_correlations/types'; +import type { SearchStrategyParams } from '../../../../common/correlations/types'; +import type { FailedTransactionsCorrelation } from '../../../../common/correlations/failed_transactions_correlations/types'; import { ERROR_CORRELATION_THRESHOLD } from '../constants'; diff --git a/x-pack/plugins/apm/server/lib/search_strategies/queries/query_percentiles.ts b/x-pack/plugins/apm/server/lib/search_strategies/queries/query_percentiles.ts index 70b5b70ce8912..da546315e812d 100644 --- a/x-pack/plugins/apm/server/lib/search_strategies/queries/query_percentiles.ts +++ b/x-pack/plugins/apm/server/lib/search_strategies/queries/query_percentiles.ts @@ -14,7 +14,7 @@ import type { FieldValuePair, ResponseHit, SearchStrategyParams, -} from '../../../../common/search_strategies/types'; +} from '../../../../common/correlations/types'; import { getQueryWithParams } from './get_query_with_params'; import { getRequestBase } from './get_request_base'; diff --git a/x-pack/plugins/apm/server/lib/search_strategies/queries/query_ranges.ts b/x-pack/plugins/apm/server/lib/search_strategies/queries/query_ranges.ts index a530c997876c4..13ba83cdb8c36 100644 --- a/x-pack/plugins/apm/server/lib/search_strategies/queries/query_ranges.ts +++ b/x-pack/plugins/apm/server/lib/search_strategies/queries/query_ranges.ts @@ -14,7 +14,7 @@ import type { FieldValuePair, ResponseHit, SearchStrategyParams, -} from '../../../../common/search_strategies/types'; +} from '../../../../common/correlations/types'; import { getQueryWithParams } from './get_query_with_params'; import { getRequestBase } from './get_request_base'; diff --git a/x-pack/plugins/apm/server/lib/search_strategies/queries/query_significant_correlations.ts b/x-pack/plugins/apm/server/lib/search_strategies/queries/query_significant_correlations.ts index b88374adc2dcd..df44e0c7c9fbd 100644 --- a/x-pack/plugins/apm/server/lib/search_strategies/queries/query_significant_correlations.ts +++ b/x-pack/plugins/apm/server/lib/search_strategies/queries/query_significant_correlations.ts @@ -12,8 +12,8 @@ import type { ElasticsearchClient } from 'src/core/server'; import type { FieldValuePair, SearchStrategyParams, -} from '../../../../common/search_strategies/types'; -import type { LatencyCorrelation } from '../../../../common/search_strategies/latency_correlations/types'; +} from '../../../../common/correlations/types'; +import type { LatencyCorrelation } from '../../../../common/correlations/latency_correlations/types'; import { fetchTransactionDurationFractions, From 83f5ae0cb31e24dced0125721e2ee7743841e45b Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Sun, 17 Oct 2021 13:46:26 +0200 Subject: [PATCH 07/39] [ML] Rename server/lib/search_strategies to server/lib/correlations. --- .../lib/{search_strategies => correlations}/constants.ts | 0 .../queries/field_stats/get_boolean_field_stats.ts | 0 .../queries/field_stats/get_field_stats.test.ts | 0 .../queries/field_stats/get_fields_stats.ts | 6 +++--- .../queries/field_stats/get_keyword_field_stats.ts | 4 ++-- .../queries/field_stats/get_numeric_field_stats.ts | 0 .../queries/get_filters.ts | 0 .../queries/get_prioritized_field_value_pairs.test.ts | 0 .../queries/get_prioritized_field_value_pairs.ts | 0 .../queries/get_query_with_params.test.ts | 0 .../queries/get_query_with_params.ts | 0 .../queries/get_request_base.test.ts | 0 .../queries/get_request_base.ts | 0 .../{search_strategies => correlations}/queries/index.ts | 0 .../queries/query_correlation.test.ts | 0 .../queries/query_correlation.ts | 0 .../queries/query_failure_correlation.ts | 0 .../queries/query_field_candidates.test.ts | 0 .../queries/query_field_candidates.ts | 0 .../queries/query_field_value_pairs.test.ts | 0 .../queries/query_field_value_pairs.ts | 0 .../queries/query_fractions.test.ts | 0 .../queries/query_fractions.ts | 0 .../queries/query_histogram.test.ts | 0 .../queries/query_histogram.ts | 0 .../queries/query_histogram_range_steps.test.ts | 0 .../queries/query_histogram_range_steps.ts | 0 .../queries/query_histograms_generator.test.ts | 0 .../queries/query_histograms_generator.ts | 0 .../queries/query_p_values.ts | 0 .../queries/query_percentiles.test.ts | 0 .../queries/query_percentiles.ts | 0 .../queries/query_ranges.test.ts | 0 .../queries/query_ranges.ts | 0 .../queries/query_significant_correlations.ts | 0 .../utils/compute_expectations_and_ranges.test.ts | 0 .../utils/compute_expectations_and_ranges.ts | 0 .../utils/field_stats_utils.ts | 0 .../utils/has_prefix_to_include.test.ts | 0 .../utils/has_prefix_to_include.ts | 0 .../lib/{search_strategies => correlations}/utils/index.ts | 0 .../server/lib/latency/get_overall_latency_distribution.ts | 4 ++-- .../server/lib/latency/get_percentile_threshold_value.ts | 2 +- x-pack/plugins/apm/server/routes/correlations.ts | 4 ++-- 44 files changed, 10 insertions(+), 10 deletions(-) rename x-pack/plugins/apm/server/lib/{search_strategies => correlations}/constants.ts (100%) rename x-pack/plugins/apm/server/lib/{search_strategies => correlations}/queries/field_stats/get_boolean_field_stats.ts (100%) rename x-pack/plugins/apm/server/lib/{search_strategies => correlations}/queries/field_stats/get_field_stats.test.ts (100%) rename x-pack/plugins/apm/server/lib/{search_strategies => correlations}/queries/field_stats/get_fields_stats.ts (100%) rename x-pack/plugins/apm/server/lib/{search_strategies => correlations}/queries/field_stats/get_keyword_field_stats.ts (100%) rename x-pack/plugins/apm/server/lib/{search_strategies => correlations}/queries/field_stats/get_numeric_field_stats.ts (100%) rename x-pack/plugins/apm/server/lib/{search_strategies => correlations}/queries/get_filters.ts (100%) rename x-pack/plugins/apm/server/lib/{search_strategies => correlations}/queries/get_prioritized_field_value_pairs.test.ts (100%) rename x-pack/plugins/apm/server/lib/{search_strategies => correlations}/queries/get_prioritized_field_value_pairs.ts (100%) rename x-pack/plugins/apm/server/lib/{search_strategies => correlations}/queries/get_query_with_params.test.ts (100%) rename x-pack/plugins/apm/server/lib/{search_strategies => correlations}/queries/get_query_with_params.ts (100%) rename x-pack/plugins/apm/server/lib/{search_strategies => correlations}/queries/get_request_base.test.ts (100%) rename x-pack/plugins/apm/server/lib/{search_strategies => correlations}/queries/get_request_base.ts (100%) rename x-pack/plugins/apm/server/lib/{search_strategies => correlations}/queries/index.ts (100%) rename x-pack/plugins/apm/server/lib/{search_strategies => correlations}/queries/query_correlation.test.ts (100%) rename x-pack/plugins/apm/server/lib/{search_strategies => correlations}/queries/query_correlation.ts (100%) rename x-pack/plugins/apm/server/lib/{search_strategies => correlations}/queries/query_failure_correlation.ts (100%) rename x-pack/plugins/apm/server/lib/{search_strategies => correlations}/queries/query_field_candidates.test.ts (100%) rename x-pack/plugins/apm/server/lib/{search_strategies => correlations}/queries/query_field_candidates.ts (100%) rename x-pack/plugins/apm/server/lib/{search_strategies => correlations}/queries/query_field_value_pairs.test.ts (100%) rename x-pack/plugins/apm/server/lib/{search_strategies => correlations}/queries/query_field_value_pairs.ts (100%) rename x-pack/plugins/apm/server/lib/{search_strategies => correlations}/queries/query_fractions.test.ts (100%) rename x-pack/plugins/apm/server/lib/{search_strategies => correlations}/queries/query_fractions.ts (100%) rename x-pack/plugins/apm/server/lib/{search_strategies => correlations}/queries/query_histogram.test.ts (100%) rename x-pack/plugins/apm/server/lib/{search_strategies => correlations}/queries/query_histogram.ts (100%) rename x-pack/plugins/apm/server/lib/{search_strategies => correlations}/queries/query_histogram_range_steps.test.ts (100%) rename x-pack/plugins/apm/server/lib/{search_strategies => correlations}/queries/query_histogram_range_steps.ts (100%) rename x-pack/plugins/apm/server/lib/{search_strategies => correlations}/queries/query_histograms_generator.test.ts (100%) rename x-pack/plugins/apm/server/lib/{search_strategies => correlations}/queries/query_histograms_generator.ts (100%) rename x-pack/plugins/apm/server/lib/{search_strategies => correlations}/queries/query_p_values.ts (100%) rename x-pack/plugins/apm/server/lib/{search_strategies => correlations}/queries/query_percentiles.test.ts (100%) rename x-pack/plugins/apm/server/lib/{search_strategies => correlations}/queries/query_percentiles.ts (100%) rename x-pack/plugins/apm/server/lib/{search_strategies => correlations}/queries/query_ranges.test.ts (100%) rename x-pack/plugins/apm/server/lib/{search_strategies => correlations}/queries/query_ranges.ts (100%) rename x-pack/plugins/apm/server/lib/{search_strategies => correlations}/queries/query_significant_correlations.ts (100%) rename x-pack/plugins/apm/server/lib/{search_strategies => correlations}/utils/compute_expectations_and_ranges.test.ts (100%) rename x-pack/plugins/apm/server/lib/{search_strategies => correlations}/utils/compute_expectations_and_ranges.ts (100%) rename x-pack/plugins/apm/server/lib/{search_strategies => correlations}/utils/field_stats_utils.ts (100%) rename x-pack/plugins/apm/server/lib/{search_strategies => correlations}/utils/has_prefix_to_include.test.ts (100%) rename x-pack/plugins/apm/server/lib/{search_strategies => correlations}/utils/has_prefix_to_include.ts (100%) rename x-pack/plugins/apm/server/lib/{search_strategies => correlations}/utils/index.ts (100%) diff --git a/x-pack/plugins/apm/server/lib/search_strategies/constants.ts b/x-pack/plugins/apm/server/lib/correlations/constants.ts similarity index 100% rename from x-pack/plugins/apm/server/lib/search_strategies/constants.ts rename to x-pack/plugins/apm/server/lib/correlations/constants.ts diff --git a/x-pack/plugins/apm/server/lib/search_strategies/queries/field_stats/get_boolean_field_stats.ts b/x-pack/plugins/apm/server/lib/correlations/queries/field_stats/get_boolean_field_stats.ts similarity index 100% rename from x-pack/plugins/apm/server/lib/search_strategies/queries/field_stats/get_boolean_field_stats.ts rename to x-pack/plugins/apm/server/lib/correlations/queries/field_stats/get_boolean_field_stats.ts diff --git a/x-pack/plugins/apm/server/lib/search_strategies/queries/field_stats/get_field_stats.test.ts b/x-pack/plugins/apm/server/lib/correlations/queries/field_stats/get_field_stats.test.ts similarity index 100% rename from x-pack/plugins/apm/server/lib/search_strategies/queries/field_stats/get_field_stats.test.ts rename to x-pack/plugins/apm/server/lib/correlations/queries/field_stats/get_field_stats.test.ts diff --git a/x-pack/plugins/apm/server/lib/search_strategies/queries/field_stats/get_fields_stats.ts b/x-pack/plugins/apm/server/lib/correlations/queries/field_stats/get_fields_stats.ts similarity index 100% rename from x-pack/plugins/apm/server/lib/search_strategies/queries/field_stats/get_fields_stats.ts rename to x-pack/plugins/apm/server/lib/correlations/queries/field_stats/get_fields_stats.ts index 7b5e53e0dfee7..a294c4588d0cb 100644 --- a/x-pack/plugins/apm/server/lib/search_strategies/queries/field_stats/get_fields_stats.ts +++ b/x-pack/plugins/apm/server/lib/correlations/queries/field_stats/get_fields_stats.ts @@ -12,13 +12,13 @@ import { FieldValuePair, SearchStrategyParams, } from '../../../../../common/correlations/types'; -import { getRequestBase } from '../get_request_base'; -import { fetchKeywordFieldStats } from './get_keyword_field_stats'; -import { fetchNumericFieldStats } from './get_numeric_field_stats'; import { FieldStats, FieldStatsCommonRequestParams, } from '../../../../../common/correlations/field_stats_types'; +import { getRequestBase } from '../get_request_base'; +import { fetchKeywordFieldStats } from './get_keyword_field_stats'; +import { fetchNumericFieldStats } from './get_numeric_field_stats'; import { fetchBooleanFieldStats } from './get_boolean_field_stats'; export const fetchFieldsStats = async ( diff --git a/x-pack/plugins/apm/server/lib/search_strategies/queries/field_stats/get_keyword_field_stats.ts b/x-pack/plugins/apm/server/lib/correlations/queries/field_stats/get_keyword_field_stats.ts similarity index 100% rename from x-pack/plugins/apm/server/lib/search_strategies/queries/field_stats/get_keyword_field_stats.ts rename to x-pack/plugins/apm/server/lib/correlations/queries/field_stats/get_keyword_field_stats.ts index 0b2bd797b83d8..a676ae67652cc 100644 --- a/x-pack/plugins/apm/server/lib/search_strategies/queries/field_stats/get_keyword_field_stats.ts +++ b/x-pack/plugins/apm/server/lib/correlations/queries/field_stats/get_keyword_field_stats.ts @@ -9,14 +9,14 @@ import { ElasticsearchClient } from 'kibana/server'; import { SearchRequest } from '@elastic/elasticsearch/api/types'; import { estypes } from '@elastic/elasticsearch'; import { FieldValuePair } from '../../../../../common/correlations/types'; -import { getQueryWithParams } from '../get_query_with_params'; -import { buildSamplerAggregation } from '../../utils/field_stats_utils'; import { FieldStatsCommonRequestParams, KeywordFieldStats, Aggs, TopValueBucket, } from '../../../../../common/correlations/field_stats_types'; +import { buildSamplerAggregation } from '../../utils/field_stats_utils'; +import { getQueryWithParams } from '../get_query_with_params'; export const getKeywordFieldStatsRequest = ( params: FieldStatsCommonRequestParams, diff --git a/x-pack/plugins/apm/server/lib/search_strategies/queries/field_stats/get_numeric_field_stats.ts b/x-pack/plugins/apm/server/lib/correlations/queries/field_stats/get_numeric_field_stats.ts similarity index 100% rename from x-pack/plugins/apm/server/lib/search_strategies/queries/field_stats/get_numeric_field_stats.ts rename to x-pack/plugins/apm/server/lib/correlations/queries/field_stats/get_numeric_field_stats.ts diff --git a/x-pack/plugins/apm/server/lib/search_strategies/queries/get_filters.ts b/x-pack/plugins/apm/server/lib/correlations/queries/get_filters.ts similarity index 100% rename from x-pack/plugins/apm/server/lib/search_strategies/queries/get_filters.ts rename to x-pack/plugins/apm/server/lib/correlations/queries/get_filters.ts diff --git a/x-pack/plugins/apm/server/lib/search_strategies/queries/get_prioritized_field_value_pairs.test.ts b/x-pack/plugins/apm/server/lib/correlations/queries/get_prioritized_field_value_pairs.test.ts similarity index 100% rename from x-pack/plugins/apm/server/lib/search_strategies/queries/get_prioritized_field_value_pairs.test.ts rename to x-pack/plugins/apm/server/lib/correlations/queries/get_prioritized_field_value_pairs.test.ts diff --git a/x-pack/plugins/apm/server/lib/search_strategies/queries/get_prioritized_field_value_pairs.ts b/x-pack/plugins/apm/server/lib/correlations/queries/get_prioritized_field_value_pairs.ts similarity index 100% rename from x-pack/plugins/apm/server/lib/search_strategies/queries/get_prioritized_field_value_pairs.ts rename to x-pack/plugins/apm/server/lib/correlations/queries/get_prioritized_field_value_pairs.ts diff --git a/x-pack/plugins/apm/server/lib/search_strategies/queries/get_query_with_params.test.ts b/x-pack/plugins/apm/server/lib/correlations/queries/get_query_with_params.test.ts similarity index 100% rename from x-pack/plugins/apm/server/lib/search_strategies/queries/get_query_with_params.test.ts rename to x-pack/plugins/apm/server/lib/correlations/queries/get_query_with_params.test.ts diff --git a/x-pack/plugins/apm/server/lib/search_strategies/queries/get_query_with_params.ts b/x-pack/plugins/apm/server/lib/correlations/queries/get_query_with_params.ts similarity index 100% rename from x-pack/plugins/apm/server/lib/search_strategies/queries/get_query_with_params.ts rename to x-pack/plugins/apm/server/lib/correlations/queries/get_query_with_params.ts diff --git a/x-pack/plugins/apm/server/lib/search_strategies/queries/get_request_base.test.ts b/x-pack/plugins/apm/server/lib/correlations/queries/get_request_base.test.ts similarity index 100% rename from x-pack/plugins/apm/server/lib/search_strategies/queries/get_request_base.test.ts rename to x-pack/plugins/apm/server/lib/correlations/queries/get_request_base.test.ts diff --git a/x-pack/plugins/apm/server/lib/search_strategies/queries/get_request_base.ts b/x-pack/plugins/apm/server/lib/correlations/queries/get_request_base.ts similarity index 100% rename from x-pack/plugins/apm/server/lib/search_strategies/queries/get_request_base.ts rename to x-pack/plugins/apm/server/lib/correlations/queries/get_request_base.ts diff --git a/x-pack/plugins/apm/server/lib/search_strategies/queries/index.ts b/x-pack/plugins/apm/server/lib/correlations/queries/index.ts similarity index 100% rename from x-pack/plugins/apm/server/lib/search_strategies/queries/index.ts rename to x-pack/plugins/apm/server/lib/correlations/queries/index.ts diff --git a/x-pack/plugins/apm/server/lib/search_strategies/queries/query_correlation.test.ts b/x-pack/plugins/apm/server/lib/correlations/queries/query_correlation.test.ts similarity index 100% rename from x-pack/plugins/apm/server/lib/search_strategies/queries/query_correlation.test.ts rename to x-pack/plugins/apm/server/lib/correlations/queries/query_correlation.test.ts diff --git a/x-pack/plugins/apm/server/lib/search_strategies/queries/query_correlation.ts b/x-pack/plugins/apm/server/lib/correlations/queries/query_correlation.ts similarity index 100% rename from x-pack/plugins/apm/server/lib/search_strategies/queries/query_correlation.ts rename to x-pack/plugins/apm/server/lib/correlations/queries/query_correlation.ts diff --git a/x-pack/plugins/apm/server/lib/search_strategies/queries/query_failure_correlation.ts b/x-pack/plugins/apm/server/lib/correlations/queries/query_failure_correlation.ts similarity index 100% rename from x-pack/plugins/apm/server/lib/search_strategies/queries/query_failure_correlation.ts rename to x-pack/plugins/apm/server/lib/correlations/queries/query_failure_correlation.ts diff --git a/x-pack/plugins/apm/server/lib/search_strategies/queries/query_field_candidates.test.ts b/x-pack/plugins/apm/server/lib/correlations/queries/query_field_candidates.test.ts similarity index 100% rename from x-pack/plugins/apm/server/lib/search_strategies/queries/query_field_candidates.test.ts rename to x-pack/plugins/apm/server/lib/correlations/queries/query_field_candidates.test.ts diff --git a/x-pack/plugins/apm/server/lib/search_strategies/queries/query_field_candidates.ts b/x-pack/plugins/apm/server/lib/correlations/queries/query_field_candidates.ts similarity index 100% rename from x-pack/plugins/apm/server/lib/search_strategies/queries/query_field_candidates.ts rename to x-pack/plugins/apm/server/lib/correlations/queries/query_field_candidates.ts diff --git a/x-pack/plugins/apm/server/lib/search_strategies/queries/query_field_value_pairs.test.ts b/x-pack/plugins/apm/server/lib/correlations/queries/query_field_value_pairs.test.ts similarity index 100% rename from x-pack/plugins/apm/server/lib/search_strategies/queries/query_field_value_pairs.test.ts rename to x-pack/plugins/apm/server/lib/correlations/queries/query_field_value_pairs.test.ts diff --git a/x-pack/plugins/apm/server/lib/search_strategies/queries/query_field_value_pairs.ts b/x-pack/plugins/apm/server/lib/correlations/queries/query_field_value_pairs.ts similarity index 100% rename from x-pack/plugins/apm/server/lib/search_strategies/queries/query_field_value_pairs.ts rename to x-pack/plugins/apm/server/lib/correlations/queries/query_field_value_pairs.ts diff --git a/x-pack/plugins/apm/server/lib/search_strategies/queries/query_fractions.test.ts b/x-pack/plugins/apm/server/lib/correlations/queries/query_fractions.test.ts similarity index 100% rename from x-pack/plugins/apm/server/lib/search_strategies/queries/query_fractions.test.ts rename to x-pack/plugins/apm/server/lib/correlations/queries/query_fractions.test.ts diff --git a/x-pack/plugins/apm/server/lib/search_strategies/queries/query_fractions.ts b/x-pack/plugins/apm/server/lib/correlations/queries/query_fractions.ts similarity index 100% rename from x-pack/plugins/apm/server/lib/search_strategies/queries/query_fractions.ts rename to x-pack/plugins/apm/server/lib/correlations/queries/query_fractions.ts diff --git a/x-pack/plugins/apm/server/lib/search_strategies/queries/query_histogram.test.ts b/x-pack/plugins/apm/server/lib/correlations/queries/query_histogram.test.ts similarity index 100% rename from x-pack/plugins/apm/server/lib/search_strategies/queries/query_histogram.test.ts rename to x-pack/plugins/apm/server/lib/correlations/queries/query_histogram.test.ts diff --git a/x-pack/plugins/apm/server/lib/search_strategies/queries/query_histogram.ts b/x-pack/plugins/apm/server/lib/correlations/queries/query_histogram.ts similarity index 100% rename from x-pack/plugins/apm/server/lib/search_strategies/queries/query_histogram.ts rename to x-pack/plugins/apm/server/lib/correlations/queries/query_histogram.ts diff --git a/x-pack/plugins/apm/server/lib/search_strategies/queries/query_histogram_range_steps.test.ts b/x-pack/plugins/apm/server/lib/correlations/queries/query_histogram_range_steps.test.ts similarity index 100% rename from x-pack/plugins/apm/server/lib/search_strategies/queries/query_histogram_range_steps.test.ts rename to x-pack/plugins/apm/server/lib/correlations/queries/query_histogram_range_steps.test.ts diff --git a/x-pack/plugins/apm/server/lib/search_strategies/queries/query_histogram_range_steps.ts b/x-pack/plugins/apm/server/lib/correlations/queries/query_histogram_range_steps.ts similarity index 100% rename from x-pack/plugins/apm/server/lib/search_strategies/queries/query_histogram_range_steps.ts rename to x-pack/plugins/apm/server/lib/correlations/queries/query_histogram_range_steps.ts diff --git a/x-pack/plugins/apm/server/lib/search_strategies/queries/query_histograms_generator.test.ts b/x-pack/plugins/apm/server/lib/correlations/queries/query_histograms_generator.test.ts similarity index 100% rename from x-pack/plugins/apm/server/lib/search_strategies/queries/query_histograms_generator.test.ts rename to x-pack/plugins/apm/server/lib/correlations/queries/query_histograms_generator.test.ts diff --git a/x-pack/plugins/apm/server/lib/search_strategies/queries/query_histograms_generator.ts b/x-pack/plugins/apm/server/lib/correlations/queries/query_histograms_generator.ts similarity index 100% rename from x-pack/plugins/apm/server/lib/search_strategies/queries/query_histograms_generator.ts rename to x-pack/plugins/apm/server/lib/correlations/queries/query_histograms_generator.ts diff --git a/x-pack/plugins/apm/server/lib/search_strategies/queries/query_p_values.ts b/x-pack/plugins/apm/server/lib/correlations/queries/query_p_values.ts similarity index 100% rename from x-pack/plugins/apm/server/lib/search_strategies/queries/query_p_values.ts rename to x-pack/plugins/apm/server/lib/correlations/queries/query_p_values.ts diff --git a/x-pack/plugins/apm/server/lib/search_strategies/queries/query_percentiles.test.ts b/x-pack/plugins/apm/server/lib/correlations/queries/query_percentiles.test.ts similarity index 100% rename from x-pack/plugins/apm/server/lib/search_strategies/queries/query_percentiles.test.ts rename to x-pack/plugins/apm/server/lib/correlations/queries/query_percentiles.test.ts diff --git a/x-pack/plugins/apm/server/lib/search_strategies/queries/query_percentiles.ts b/x-pack/plugins/apm/server/lib/correlations/queries/query_percentiles.ts similarity index 100% rename from x-pack/plugins/apm/server/lib/search_strategies/queries/query_percentiles.ts rename to x-pack/plugins/apm/server/lib/correlations/queries/query_percentiles.ts diff --git a/x-pack/plugins/apm/server/lib/search_strategies/queries/query_ranges.test.ts b/x-pack/plugins/apm/server/lib/correlations/queries/query_ranges.test.ts similarity index 100% rename from x-pack/plugins/apm/server/lib/search_strategies/queries/query_ranges.test.ts rename to x-pack/plugins/apm/server/lib/correlations/queries/query_ranges.test.ts diff --git a/x-pack/plugins/apm/server/lib/search_strategies/queries/query_ranges.ts b/x-pack/plugins/apm/server/lib/correlations/queries/query_ranges.ts similarity index 100% rename from x-pack/plugins/apm/server/lib/search_strategies/queries/query_ranges.ts rename to x-pack/plugins/apm/server/lib/correlations/queries/query_ranges.ts diff --git a/x-pack/plugins/apm/server/lib/search_strategies/queries/query_significant_correlations.ts b/x-pack/plugins/apm/server/lib/correlations/queries/query_significant_correlations.ts similarity index 100% rename from x-pack/plugins/apm/server/lib/search_strategies/queries/query_significant_correlations.ts rename to x-pack/plugins/apm/server/lib/correlations/queries/query_significant_correlations.ts diff --git a/x-pack/plugins/apm/server/lib/search_strategies/utils/compute_expectations_and_ranges.test.ts b/x-pack/plugins/apm/server/lib/correlations/utils/compute_expectations_and_ranges.test.ts similarity index 100% rename from x-pack/plugins/apm/server/lib/search_strategies/utils/compute_expectations_and_ranges.test.ts rename to x-pack/plugins/apm/server/lib/correlations/utils/compute_expectations_and_ranges.test.ts diff --git a/x-pack/plugins/apm/server/lib/search_strategies/utils/compute_expectations_and_ranges.ts b/x-pack/plugins/apm/server/lib/correlations/utils/compute_expectations_and_ranges.ts similarity index 100% rename from x-pack/plugins/apm/server/lib/search_strategies/utils/compute_expectations_and_ranges.ts rename to x-pack/plugins/apm/server/lib/correlations/utils/compute_expectations_and_ranges.ts diff --git a/x-pack/plugins/apm/server/lib/search_strategies/utils/field_stats_utils.ts b/x-pack/plugins/apm/server/lib/correlations/utils/field_stats_utils.ts similarity index 100% rename from x-pack/plugins/apm/server/lib/search_strategies/utils/field_stats_utils.ts rename to x-pack/plugins/apm/server/lib/correlations/utils/field_stats_utils.ts diff --git a/x-pack/plugins/apm/server/lib/search_strategies/utils/has_prefix_to_include.test.ts b/x-pack/plugins/apm/server/lib/correlations/utils/has_prefix_to_include.test.ts similarity index 100% rename from x-pack/plugins/apm/server/lib/search_strategies/utils/has_prefix_to_include.test.ts rename to x-pack/plugins/apm/server/lib/correlations/utils/has_prefix_to_include.test.ts diff --git a/x-pack/plugins/apm/server/lib/search_strategies/utils/has_prefix_to_include.ts b/x-pack/plugins/apm/server/lib/correlations/utils/has_prefix_to_include.ts similarity index 100% rename from x-pack/plugins/apm/server/lib/search_strategies/utils/has_prefix_to_include.ts rename to x-pack/plugins/apm/server/lib/correlations/utils/has_prefix_to_include.ts diff --git a/x-pack/plugins/apm/server/lib/search_strategies/utils/index.ts b/x-pack/plugins/apm/server/lib/correlations/utils/index.ts similarity index 100% rename from x-pack/plugins/apm/server/lib/search_strategies/utils/index.ts rename to x-pack/plugins/apm/server/lib/correlations/utils/index.ts diff --git a/x-pack/plugins/apm/server/lib/latency/get_overall_latency_distribution.ts b/x-pack/plugins/apm/server/lib/latency/get_overall_latency_distribution.ts index 787304c3f8dcd..aec946fb8c1df 100644 --- a/x-pack/plugins/apm/server/lib/latency/get_overall_latency_distribution.ts +++ b/x-pack/plugins/apm/server/lib/latency/get_overall_latency_distribution.ts @@ -14,8 +14,8 @@ import { withApmSpan } from '../../utils/with_apm_span'; import { getHistogramIntervalRequest, getHistogramRangeSteps, -} from '../search_strategies/queries/query_histogram_range_steps'; -import { getTransactionDurationRangesRequest } from '../search_strategies/queries/query_ranges'; +} from '../correlations/queries/query_histogram_range_steps'; +import { getTransactionDurationRangesRequest } from '../correlations/queries/query_ranges'; import { getPercentileThresholdValue } from './get_percentile_threshold_value'; import type { diff --git a/x-pack/plugins/apm/server/lib/latency/get_percentile_threshold_value.ts b/x-pack/plugins/apm/server/lib/latency/get_percentile_threshold_value.ts index 0d417a370e0b6..90129d966e411 100644 --- a/x-pack/plugins/apm/server/lib/latency/get_percentile_threshold_value.ts +++ b/x-pack/plugins/apm/server/lib/latency/get_percentile_threshold_value.ts @@ -9,7 +9,7 @@ import type { estypes } from '@elastic/elasticsearch'; import { ProcessorEvent } from '../../../common/processor_event'; -import { getTransactionDurationPercentilesRequest } from '../search_strategies/queries/query_percentiles'; +import { getTransactionDurationPercentilesRequest } from '../correlations/queries/query_percentiles'; import type { OverallLatencyDistributionOptions } from './types'; diff --git a/x-pack/plugins/apm/server/routes/correlations.ts b/x-pack/plugins/apm/server/routes/correlations.ts index a4d0e4979a5be..e64e802187e45 100644 --- a/x-pack/plugins/apm/server/routes/correlations.ts +++ b/x-pack/plugins/apm/server/routes/correlations.ts @@ -15,8 +15,8 @@ import { fetchSignificantCorrelations, fetchTransactionDurationFieldCandidates, fetchTransactionDurationFieldValuePairs, -} from '../lib/search_strategies/queries'; -import { fetchFieldsStats } from '../lib/search_strategies/queries/field_stats/get_fields_stats'; +} from '../lib/correlations/queries'; +import { fetchFieldsStats } from '../lib/correlations/queries/field_stats/get_fields_stats'; import { withApmSpan } from '../utils/with_apm_span'; From 8d92d949b3c825b7affb45595fbaf797e755bca6 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Sun, 17 Oct 2021 16:49:52 +0200 Subject: [PATCH 08/39] [ML] Fix API integration tests. --- .../use_failed_transactions_correlations.ts | 10 +- .../correlations/use_latency_correlations.ts | 9 +- .../correlations/queries/query_fractions.ts | 8 + .../queries/query_significant_correlations.ts | 2 +- .../utils/compute_expectations_and_ranges.ts | 20 +- .../tests/correlations/failed_transactions.ts | 335 ++++++++--------- .../tests/correlations/field_candidates.ts | 55 +++ .../tests/correlations/field_value_pairs.ts | 71 ++++ .../tests/correlations/latency.ts | 353 ++++++++---------- .../tests/correlations/p_values.ts | 71 ++++ .../correlations/significant_correlations.ts | 95 +++++ .../test/apm_api_integration/tests/index.ts | 10 +- 12 files changed, 639 insertions(+), 400 deletions(-) create mode 100644 x-pack/test/apm_api_integration/tests/correlations/field_candidates.ts create mode 100644 x-pack/test/apm_api_integration/tests/correlations/field_value_pairs.ts create mode 100644 x-pack/test/apm_api_integration/tests/correlations/p_values.ts create mode 100644 x-pack/test/apm_api_integration/tests/correlations/significant_correlations.ts diff --git a/x-pack/plugins/apm/public/components/app/correlations/use_failed_transactions_correlations.ts b/x-pack/plugins/apm/public/components/app/correlations/use_failed_transactions_correlations.ts index ee13d5967d720..26bbc92d50e8f 100644 --- a/x-pack/plugins/apm/public/components/app/correlations/use_failed_transactions_correlations.ts +++ b/x-pack/plugins/apm/public/components/app/correlations/use_failed_transactions_correlations.ts @@ -35,6 +35,12 @@ interface SearchStrategyProgress { total: number; } +function getFailedTransactionsCorrelationsSortedByScore( + failedTransactionsCorrelations: FailedTransactionsCorrelation[] +) { + return failedTransactionsCorrelations.sort((a, b) => b.score - a.score); +} + const getInitialRawResponse = (): Response => ({ ccsWarning: false, @@ -178,7 +184,9 @@ export function useFailedTransactionsCorrelations() { ...pValues.failedTransactionsCorrelations ); rawResponseUpdate.failedTransactionsCorrelations = - failedTransactionsCorrelations; + getFailedTransactionsCorrelationsSortedByScore( + failedTransactionsCorrelations + ); setRawResponse(rawResponseUpdate); } diff --git a/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts b/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts index cd7f5e6abd3dc..6768156279d23 100644 --- a/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts +++ b/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts @@ -33,6 +33,12 @@ interface SearchStrategyProgress { total: number; } +function getLatencyCorrelationsSortedByCorrelation( + latencyCorrelations: LatencyCorrelation[] +) { + return latencyCorrelations.sort((a, b) => b.correlation - a.correlation); +} + const getInitialRawResponse = (): Response => ({ ccsWarning: false, @@ -174,7 +180,8 @@ export function useLatencyCorrelations() { latencyCorrelations.push( ...significantCorrelations.latencyCorrelations ); - rawResponseUpdate.latencyCorrelations = latencyCorrelations; + rawResponseUpdate.latencyCorrelations = + getLatencyCorrelationsSortedByCorrelation(latencyCorrelations); setRawResponse(rawResponseUpdate); } diff --git a/x-pack/plugins/apm/server/lib/correlations/queries/query_fractions.ts b/x-pack/plugins/apm/server/lib/correlations/queries/query_fractions.ts index 0f59c5e18ee0f..7e47b6edfb018 100644 --- a/x-pack/plugins/apm/server/lib/correlations/queries/query_fractions.ts +++ b/x-pack/plugins/apm/server/lib/correlations/queries/query_fractions.ts @@ -44,6 +44,14 @@ export const fetchTransactionDurationFractions = async ( const resp = await esClient.search( getTransactionDurationRangesRequest(params, ranges) ); + + if ((resp.body.hits.total as estypes.SearchTotalHits).value === 0) { + return { + fractions: [], + totalDocCount: 0, + }; + } + if (resp.body.aggregations === undefined) { throw new Error( 'fetchTransactionDurationFractions failed, did not return aggregations.' diff --git a/x-pack/plugins/apm/server/lib/correlations/queries/query_significant_correlations.ts b/x-pack/plugins/apm/server/lib/correlations/queries/query_significant_correlations.ts index df44e0c7c9fbd..42ae309dd3463 100644 --- a/x-pack/plugins/apm/server/lib/correlations/queries/query_significant_correlations.ts +++ b/x-pack/plugins/apm/server/lib/correlations/queries/query_significant_correlations.ts @@ -69,5 +69,5 @@ export const fetchSignificantCorrelations = async ( } // TODO Fix CCS warning - return { latencyCorrelations, ccsWarning: false }; + return { latencyCorrelations, ccsWarning: false, totalDocCount }; }; diff --git a/x-pack/plugins/apm/server/lib/correlations/utils/compute_expectations_and_ranges.ts b/x-pack/plugins/apm/server/lib/correlations/utils/compute_expectations_and_ranges.ts index 9ab9689fd6d30..9683af1fa341f 100644 --- a/x-pack/plugins/apm/server/lib/correlations/utils/compute_expectations_and_ranges.ts +++ b/x-pack/plugins/apm/server/lib/correlations/utils/compute_expectations_and_ranges.ts @@ -29,15 +29,17 @@ export const computeExpectationsAndRanges = ( } tempFractions.push(PERCENTILES_STEP / 100); - const ranges = tempPercentiles.reduce((p, to) => { - const from = p[p.length - 1]?.to; - if (from !== undefined) { - p.push({ from, to }); - } else { - p.push({ to }); - } - return p; - }, [] as Array<{ from?: number; to?: number }>); + const ranges = tempPercentiles + .map((tP) => Math.round(tP)) + .reduce((p, to) => { + const from = p[p.length - 1]?.to; + if (from !== undefined) { + p.push({ from, to }); + } else { + p.push({ to }); + } + return p; + }, [] as Array<{ from?: number; to?: number }>); if (ranges.length > 0) { ranges.push({ from: ranges[ranges.length - 1].to }); } diff --git a/x-pack/test/apm_api_integration/tests/correlations/failed_transactions.ts b/x-pack/test/apm_api_integration/tests/correlations/failed_transactions.ts index 3388d5b4aa379..f46d5ddfb38a5 100644 --- a/x-pack/test/apm_api_integration/tests/correlations/failed_transactions.ts +++ b/x-pack/test/apm_api_integration/tests/correlations/failed_transactions.ts @@ -7,234 +7,201 @@ import expect from '@kbn/expect'; -import { IKibanaSearchRequest } from '../../../../../src/plugins/data/common'; - -import type { FailedTransactionsCorrelationsParams } from '../../../../plugins/apm/common/search_strategies/failed_transactions_correlations/types'; -import type { RawSearchStrategyClientParams } from '../../../../plugins/apm/common/search_strategies/types'; -import { APM_SEARCH_STRATEGIES } from '../../../../plugins/apm/common/search_strategies/constants'; - import { FtrProviderContext } from '../../common/ftr_provider_context'; import { registry } from '../../common/registry'; -import { parseBfetchResponse } from '../../common/utils/parse_b_fetch'; +import type { RawResponseBase } from '../../../../plugins/apm/common/correlations/types'; +import type { FailedTransactionsCorrelationsRawResponse } from '../../../../plugins/apm/common/correlations/failed_transactions_correlations/types'; +import { EVENT_OUTCOME } from '../../../../plugins/apm/common/elasticsearch_fieldnames'; +import { EventOutcome } from '../../../../plugins/apm/common/event_outcome'; +// These tests go through the full sequence of queries required +// to get the final results for a failed transactions correlation analysis. export default function ApiTest({ getService }: FtrProviderContext) { - const retry = getService('retry'); - const supertest = getService('legacySupertestAsApmReadUser'); - - const getRequestBody = () => { - const request: IKibanaSearchRequest< - FailedTransactionsCorrelationsParams & RawSearchStrategyClientParams - > = { - params: { - environment: 'ENVIRONMENT_ALL', - start: '2020', - end: '2021', - kuery: '', - percentileThreshold: 95, - }, - }; - - return { - batch: [ - { - request, - options: { strategy: APM_SEARCH_STRATEGIES.APM_FAILED_TRANSACTIONS_CORRELATIONS }, - }, - ], - }; - }; + const apmApiClient = getService('apmApiClient'); + + // This matches the parameters used for the other tab's queries in `../correlations/*`. + const getOptions = () => ({ + environment: 'ENVIRONMENT_ALL', + start: '2020', + end: '2021', + kuery: '', + }); registry.when('failed transactions without data', { config: 'trial', archives: [] }, () => { - it('queries the search strategy and returns results', async () => { - const intialResponse = await supertest - .post(`/internal/bsearch`) - .set('kbn-xsrf', 'foo') - .send(getRequestBody()); + it('handles the empty state', async () => { + const overallDistributionResponse = await apmApiClient.readUser({ + endpoint: 'POST /internal/apm/latency/overall_distribution', + params: { + body: { + ...getOptions(), + percentileThreshold: 95, + }, + }, + }); - expect(intialResponse.status).to.eql( + expect(overallDistributionResponse.status).to.eql( 200, - `Expected status to be '200', got '${intialResponse.status}'` - ); - expect(intialResponse.body).to.eql( - {}, - `Expected response body to be an empty object, actual response is in the text attribute. Got: '${JSON.stringify( - intialResponse.body - )}'` + `Expected status to be '200', got '${overallDistributionResponse.status}'` ); - const body = parseBfetchResponse(intialResponse)[0]; - - expect(typeof body.result).to.be('object'); - const { result } = body; - - expect(typeof result?.id).to.be('string'); - - // pass on id for follow up queries - const searchStrategyId = result.id; - - // follow up request body including search strategy ID - const reqBody = getRequestBody(); - reqBody.batch[0].request.id = searchStrategyId; - - let followUpResponse: Record = {}; + const errorDistributionResponse = await apmApiClient.readUser({ + endpoint: 'POST /internal/apm/latency/overall_distribution', + params: { + body: { + ...getOptions(), + percentileThreshold: 95, + termFilters: [{ fieldName: EVENT_OUTCOME, fieldValue: EventOutcome.failure }], + }, + }, + }); - // continues querying until the search strategy finishes - await retry.waitForWithTimeout( - 'search strategy eventually completes and returns full results', - 5000, - async () => { - const response = await supertest - .post(`/internal/bsearch`) - .set('kbn-xsrf', 'foo') - .send(reqBody); + expect(errorDistributionResponse.status).to.eql( + 200, + `Expected status to be '200', got '${errorDistributionResponse.status}'` + ); - followUpResponse = parseBfetchResponse(response)[0]; + const fieldCandidatesResponse = await apmApiClient.readUser({ + endpoint: 'GET /internal/apm/correlations/field_candidates', + params: { + query: getOptions(), + }, + }); - return ( - followUpResponse?.result?.isRunning === false || followUpResponse?.error !== undefined - ); - } + expect(fieldCandidatesResponse.status).to.eql( + 200, + `Expected status to be '200', got '${fieldCandidatesResponse.status}'` ); - expect(followUpResponse?.error).to.eql( - undefined, - `search strategy should not return an error, got: ${JSON.stringify( - followUpResponse?.error - )}` - ); + const failedTransactionsCorrelationsResponse = await apmApiClient.readUser({ + endpoint: 'POST /internal/apm/correlations/p_values', + params: { + body: { + ...getOptions(), + fieldCandidates: fieldCandidatesResponse.body?.fieldCandidates, + }, + }, + }); - const followUpResult = followUpResponse.result; - expect(followUpResult?.isRunning).to.eql(false, 'search strategy should not be running'); - expect(followUpResult?.isPartial).to.eql( - false, - 'search strategy result should not be partial' - ); - expect(followUpResult?.id).to.eql( - searchStrategyId, - 'search strategy id should match original id' - ); - expect(followUpResult?.isRestored).to.eql( - true, - 'search strategy response should be restored' + expect(failedTransactionsCorrelationsResponse.status).to.eql( + 200, + `Expected status to be '200', got '${failedTransactionsCorrelationsResponse.status}'` ); - expect(followUpResult?.loaded).to.eql(100, 'loaded state should be 100'); - expect(followUpResult?.total).to.eql(100, 'total state should be 100'); - expect(typeof followUpResult?.rawResponse).to.be('object'); - - const { rawResponse: finalRawResponse } = followUpResult; + const finalRawResponse: FailedTransactionsCorrelationsRawResponse & RawResponseBase = { + took: 0, + ccsWarning: failedTransactionsCorrelationsResponse.body?.ccsWarning, + percentileThresholdValue: overallDistributionResponse.body?.percentileThresholdValue, + overallHistogram: overallDistributionResponse.body?.overallHistogram, + failedTransactionsCorrelations: + failedTransactionsCorrelationsResponse.body?.failedTransactionsCorrelations, + log: [], + }; expect(typeof finalRawResponse?.took).to.be('number'); - expect(finalRawResponse?.failedTransactionsCorrelations.length).to.eql( + expect(finalRawResponse?.failedTransactionsCorrelations?.length).to.eql( 0, - `Expected 0 identified correlations, got ${finalRawResponse?.failedTransactionsCorrelations.length}.` + `Expected 0 identified correlations, got ${finalRawResponse?.failedTransactionsCorrelations?.length}.` ); }); }); registry.when('failed transactions with data', { config: 'trial', archives: ['8.0.0'] }, () => { - it('queries the search strategy and returns results', async () => { - const intialResponse = await supertest - .post(`/internal/bsearch`) - .set('kbn-xsrf', 'foo') - .send(getRequestBody()); + it('runs queries and returns results', async () => { + const overallDistributionResponse = await apmApiClient.readUser({ + endpoint: 'POST /internal/apm/latency/overall_distribution', + params: { + body: { + ...getOptions(), + percentileThreshold: 95, + }, + }, + }); - expect(intialResponse.status).to.eql( + expect(overallDistributionResponse.status).to.eql( 200, - `Expected status to be '200', got '${intialResponse.status}'` - ); - expect(intialResponse.body).to.eql( - {}, - `Expected response body to be an empty object, actual response is in the text attribute. Got: '${JSON.stringify( - intialResponse.body - )}'` + `Expected status to be '200', got '${overallDistributionResponse.status}'` ); - const body = parseBfetchResponse(intialResponse)[0]; - - expect(typeof body.result).to.be('object'); - const { result } = body; - - expect(typeof result?.id).to.be('string'); - - // pass on id for follow up queries - const searchStrategyId = result.id; - - // follow up request body including search strategy ID - const reqBody = getRequestBody(); - reqBody.batch[0].request.id = searchStrategyId; - - let followUpResponse: Record = {}; + const errorDistributionResponse = await apmApiClient.readUser({ + endpoint: 'POST /internal/apm/latency/overall_distribution', + params: { + body: { + ...getOptions(), + percentileThreshold: 95, + termFilters: [{ fieldName: EVENT_OUTCOME, fieldValue: EventOutcome.failure }], + }, + }, + }); - // continues querying until the search strategy finishes - await retry.waitForWithTimeout( - 'search strategy eventually completes and returns full results', - 5000, - async () => { - const response = await supertest - .post(`/internal/bsearch`) - .set('kbn-xsrf', 'foo') - .send(reqBody); + expect(errorDistributionResponse.status).to.eql( + 200, + `Expected status to be '200', got '${errorDistributionResponse.status}'` + ); - followUpResponse = parseBfetchResponse(response)[0]; + const fieldCandidatesResponse = await apmApiClient.readUser({ + endpoint: 'GET /internal/apm/correlations/field_candidates', + params: { + query: getOptions(), + }, + }); - return ( - followUpResponse?.result?.isRunning === false || followUpResponse?.error !== undefined - ); - } + expect(fieldCandidatesResponse.status).to.eql( + 200, + `Expected status to be '200', got '${fieldCandidatesResponse.status}'` ); - expect(followUpResponse?.error).to.eql( - undefined, - `search strategy should not return an error, got: ${JSON.stringify( - followUpResponse?.error - )}` + const fieldCandidates = fieldCandidatesResponse.body?.fieldCandidates.filter( + (t) => !(t === EVENT_OUTCOME) ); - const followUpResult = followUpResponse.result; - expect(followUpResult?.isRunning).to.eql(false, 'search strategy should not be running'); - expect(followUpResult?.isPartial).to.eql( - false, - 'search strategy result should not be partial' - ); - expect(followUpResult?.id).to.eql( - searchStrategyId, - 'search strategy id should match original id' + // Identified 68 fieldCandidates. + expect(fieldCandidates.length).to.eql( + 68, + `Expected field candidates length to be '68', got '${fieldCandidates.length}'` ); - expect(followUpResult?.isRestored).to.eql( - true, - 'search strategy response should be restored' - ); - expect(followUpResult?.loaded).to.eql(100, 'loaded state should be 100'); - expect(followUpResult?.total).to.eql(100, 'total state should be 100'); - expect(typeof followUpResult?.rawResponse).to.be('object'); + const failedTransactionsCorrelationsResponse = await apmApiClient.readUser({ + endpoint: 'POST /internal/apm/correlations/p_values', + params: { + body: { + ...getOptions(), + fieldCandidates, + }, + }, + }); - const { rawResponse: finalRawResponse } = followUpResult; + expect(failedTransactionsCorrelationsResponse.status).to.eql( + 200, + `Expected status to be '200', got '${failedTransactionsCorrelationsResponse.status}'` + ); + + const finalRawResponse: FailedTransactionsCorrelationsRawResponse & RawResponseBase = { + took: 0, + ccsWarning: failedTransactionsCorrelationsResponse.body?.ccsWarning, + percentileThresholdValue: overallDistributionResponse.body?.percentileThresholdValue, + overallHistogram: overallDistributionResponse.body?.overallHistogram, + errorHistogram: errorDistributionResponse.body?.overallHistogram, + failedTransactionsCorrelations: + failedTransactionsCorrelationsResponse.body?.failedTransactionsCorrelations, + log: [], + }; expect(typeof finalRawResponse?.took).to.be('number'); expect(finalRawResponse?.percentileThresholdValue).to.be(1309695.875); - expect(finalRawResponse?.errorHistogram.length).to.be(101); - expect(finalRawResponse?.overallHistogram.length).to.be(101); - expect(finalRawResponse?.fieldStats.length).to.be(26); + expect(finalRawResponse?.errorHistogram?.length).to.be(101); + expect(finalRawResponse?.overallHistogram?.length).to.be(101); + expect(finalRawResponse?.fieldStats?.length).to.be(26); - expect(finalRawResponse?.failedTransactionsCorrelations.length).to.eql( + expect(finalRawResponse?.failedTransactionsCorrelations?.length).to.eql( 30, - `Expected 30 identified correlations, got ${finalRawResponse?.failedTransactionsCorrelations.length}.` + `Expected 30 identified correlations, got ${finalRawResponse?.failedTransactionsCorrelations?.length}.` ); - expect(finalRawResponse?.log.map((d: string) => d.split(': ')[1])).to.eql([ - 'Fetched 95th percentile value of 1309695.875 based on 1244 documents.', - 'Identified 68 fieldCandidates.', - 'Identified correlations for 68 fields out of 68 candidates.', - 'Identified 26 fields to sample for field statistics.', - 'Retrieved field statistics for 26 fields out of 26 fields.', - 'Identified 30 significant correlations relating to failed transactions.', - ]); - - const sortedCorrelations = finalRawResponse?.failedTransactionsCorrelations.sort(); - const correlation = sortedCorrelations[0]; + const sortedCorrelations = finalRawResponse?.failedTransactionsCorrelations?.sort( + (a, b) => b.score - a.score + ); + const correlation = sortedCorrelations?.[0]; expect(typeof correlation).to.be('object'); expect(correlation?.doc_count).to.be(31); @@ -247,10 +214,12 @@ export default function ApiTest({ getService }: FtrProviderContext) { expect(typeof correlation?.failurePercentage).to.be('number'); expect(typeof correlation?.successPercentage).to.be('number'); - const fieldStats = finalRawResponse?.fieldStats[0]; + const fieldStats = finalRawResponse?.fieldStats?.[0]; expect(typeof fieldStats).to.be('object'); - expect(fieldStats.topValues.length).to.greaterThan(0); - expect(fieldStats.topValuesSampleSize).to.greaterThan(0); + expect(Array.isArray(fieldStats?.topValues) && fieldStats?.topValues?.length).to.greaterThan( + 0 + ); + expect(fieldStats?.topValuesSampleSize).to.greaterThan(0); }); }); } diff --git a/x-pack/test/apm_api_integration/tests/correlations/field_candidates.ts b/x-pack/test/apm_api_integration/tests/correlations/field_candidates.ts new file mode 100644 index 0000000000000..16c157d026d33 --- /dev/null +++ b/x-pack/test/apm_api_integration/tests/correlations/field_candidates.ts @@ -0,0 +1,55 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { FtrProviderContext } from '../../common/ftr_provider_context'; +import { registry } from '../../common/registry'; + +export default function ApiTest({ getService }: FtrProviderContext) { + const apmApiClient = getService('apmApiClient'); + + const endpoint = 'GET /internal/apm/correlations/field_candidates'; + + const getOptions = () => ({ + params: { + query: { + environment: 'ENVIRONMENT_ALL', + start: '2020', + end: '2021', + kuery: '', + }, + }, + }); + + registry.when('field candidates without data', { config: 'trial', archives: [] }, () => { + it('handles the empty state', async () => { + const response = await apmApiClient.readUser({ + endpoint, + ...getOptions(), + }); + + expect(response.status).to.be(200); + expect(response.body?.fieldCandidates.length).to.be(14); + }); + }); + + registry.when( + 'field candidates with data and default args', + { config: 'trial', archives: ['8.0.0'] }, + () => { + it('returns field candidates', async () => { + const response = await apmApiClient.readUser({ + endpoint, + ...getOptions(), + }); + + expect(response.status).to.eql(200); + expect(response.body?.fieldCandidates.length).to.be(69); + }); + } + ); +} diff --git a/x-pack/test/apm_api_integration/tests/correlations/field_value_pairs.ts b/x-pack/test/apm_api_integration/tests/correlations/field_value_pairs.ts new file mode 100644 index 0000000000000..800938a36d127 --- /dev/null +++ b/x-pack/test/apm_api_integration/tests/correlations/field_value_pairs.ts @@ -0,0 +1,71 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { FtrProviderContext } from '../../common/ftr_provider_context'; +import { registry } from '../../common/registry'; + +export default function ApiTest({ getService }: FtrProviderContext) { + const apmApiClient = getService('apmApiClient'); + + const endpoint = 'GET /internal/apm/correlations/field_value_pairs'; + + const getOptions = () => ({ + params: { + query: { + environment: 'ENVIRONMENT_ALL', + start: '2020', + end: '2021', + kuery: '', + fieldCandidates: [ + 'service.version', + 'service.node.name', + 'service.framework.version', + 'service.language.version', + 'service.runtime.version', + 'kubernetes.pod.name', + 'kubernetes.pod.uid', + 'container.id', + 'source.ip', + 'client.ip', + 'host.ip', + 'service.environment', + 'process.args', + 'http.response.status_code', + ], + }, + }, + }); + + registry.when('field value pairs without data', { config: 'trial', archives: [] }, () => { + it('handles the empty state', async () => { + const response = await apmApiClient.readUser({ + endpoint, + ...getOptions(), + }); + + expect(response.status).to.be(200); + expect(response.body?.fieldValuePairs.length).to.be(0); + }); + }); + + registry.when( + 'field value pairs with data and default args', + { config: 'trial', archives: ['8.0.0'] }, + () => { + it('returns field value pairs', async () => { + const response = await apmApiClient.readUser({ + endpoint, + ...getOptions(), + }); + + expect(response.status).to.eql(200); + expect(response.body?.fieldValuePairs.length).to.be(124); + }); + } + ); +} diff --git a/x-pack/test/apm_api_integration/tests/correlations/latency.ts b/x-pack/test/apm_api_integration/tests/correlations/latency.ts index 75a4edd447c70..aee5b47b1c8f6 100644 --- a/x-pack/test/apm_api_integration/tests/correlations/latency.ts +++ b/x-pack/test/apm_api_integration/tests/correlations/latency.ts @@ -7,134 +7,99 @@ import expect from '@kbn/expect'; -import { IKibanaSearchRequest } from '../../../../../src/plugins/data/common'; - -import type { LatencyCorrelationsParams } from '../../../../plugins/apm/common/search_strategies/latency_correlations/types'; -import type { RawSearchStrategyClientParams } from '../../../../plugins/apm/common/search_strategies/types'; -import { APM_SEARCH_STRATEGIES } from '../../../../plugins/apm/common/search_strategies/constants'; - import { FtrProviderContext } from '../../common/ftr_provider_context'; import { registry } from '../../common/registry'; -import { parseBfetchResponse } from '../../common/utils/parse_b_fetch'; +import type { RawResponseBase } from '../../../../plugins/apm/common/correlations/types'; +import type { LatencyCorrelationsRawResponse } from '../../../../plugins/apm/common/correlations/latency_correlations/types'; +// These tests go through the full sequence of queries required +// to get the final results for a latency correlation analysis. export default function ApiTest({ getService }: FtrProviderContext) { - const retry = getService('retry'); - const supertest = getService('legacySupertestAsApmReadUser'); - - const getRequestBody = () => { - const request: IKibanaSearchRequest = - { - params: { - environment: 'ENVIRONMENT_ALL', - start: '2020', - end: '2021', - kuery: '', - percentileThreshold: 95, - analyzeCorrelations: true, - }, - }; + const apmApiClient = getService('apmApiClient'); - return { - batch: [ - { - request, - options: { strategy: APM_SEARCH_STRATEGIES.APM_LATENCY_CORRELATIONS }, - }, - ], - }; - }; + // This matches the parameters used for the other tab's queries in `../correlations/*`. + const getOptions = () => ({ + environment: 'ENVIRONMENT_ALL', + start: '2020', + end: '2021', + kuery: '', + }); registry.when( - 'correlations latency_ml overall without data', + 'correlations latency overall without data', { config: 'trial', archives: [] }, () => { it('handles the empty state', async () => { - const intialResponse = await supertest - .post(`/internal/bsearch`) - .set('kbn-xsrf', 'foo') - .send(getRequestBody()); - - expect(intialResponse.status).to.eql( + const overallDistributionResponse = await apmApiClient.readUser({ + endpoint: 'POST /internal/apm/latency/overall_distribution', + params: { + body: { + ...getOptions(), + percentileThreshold: 95, + }, + }, + }); + + expect(overallDistributionResponse.status).to.eql( 200, - `Expected status to be '200', got '${intialResponse.status}'` - ); - expect(intialResponse.body).to.eql( - {}, - `Expected response body to be an empty object, actual response is in the text attribute. Got: '${JSON.stringify( - intialResponse.body - )}'` + `Expected status to be '200', got '${overallDistributionResponse.status}'` ); - const body = parseBfetchResponse(intialResponse)[0]; - - expect(typeof body.result).to.be('object'); - const { result } = body; - - expect(typeof result?.id).to.be('string'); - - // pass on id for follow up queries - const searchStrategyId = result.id; - - // follow up request body including search strategy ID - const reqBody = getRequestBody(); - reqBody.batch[0].request.id = searchStrategyId; + const fieldCandidatesResponse = await apmApiClient.readUser({ + endpoint: 'GET /internal/apm/correlations/field_candidates', + params: { + query: getOptions(), + }, + }); - let followUpResponse: Record = {}; - - // continues querying until the search strategy finishes - await retry.waitForWithTimeout( - 'search strategy eventually completes and returns full results', - 5000, - async () => { - const response = await supertest - .post(`/internal/bsearch`) - .set('kbn-xsrf', 'foo') - .send(reqBody); - - followUpResponse = parseBfetchResponse(response)[0]; - - return ( - followUpResponse?.result?.isRunning === false || followUpResponse?.error !== undefined - ); - } + expect(fieldCandidatesResponse.status).to.eql( + 200, + `Expected status to be '200', got '${fieldCandidatesResponse.status}'` ); - expect(followUpResponse?.error).to.eql( - undefined, - `search strategy should not return an error, got: ${JSON.stringify( - followUpResponse?.error - )}` + const fieldValuePairsResponse = await apmApiClient.readUser({ + endpoint: 'GET /internal/apm/correlations/field_value_pairs', + params: { + query: { + ...getOptions(), + fieldCandidates: fieldCandidatesResponse.body?.fieldCandidates, + }, + }, + }); + + expect(fieldValuePairsResponse.status).to.eql( + 200, + `Expected status to be '200', got '${fieldValuePairsResponse.status}'` ); - const followUpResult = followUpResponse.result; - expect(followUpResult?.isRunning).to.eql(false, 'search strategy should not be running'); - expect(followUpResult?.isPartial).to.eql( - false, - 'search strategy result should not be partial' - ); - expect(followUpResult?.id).to.eql( - searchStrategyId, - 'search strategy id should match original id' - ); - expect(followUpResult?.isRestored).to.eql( - true, - 'search strategy response should be restored' + const significantCorrelationsResponse = await apmApiClient.readUser({ + endpoint: 'POST /internal/apm/correlations/significant_correlations', + params: { + body: { + ...getOptions(), + fieldValuePairs: fieldValuePairsResponse.body?.fieldValuePairs, + }, + }, + }); + + expect(significantCorrelationsResponse.status).to.eql( + 200, + `Expected status to be '200', got '${significantCorrelationsResponse.status}'` ); - expect(followUpResult?.loaded).to.eql(100, 'loaded state should be 100'); - expect(followUpResult?.total).to.eql(100, 'total state should be 100'); - expect(typeof followUpResult?.rawResponse).to.be('object'); - - const { rawResponse: finalRawResponse } = followUpResult; + const finalRawResponse: LatencyCorrelationsRawResponse & RawResponseBase = { + took: 0, + ccsWarning: significantCorrelationsResponse.body?.ccsWarning, + percentileThresholdValue: overallDistributionResponse.body?.percentileThresholdValue, + overallHistogram: overallDistributionResponse.body?.overallHistogram, + latencyCorrelations: significantCorrelationsResponse.body?.latencyCorrelations, + log: [], + }; expect(typeof finalRawResponse?.took).to.be('number'); expect(finalRawResponse?.percentileThresholdValue).to.be(undefined); expect(finalRawResponse?.overallHistogram).to.be(undefined); - expect(finalRawResponse?.latencyCorrelations.length).to.be(0); - expect(finalRawResponse?.log.map((d: string) => d.split(': ')[1])).to.eql([ - 'Fetched 95th percentile value of undefined based on 0 documents.', - 'Abort service since percentileThresholdValue could not be determined.', - ]); + expect(finalRawResponse?.latencyCorrelations?.length).to.be(0); }); } ); @@ -144,120 +109,106 @@ export default function ApiTest({ getService }: FtrProviderContext) { { config: 'trial', archives: ['8.0.0'] }, () => { // putting this into a single `it` because the responses depend on each other - it('queries the search strategy and returns results', async () => { - const intialResponse = await supertest - .post(`/internal/bsearch`) - .set('kbn-xsrf', 'foo') - .send(getRequestBody()); - - expect(intialResponse.status).to.eql( + it('runs queries and returns results', async () => { + const overallDistributionResponse = await apmApiClient.readUser({ + endpoint: 'POST /internal/apm/latency/overall_distribution', + params: { + body: { + ...getOptions(), + percentileThreshold: 95, + }, + }, + }); + + expect(overallDistributionResponse.status).to.eql( 200, - `Expected status to be '200', got '${intialResponse.status}'` - ); - expect(intialResponse.body).to.eql( - {}, - `Expected response body to be an empty object, actual response is in the text attribute. Got: '${JSON.stringify( - intialResponse.body - )}'` + `Expected status to be '200', got '${overallDistributionResponse.status}'` ); - const body = parseBfetchResponse(intialResponse)[0]; - - expect(typeof body?.result).to.be('object'); - const { result } = body; - - expect(typeof result?.id).to.be('string'); + const fieldCandidatesResponse = await apmApiClient.readUser({ + endpoint: 'GET /internal/apm/correlations/field_candidates', + params: { + query: getOptions(), + }, + }); - // pass on id for follow up queries - const searchStrategyId = result.id; - - expect(result?.loaded).to.be(0); - expect(result?.total).to.be(100); - expect(result?.isRunning).to.be(true); - expect(result?.isPartial).to.be(true); - expect(result?.isRestored).to.eql( - false, - `Expected response result to be not restored. Got: '${result?.isRestored}'` + expect(fieldCandidatesResponse.status).to.eql( + 200, + `Expected status to be '200', got '${fieldCandidatesResponse.status}'` ); - expect(typeof result?.rawResponse).to.be('object'); - - const { rawResponse } = result; - - expect(typeof rawResponse?.took).to.be('number'); - expect(rawResponse?.latencyCorrelations).to.eql([]); - // follow up request body including search strategy ID - const reqBody = getRequestBody(); - reqBody.batch[0].request.id = searchStrategyId; - - let followUpResponse: Record = {}; - - // continues querying until the search strategy finishes - await retry.waitForWithTimeout( - 'search strategy eventually completes and returns full results', - 5000, - async () => { - const response = await supertest - .post(`/internal/bsearch`) - .set('kbn-xsrf', 'foo') - .send(reqBody); - followUpResponse = parseBfetchResponse(response)[0]; - - return ( - followUpResponse?.result?.isRunning === false || followUpResponse?.error !== undefined - ); - } + // Identified 69 fieldCandidates. + expect(fieldCandidatesResponse.body?.fieldCandidates.length).to.eql( + 69, + `Expected field candidates length to be '69', got '${fieldCandidatesResponse.body?.fieldCandidates.length}'` ); - expect(followUpResponse?.error).to.eql( - undefined, - `Finished search strategy should not return an error, got: ${JSON.stringify( - followUpResponse?.error - )}` + const fieldValuePairsResponse = await apmApiClient.readUser({ + endpoint: 'GET /internal/apm/correlations/field_value_pairs', + params: { + query: { + ...getOptions(), + fieldCandidates: fieldCandidatesResponse.body?.fieldCandidates, + }, + }, + }); + + expect(fieldValuePairsResponse.status).to.eql( + 200, + `Expected status to be '200', got '${fieldValuePairsResponse.status}'` ); - const followUpResult = followUpResponse.result; - expect(followUpResult?.isRunning).to.eql( - false, - `Expected finished result not to be running. Got: ${followUpResult?.isRunning}` + // Identified 379 fieldValuePairs. + expect(fieldValuePairsResponse.body?.fieldValuePairs.length).to.eql( + 379, + `Expected field value pairs length to be '379', got '${fieldValuePairsResponse.body?.fieldValuePairs.length}'` ); - expect(followUpResult?.isPartial).to.eql( - false, - `Expected finished result not to be partial. Got: ${followUpResult?.isPartial}` + + const significantCorrelationsResponse = await apmApiClient.readUser({ + endpoint: 'POST /internal/apm/correlations/significant_correlations', + params: { + body: { + ...getOptions(), + fieldValuePairs: fieldValuePairsResponse.body?.fieldValuePairs, + }, + }, + }); + + expect(significantCorrelationsResponse.status).to.eql( + 200, + `Expected status to be '200', got '${significantCorrelationsResponse.status}'` ); - expect(followUpResult?.id).to.be(searchStrategyId); - expect(followUpResult?.isRestored).to.be(true); - expect(followUpResult?.loaded).to.be(100); - expect(followUpResult?.total).to.be(100); - expect(typeof followUpResult?.rawResponse).to.be('object'); + // Loaded fractions and totalDocCount of 1244. + expect(significantCorrelationsResponse.body?.totalDocCount).to.eql( + 1244, + `Expected 1244 total doc count, got ${significantCorrelationsResponse.body?.totalDocCount}.` + ); - const { rawResponse: finalRawResponse } = followUpResult; + const finalRawResponse: LatencyCorrelationsRawResponse & RawResponseBase = { + took: 0, + ccsWarning: significantCorrelationsResponse.body?.ccsWarning, + percentileThresholdValue: overallDistributionResponse.body?.percentileThresholdValue, + overallHistogram: overallDistributionResponse.body?.overallHistogram, + latencyCorrelations: significantCorrelationsResponse.body?.latencyCorrelations, + log: [], + }; expect(typeof finalRawResponse?.took).to.be('number'); + // Fetched 95th percentile value of 1309695.875 based on 1244 documents. expect(finalRawResponse?.percentileThresholdValue).to.be(1309695.875); - expect(finalRawResponse?.overallHistogram.length).to.be(101); - expect(finalRawResponse?.fieldStats.length).to.be(12); + expect(finalRawResponse?.overallHistogram?.length).to.be(101); + expect(finalRawResponse?.fieldStats?.length).to.be(12); - expect(finalRawResponse?.latencyCorrelations.length).to.eql( + // Identified 13 significant correlations out of 379 field/value pairs. + expect(finalRawResponse?.latencyCorrelations?.length).to.eql( 13, - `Expected 13 identified correlations, got ${finalRawResponse?.latencyCorrelations.length}.` + `Expected 13 identified correlations, got ${finalRawResponse?.latencyCorrelations?.length}.` ); - expect(finalRawResponse?.log.map((d: string) => d.split(': ')[1])).to.eql([ - 'Fetched 95th percentile value of 1309695.875 based on 1244 documents.', - 'Loaded histogram range steps.', - 'Loaded overall histogram chart data.', - 'Loaded percentiles.', - 'Identified 69 fieldCandidates.', - 'Identified 379 fieldValuePairs.', - 'Loaded fractions and totalDocCount of 1244.', - 'Identified 13 significant correlations out of 379 field/value pairs.', - 'Identified 12 fields to sample for field statistics.', - 'Retrieved field statistics for 12 fields out of 12 fields.', - ]); - - const correlation = finalRawResponse?.latencyCorrelations[0]; + const correlation = finalRawResponse?.latencyCorrelations?.sort( + (a, b) => b.correlation - a.correlation + )[0]; expect(typeof correlation).to.be('object'); expect(correlation?.fieldName).to.be('transaction.result'); expect(correlation?.fieldValue).to.be('success'); @@ -265,10 +216,12 @@ export default function ApiTest({ getService }: FtrProviderContext) { expect(correlation?.ksTest).to.be(4.806503252860024e-13); expect(correlation?.histogram.length).to.be(101); - const fieldStats = finalRawResponse?.fieldStats[0]; + const fieldStats = finalRawResponse?.fieldStats?.[0]; expect(typeof fieldStats).to.be('object'); - expect(fieldStats.topValues.length).to.greaterThan(0); - expect(fieldStats.topValuesSampleSize).to.greaterThan(0); + expect( + Array.isArray(fieldStats?.topValues) && fieldStats?.topValues?.length + ).to.greaterThan(0); + expect(fieldStats?.topValuesSampleSize).to.greaterThan(0); }); } ); diff --git a/x-pack/test/apm_api_integration/tests/correlations/p_values.ts b/x-pack/test/apm_api_integration/tests/correlations/p_values.ts new file mode 100644 index 0000000000000..444cf57d35325 --- /dev/null +++ b/x-pack/test/apm_api_integration/tests/correlations/p_values.ts @@ -0,0 +1,71 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { FtrProviderContext } from '../../common/ftr_provider_context'; +import { registry } from '../../common/registry'; + +export default function ApiTest({ getService }: FtrProviderContext) { + const apmApiClient = getService('apmApiClient'); + + const endpoint = 'POST /internal/apm/correlations/p_values'; + + const getOptions = () => ({ + params: { + body: { + environment: 'ENVIRONMENT_ALL', + start: '2020', + end: '2021', + kuery: '', + fieldCandidates: [ + 'service.version', + 'service.node.name', + 'service.framework.version', + 'service.language.version', + 'service.runtime.version', + 'kubernetes.pod.name', + 'kubernetes.pod.uid', + 'container.id', + 'source.ip', + 'client.ip', + 'host.ip', + 'service.environment', + 'process.args', + 'http.response.status_code', + ], + }, + }, + }); + + registry.when('p values without data', { config: 'trial', archives: [] }, () => { + it('handles the empty state', async () => { + const response = await apmApiClient.readUser({ + endpoint, + ...getOptions(), + }); + + expect(response.status).to.be(200); + expect(response.body?.failedTransactionsCorrelations.length).to.be(0); + }); + }); + + registry.when( + 'p values with data and default args', + { config: 'trial', archives: ['8.0.0'] }, + () => { + it('returns p values', async () => { + const response = await apmApiClient.readUser({ + endpoint, + ...getOptions(), + }); + + expect(response.status).to.eql(200); + expect(response.body?.failedTransactionsCorrelations.length).to.be(15); + }); + } + ); +} diff --git a/x-pack/test/apm_api_integration/tests/correlations/significant_correlations.ts b/x-pack/test/apm_api_integration/tests/correlations/significant_correlations.ts new file mode 100644 index 0000000000000..404584f959380 --- /dev/null +++ b/x-pack/test/apm_api_integration/tests/correlations/significant_correlations.ts @@ -0,0 +1,95 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { FtrProviderContext } from '../../common/ftr_provider_context'; +import { registry } from '../../common/registry'; + +export default function ApiTest({ getService }: FtrProviderContext) { + const apmApiClient = getService('apmApiClient'); + + const endpoint = 'POST /internal/apm/correlations/significant_correlations'; + + const getOptions = () => ({ + params: { + body: { + environment: 'ENVIRONMENT_ALL', + start: '2020', + end: '2021', + kuery: '', + fieldValuePairs: [ + { fieldName: 'service.version', fieldValue: '2020-08-26 02:09:20' }, + { fieldName: 'service.version', fieldValue: 'None' }, + { + fieldName: 'service.node.name', + fieldValue: 'af586da824b28435f3a8c8f0c016096502cd2495d64fb332db23312be88cfff6', + }, + { + fieldName: 'service.node.name', + fieldValue: 'asdf', + }, + { fieldName: 'service.runtime.version', fieldValue: '12.18.3' }, + { fieldName: 'service.runtime.version', fieldValue: '2.6.6' }, + { + fieldName: 'kubernetes.pod.name', + fieldValue: 'opbeans-node-6cf6cf6f58-r5q9l', + }, + { + fieldName: 'kubernetes.pod.name', + fieldValue: 'opbeans-java-6dc7465984-h9sh5', + }, + { + fieldName: 'kubernetes.pod.uid', + fieldValue: '8da9c944-e741-11ea-819e-42010a84004a', + }, + { + fieldName: 'kubernetes.pod.uid', + fieldValue: '8e192c6c-e741-11ea-819e-42010a84004a', + }, + { + fieldName: 'container.id', + fieldValue: 'af586da824b28435f3a8c8f0c016096502cd2495d64fb332db23312be88cfff6', + }, + { + fieldName: 'container.id', + fieldValue: 'asdf', + }, + { fieldName: 'host.ip', fieldValue: '10.52.6.48' }, + { fieldName: 'host.ip', fieldValue: '10.52.6.50' }, + ], + }, + }, + }); + + registry.when('significant correlations without data', { config: 'trial', archives: [] }, () => { + it('handles the empty state', async () => { + const response = await apmApiClient.readUser({ + endpoint, + ...getOptions(), + }); + + expect(response.status).to.be(200); + expect(response.body?.latencyCorrelations.length).to.be(0); + }); + }); + + registry.when( + 'significant correlations with data and default args', + { config: 'trial', archives: ['8.0.0'] }, + () => { + it('returns significant correlations', async () => { + const response = await apmApiClient.readUser({ + endpoint, + ...getOptions(), + }); + + expect(response.status).to.eql(200); + expect(response.body?.latencyCorrelations.length).to.be(7); + }); + } + ); +} diff --git a/x-pack/test/apm_api_integration/tests/index.ts b/x-pack/test/apm_api_integration/tests/index.ts index f68a49658f2ee..ae91fe118c781 100644 --- a/x-pack/test/apm_api_integration/tests/index.ts +++ b/x-pack/test/apm_api_integration/tests/index.ts @@ -28,13 +28,13 @@ export default function apmApiIntegrationTests(providerContext: FtrProviderConte loadTestFile(require.resolve('./alerts/rule_registry')); }); - // correlations - describe('correlations/failed_transactions', function () { + describe('correlations', function () { loadTestFile(require.resolve('./correlations/failed_transactions')); - }); - - describe('correlations/latency', function () { + loadTestFile(require.resolve('./correlations/field_candidates')); + loadTestFile(require.resolve('./correlations/field_value_pairs')); loadTestFile(require.resolve('./correlations/latency')); + loadTestFile(require.resolve('./correlations/p_values')); + loadTestFile(require.resolve('./correlations/significant_correlations')); }); describe('metadata/event_metadata', function () { From 9de6ff69358fb130ab746d9f0505629c9a2cc964 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Sun, 17 Oct 2021 17:00:02 +0200 Subject: [PATCH 09/39] [ML] Remove the no longer needed 'took' attribute. --- x-pack/plugins/apm/common/correlations/types.ts | 1 - .../app/correlations/latency_correlations.test.tsx | 2 -- .../app/correlations/use_failed_transactions_correlations.ts | 1 - .../components/app/correlations/use_latency_correlations.ts | 1 - .../tests/correlations/failed_transactions.ts | 5 ----- .../test/apm_api_integration/tests/correlations/latency.ts | 4 ---- 6 files changed, 14 deletions(-) diff --git a/x-pack/plugins/apm/common/correlations/types.ts b/x-pack/plugins/apm/common/correlations/types.ts index ff925f70fc9b0..3321826c49e46 100644 --- a/x-pack/plugins/apm/common/correlations/types.ts +++ b/x-pack/plugins/apm/common/correlations/types.ts @@ -28,7 +28,6 @@ export interface ResponseHit { export interface RawResponseBase { ccsWarning: boolean; - took: number; } export interface SearchStrategyClientParamsBase { diff --git a/x-pack/plugins/apm/public/components/app/correlations/latency_correlations.test.tsx b/x-pack/plugins/apm/public/components/app/correlations/latency_correlations.test.tsx index b2170c99bb6fe..cafbacae6babd 100644 --- a/x-pack/plugins/apm/public/components/app/correlations/latency_correlations.test.tsx +++ b/x-pack/plugins/apm/public/components/app/correlations/latency_correlations.test.tsx @@ -99,7 +99,6 @@ describe('correlations', () => { isRunning: true, rawResponse: { ccsWarning: false, - took: 1234, latencyCorrelations: [], log: [], }, @@ -122,7 +121,6 @@ describe('correlations', () => { isRunning: false, rawResponse: { ccsWarning: false, - took: 1234, latencyCorrelations: [], log: [], }, diff --git a/x-pack/plugins/apm/public/components/app/correlations/use_failed_transactions_correlations.ts b/x-pack/plugins/apm/public/components/app/correlations/use_failed_transactions_correlations.ts index 26bbc92d50e8f..9a1a0d4ef43c2 100644 --- a/x-pack/plugins/apm/public/components/app/correlations/use_failed_transactions_correlations.ts +++ b/x-pack/plugins/apm/public/components/app/correlations/use_failed_transactions_correlations.ts @@ -44,7 +44,6 @@ function getFailedTransactionsCorrelationsSortedByScore( const getInitialRawResponse = (): Response => ({ ccsWarning: false, - took: 0, } as Response); const getInitialProgress = (): SearchStrategyProgress => ({ diff --git a/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts b/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts index 6768156279d23..918df253a3053 100644 --- a/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts +++ b/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts @@ -42,7 +42,6 @@ function getLatencyCorrelationsSortedByCorrelation( const getInitialRawResponse = (): Response => ({ ccsWarning: false, - took: 0, } as Response); const getInitialProgress = (): SearchStrategyProgress => ({ diff --git a/x-pack/test/apm_api_integration/tests/correlations/failed_transactions.ts b/x-pack/test/apm_api_integration/tests/correlations/failed_transactions.ts index f46d5ddfb38a5..492b9c7df1715 100644 --- a/x-pack/test/apm_api_integration/tests/correlations/failed_transactions.ts +++ b/x-pack/test/apm_api_integration/tests/correlations/failed_transactions.ts @@ -88,7 +88,6 @@ export default function ApiTest({ getService }: FtrProviderContext) { ); const finalRawResponse: FailedTransactionsCorrelationsRawResponse & RawResponseBase = { - took: 0, ccsWarning: failedTransactionsCorrelationsResponse.body?.ccsWarning, percentileThresholdValue: overallDistributionResponse.body?.percentileThresholdValue, overallHistogram: overallDistributionResponse.body?.overallHistogram, @@ -97,8 +96,6 @@ export default function ApiTest({ getService }: FtrProviderContext) { log: [], }; - expect(typeof finalRawResponse?.took).to.be('number'); - expect(finalRawResponse?.failedTransactionsCorrelations?.length).to.eql( 0, `Expected 0 identified correlations, got ${finalRawResponse?.failedTransactionsCorrelations?.length}.` @@ -177,7 +174,6 @@ export default function ApiTest({ getService }: FtrProviderContext) { ); const finalRawResponse: FailedTransactionsCorrelationsRawResponse & RawResponseBase = { - took: 0, ccsWarning: failedTransactionsCorrelationsResponse.body?.ccsWarning, percentileThresholdValue: overallDistributionResponse.body?.percentileThresholdValue, overallHistogram: overallDistributionResponse.body?.overallHistogram, @@ -187,7 +183,6 @@ export default function ApiTest({ getService }: FtrProviderContext) { log: [], }; - expect(typeof finalRawResponse?.took).to.be('number'); expect(finalRawResponse?.percentileThresholdValue).to.be(1309695.875); expect(finalRawResponse?.errorHistogram?.length).to.be(101); expect(finalRawResponse?.overallHistogram?.length).to.be(101); diff --git a/x-pack/test/apm_api_integration/tests/correlations/latency.ts b/x-pack/test/apm_api_integration/tests/correlations/latency.ts index aee5b47b1c8f6..2151a9f8244ba 100644 --- a/x-pack/test/apm_api_integration/tests/correlations/latency.ts +++ b/x-pack/test/apm_api_integration/tests/correlations/latency.ts @@ -88,7 +88,6 @@ export default function ApiTest({ getService }: FtrProviderContext) { ); const finalRawResponse: LatencyCorrelationsRawResponse & RawResponseBase = { - took: 0, ccsWarning: significantCorrelationsResponse.body?.ccsWarning, percentileThresholdValue: overallDistributionResponse.body?.percentileThresholdValue, overallHistogram: overallDistributionResponse.body?.overallHistogram, @@ -96,7 +95,6 @@ export default function ApiTest({ getService }: FtrProviderContext) { log: [], }; - expect(typeof finalRawResponse?.took).to.be('number'); expect(finalRawResponse?.percentileThresholdValue).to.be(undefined); expect(finalRawResponse?.overallHistogram).to.be(undefined); expect(finalRawResponse?.latencyCorrelations?.length).to.be(0); @@ -186,7 +184,6 @@ export default function ApiTest({ getService }: FtrProviderContext) { ); const finalRawResponse: LatencyCorrelationsRawResponse & RawResponseBase = { - took: 0, ccsWarning: significantCorrelationsResponse.body?.ccsWarning, percentileThresholdValue: overallDistributionResponse.body?.percentileThresholdValue, overallHistogram: overallDistributionResponse.body?.overallHistogram, @@ -194,7 +191,6 @@ export default function ApiTest({ getService }: FtrProviderContext) { log: [], }; - expect(typeof finalRawResponse?.took).to.be('number'); // Fetched 95th percentile value of 1309695.875 based on 1244 documents. expect(finalRawResponse?.percentileThresholdValue).to.be(1309695.875); expect(finalRawResponse?.overallHistogram?.length).to.be(101); From 30d3883b370cd4eb5766bd05bcaef1f82e22269e Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Sun, 17 Oct 2021 17:29:30 +0200 Subject: [PATCH 10/39] [ML] Adds chunking to field value pair loading. --- .../correlations/use_latency_correlations.ts | 55 +++++++++---- .../queries/query_field_value_pairs.test.ts | 6 -- .../queries/query_field_value_pairs.ts | 78 ++++++------------- 3 files changed, 64 insertions(+), 75 deletions(-) diff --git a/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts b/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts index 918df253a3053..d5911f256c2f5 100644 --- a/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts +++ b/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts @@ -11,7 +11,10 @@ import { chunk } from 'lodash'; import { IHttpFetchError } from 'src/core/public'; import { DEFAULT_PERCENTILE_THRESHOLD } from '../../../../common/correlations/constants'; -import type { RawResponseBase } from '../../../../common/correlations/types'; +import type { + FieldValuePair, + RawResponseBase, +} from '../../../../common/correlations/types'; import type { LatencyCorrelation, LatencyCorrelationsRawResponse, @@ -137,16 +140,38 @@ export function useLatencyCorrelations() { loaded: 10, }); - const { fieldValuePairs } = await callApmApi({ - endpoint: 'GET /internal/apm/correlations/field_value_pairs', - signal: null, - params: { - query: { - ...query, - fieldCandidates, + const chunkSize = 10; + let chunkLoadCounter = 0; + + const fieldValuePairs: FieldValuePair[] = []; + const fieldCandidateChunks = chunk(fieldCandidates, chunkSize); + + for (const fieldCandidateChunk of fieldCandidateChunks) { + const fieldValuePairChunkResponse = await callApmApi({ + endpoint: 'GET /internal/apm/correlations/field_value_pairs', + signal: null, + params: { + query: { + ...query, + fieldCandidates: fieldCandidateChunk, + }, }, - }, - }); + }); + + if (fieldValuePairChunkResponse.fieldValuePairs.length > 0) { + fieldValuePairs.push(...fieldValuePairChunkResponse.fieldValuePairs); + } + + if (isCancelledRef.current) { + return; + } + + chunkLoadCounter += chunkSize; + setFetchState({ + loaded: + 10 + Math.round((chunkLoadCounter / fieldValuePairs.length) * 10), + }); + } if (isCancelledRef.current) { return; @@ -156,11 +181,10 @@ export function useLatencyCorrelations() { loaded: 20, }); + chunkLoadCounter = 0; + const fieldsToSample = new Set(); const latencyCorrelations: LatencyCorrelation[] = []; - const chunkSize = 10; - let loadCounter = 0; - const fieldValuePairChunks = chunk(fieldValuePairs, chunkSize); for (const fieldValuePairChunk of fieldValuePairChunks) { @@ -188,9 +212,10 @@ export function useLatencyCorrelations() { return; } - loadCounter += chunkSize; + chunkLoadCounter += chunkSize; setFetchState({ - loaded: 20 + Math.round((loadCounter / fieldValuePairs.length) * 80), + loaded: + 20 + Math.round((chunkLoadCounter / fieldValuePairs.length) * 80), }); } diff --git a/x-pack/plugins/apm/server/lib/correlations/queries/query_field_value_pairs.test.ts b/x-pack/plugins/apm/server/lib/correlations/queries/query_field_value_pairs.test.ts index 64441d1421821..d893610fee31f 100644 --- a/x-pack/plugins/apm/server/lib/correlations/queries/query_field_value_pairs.test.ts +++ b/x-pack/plugins/apm/server/lib/correlations/queries/query_field_value_pairs.test.ts @@ -69,10 +69,6 @@ describe('query_field_value_pairs', () => { fieldCandidates ); - // TODO PROGRESS - // const { progress } = state.getState(); - - // expect(progress.loadedFieldValuePairs).toBe(1); expect(resp).toEqual([ { fieldName: 'myFieldCandidate1', fieldValue: 'myValue1' }, { fieldName: 'myFieldCandidate1', fieldValue: 'myValue2' }, @@ -82,8 +78,6 @@ describe('query_field_value_pairs', () => { { fieldName: 'myFieldCandidate3', fieldValue: 'myValue2' }, ]); expect(esClientSearchMock).toHaveBeenCalledTimes(3); - // TODO Log - // expect(getLogMessages()).toEqual([]); }); }); }); diff --git a/x-pack/plugins/apm/server/lib/correlations/queries/query_field_value_pairs.ts b/x-pack/plugins/apm/server/lib/correlations/queries/query_field_value_pairs.ts index 9fd73b22fd39c..a89005fc3d5ae 100644 --- a/x-pack/plugins/apm/server/lib/correlations/queries/query_field_value_pairs.ts +++ b/x-pack/plugins/apm/server/lib/correlations/queries/query_field_value_pairs.ts @@ -43,70 +43,40 @@ const fetchTransactionDurationFieldTerms = async ( params: SearchStrategyParams, fieldName: string ): Promise => { - try { - const resp = await esClient.search(getTermsAggRequest(params, fieldName)); + const resp = await esClient.search(getTermsAggRequest(params, fieldName)); - if (resp.body.aggregations === undefined) { - // TODO LOG - // if (addLogMessage) { - // addLogMessage( - // `Failed to fetch terms for field candidate ${fieldName} fieldValuePairs, no aggregations returned.`, - // JSON.stringify(resp) - // ); - // } - return []; - } - const buckets = ( - resp.body.aggregations - .attribute_terms as estypes.AggregationsMultiBucketAggregate<{ - key: string; - }> - )?.buckets; - if (buckets?.length >= 1) { - return buckets.map((d) => ({ - fieldName, - fieldValue: d.key, - })); - } - } catch (e) { - // TODO LOG - // if (addLogMessage) { - // addLogMessage( - // `Failed to fetch terms for field candidate ${fieldName} fieldValuePairs.`, - // JSON.stringify(e) - // ); - // } + if (resp.body.aggregations === undefined) { + throw new Error( + 'fetchTransactionDurationFieldTerms failed, did not return aggregations.' + ); } - return []; -}; - -async function fetchInSequence( - fieldCandidates: string[], - fn: (fieldCandidate: string) => Promise -) { - const results = []; - - for (const fieldCandidate of fieldCandidates) { - results.push(...(await fn(fieldCandidate))); + const buckets = ( + resp.body.aggregations + .attribute_terms as estypes.AggregationsMultiBucketAggregate<{ + key: string; + }> + )?.buckets; + if (buckets?.length >= 1) { + return buckets.map((d) => ({ + fieldName, + fieldValue: d.key, + })); } - return results; -} + return []; +}; export const fetchTransactionDurationFieldValuePairs = async ( esClient: ElasticsearchClient, params: SearchStrategyParams, fieldCandidates: string[] ): Promise => { - return await fetchInSequence( - fieldCandidates, - async function (fieldCandidate: string) { - return await fetchTransactionDurationFieldTerms( - esClient, - params, - fieldCandidate - ); - } + const responses = await Promise.all( + fieldCandidates.map((fieldCandidate) => + fetchTransactionDurationFieldTerms(esClient, params, fieldCandidate) + ) ); + + return responses.flat(); }; From 91e5f13612632aeb5da7e41e4f848605b2f91dc5 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Sun, 17 Oct 2021 19:10:04 +0200 Subject: [PATCH 11/39] [ML] Fix ccsWarning. --- .../latency_correlations/types.ts | 19 ++++++------------- .../queries/query_fractions.test.ts | 1 + .../queries/query_histograms_generator.ts | 17 ++++------------- .../correlations/queries/query_p_values.ts | 12 +++++------- .../queries/query_significant_correlations.ts | 16 ++++++++++++---- 5 files changed, 28 insertions(+), 37 deletions(-) diff --git a/x-pack/plugins/apm/common/correlations/latency_correlations/types.ts b/x-pack/plugins/apm/common/correlations/latency_correlations/types.ts index ea74175a3dacb..c7906319c77f4 100644 --- a/x-pack/plugins/apm/common/correlations/latency_correlations/types.ts +++ b/x-pack/plugins/apm/common/correlations/latency_correlations/types.ts @@ -14,19 +14,12 @@ export interface LatencyCorrelation extends FieldValuePair { ksTest: number; } -export interface LatencyCorrelationSearchServiceProgress { - started: number; - loadedHistogramStepsize: number; - loadedOverallHistogram: number; - loadedFieldCandidates: number; - loadedFieldValuePairs: number; - loadedHistograms: number; -} - -export interface LatencyCorrelationsParams { - percentileThreshold: number; - analyzeCorrelations: boolean; -} +export const isLatencyCorrelation = (arg: unknown): arg is LatencyCorrelation => + typeof arg === 'object' && + arg !== null && + Object.keys(arg).length === 5 && + Object.keys(arg).includes('correlation') && + !Object.keys(arg).includes('error'); export interface LatencyCorrelationsRawResponse { log: string[]; diff --git a/x-pack/plugins/apm/server/lib/correlations/queries/query_fractions.test.ts b/x-pack/plugins/apm/server/lib/correlations/queries/query_fractions.test.ts index 9c704ef7b489a..e332fb66ebda0 100644 --- a/x-pack/plugins/apm/server/lib/correlations/queries/query_fractions.test.ts +++ b/x-pack/plugins/apm/server/lib/correlations/queries/query_fractions.test.ts @@ -47,6 +47,7 @@ describe('query_fractions', () => { } => { return { body: { + hits: { total: { value: 3 } }, aggregations: { latency_ranges: { buckets: [{ doc_count: 1 }, { doc_count: 2 }], diff --git a/x-pack/plugins/apm/server/lib/correlations/queries/query_histograms_generator.ts b/x-pack/plugins/apm/server/lib/correlations/queries/query_histograms_generator.ts index 6afeb79cea9e6..644b58b6e1179 100644 --- a/x-pack/plugins/apm/server/lib/correlations/queries/query_histograms_generator.ts +++ b/x-pack/plugins/apm/server/lib/correlations/queries/query_histograms_generator.ts @@ -14,6 +14,8 @@ import type { SearchStrategyParams, } from '../../../../common/correlations/types'; +import type { LatencyCorrelation } from '../../../../common/correlations/latency_correlations/types'; + import { CORRELATION_THRESHOLD, KS_TEST_THRESHOLD } from '../constants'; import { getPrioritizedFieldValuePairs } from './get_prioritized_field_value_pairs'; @@ -65,23 +67,12 @@ export async function* fetchTransactionDurationHistograms( correlation, ksTest, histogram: logHistogram, - }; + } as LatencyCorrelation; } else { yield undefined; } } catch (e) { - // don't fail the whole process for individual correlation queries, - // just add the error to the internal log and check if we'd want to set the - // cross-cluster search compatibility warning to true. - // addLogMessage( - // `Failed to fetch correlation/kstest for '${item.fieldName}/${item.fieldValue}'`, - // JSON.stringify(e) - // ); - // TODO return CCS warning - // if (params?.index.includes(':')) { - // state.setCcsWarning(true); - // } - yield undefined; + yield { error: e }; } } } diff --git a/x-pack/plugins/apm/server/lib/correlations/queries/query_p_values.ts b/x-pack/plugins/apm/server/lib/correlations/queries/query_p_values.ts index 4cfd3fec4482b..91b3115ad2089 100644 --- a/x-pack/plugins/apm/server/lib/correlations/queries/query_p_values.ts +++ b/x-pack/plugins/apm/server/lib/correlations/queries/query_p_values.ts @@ -40,6 +40,8 @@ export const fetchPValues = async ( ) ); + let ccsWarning = false; + results.forEach((result, idx) => { if (result.status === 'fulfilled') { failedTransactionsCorrelations.push( @@ -50,14 +52,10 @@ export const fetchPValues = async ( record.pValue < ERROR_CORRELATION_THRESHOLD ) ); - } else { - // If one of the fields in the batch had an error - // addLogMessage( - // `Error getting error correlation for field ${fieldCandidates[idx]}: ${result.reason}.` - // ); + } else if (paramsWithIndex?.index.includes(':')) { + ccsWarning = true; } }); - // TODO Fix CCS warning - return { failedTransactionsCorrelations, ccsWarning: false }; + return { failedTransactionsCorrelations, ccsWarning }; }; diff --git a/x-pack/plugins/apm/server/lib/correlations/queries/query_significant_correlations.ts b/x-pack/plugins/apm/server/lib/correlations/queries/query_significant_correlations.ts index 42ae309dd3463..d44d38c59fc6e 100644 --- a/x-pack/plugins/apm/server/lib/correlations/queries/query_significant_correlations.ts +++ b/x-pack/plugins/apm/server/lib/correlations/queries/query_significant_correlations.ts @@ -13,7 +13,10 @@ import type { FieldValuePair, SearchStrategyParams, } from '../../../../common/correlations/types'; -import type { LatencyCorrelation } from '../../../../common/correlations/latency_correlations/types'; +import { + isLatencyCorrelation, + LatencyCorrelation, +} from '../../../../common/correlations/latency_correlations/types'; import { fetchTransactionDurationFractions, @@ -52,6 +55,7 @@ export const fetchSignificantCorrelations = async ( ); const latencyCorrelations: LatencyCorrelation[] = []; + let ccsWarning = false; for await (const item of fetchTransactionDurationHistograms( esClient, @@ -63,11 +67,15 @@ export const fetchSignificantCorrelations = async ( totalDocCount, fieldValuePairs )) { - if (item !== undefined) { + if (isLatencyCorrelation(item)) { latencyCorrelations.push(item); + } else if ( + {}.hasOwnProperty.call(item, 'error') && + paramsWithIndex?.index.includes(':') + ) { + ccsWarning = true; } } - // TODO Fix CCS warning - return { latencyCorrelations, ccsWarning: false, totalDocCount }; + return { latencyCorrelations, ccsWarning, totalDocCount }; }; From c1cf1e53aa5b08a0bc85f98a2c2133f8c502806f Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Sun, 17 Oct 2021 21:39:02 +0200 Subject: [PATCH 12/39] [ML] Fix jest test. --- .../queries/query_histograms_generator.test.ts | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/apm/server/lib/correlations/queries/query_histograms_generator.test.ts b/x-pack/plugins/apm/server/lib/correlations/queries/query_histograms_generator.test.ts index 4dc71a09191e0..dc5149fe0dd41 100644 --- a/x-pack/plugins/apm/server/lib/correlations/queries/query_histograms_generator.test.ts +++ b/x-pack/plugins/apm/server/lib/correlations/queries/query_histograms_generator.test.ts @@ -9,6 +9,7 @@ import type { estypes } from '@elastic/elasticsearch'; import type { ElasticsearchClient } from 'src/core/server'; import { ENVIRONMENT_ALL } from '../../../../common/environment_filter_values'; +import { isLatencyCorrelation } from '../../../../common/correlations/latency_correlations/types'; import { fetchTransactionDurationHistograms } from './query_histograms_generator'; @@ -53,6 +54,7 @@ describe('query_histograms_generator', () => { let loadedHistograms = 0; const items = []; + const errors = []; for await (const item of fetchTransactionDurationHistograms( esClientMock, @@ -64,8 +66,10 @@ describe('query_histograms_generator', () => { totalDocCount, fieldValuePairs )) { - if (item !== undefined) { + if (isLatencyCorrelation(item)) { items.push(item); + } else if (item?.error) { + errors.push(item); } loadedHistograms++; } @@ -73,11 +77,11 @@ describe('query_histograms_generator', () => { expect(items.length).toEqual(0); expect(loadedHistograms).toEqual(3); expect(esClientSearchMock).toHaveBeenCalledTimes(3); - // expect(getLogMessages().map((d) => d.split(': ')[1])).toEqual([ - // "Failed to fetch correlation/kstest for 'the-field-name-1/the-field-value-1'", - // "Failed to fetch correlation/kstest for 'the-field-name-2/the-field-value-2'", - // "Failed to fetch correlation/kstest for 'the-field-name-2/the-field-value-3'", - // ]); + expect(errors.map((e) => e.error.toString())).toEqual([ + 'Error: fetchTransactionDurationCorrelation failed, did not return aggregations.', + 'Error: fetchTransactionDurationCorrelation failed, did not return aggregations.', + 'Error: fetchTransactionDurationCorrelation failed, did not return aggregations.', + ]); }); it('returns items with correlation and ks-test value', async () => { From 72eb07def3418fae7028eabe7bd6157065e73e54 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Sun, 17 Oct 2021 21:47:15 +0200 Subject: [PATCH 13/39] [ML] Fix item check. --- .../app/correlations/correlations_log.tsx | 38 ------------------- .../failed_transactions_correlations.tsx | 11 +----- .../app/correlations/latency_correlations.tsx | 11 +----- .../queries/query_significant_correlations.ts | 2 + 4 files changed, 6 insertions(+), 56 deletions(-) delete mode 100644 x-pack/plugins/apm/public/components/app/correlations/correlations_log.tsx diff --git a/x-pack/plugins/apm/public/components/app/correlations/correlations_log.tsx b/x-pack/plugins/apm/public/components/app/correlations/correlations_log.tsx deleted file mode 100644 index 2115918a71415..0000000000000 --- a/x-pack/plugins/apm/public/components/app/correlations/correlations_log.tsx +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { EuiAccordion, EuiCode, EuiPanel } from '@elastic/eui'; -import React from 'react'; -import { i18n } from '@kbn/i18n'; -import { asAbsoluteDateTime } from '../../../../common/utils/formatters'; - -interface Props { - logMessages: string[]; -} -export function CorrelationsLog({ logMessages }: Props) { - return ( - - - {logMessages.map((logMessage, i) => { - const [timestamp, message] = logMessage.split(': '); - return ( -

- - {asAbsoluteDateTime(timestamp)} {message} - -

- ); - })} - - - ); -} diff --git a/x-pack/plugins/apm/public/components/app/correlations/failed_transactions_correlations.tsx b/x-pack/plugins/apm/public/components/app/correlations/failed_transactions_correlations.tsx index 55cb2551f2ebb..9021bad1c62bd 100644 --- a/x-pack/plugins/apm/public/components/app/correlations/failed_transactions_correlations.tsx +++ b/x-pack/plugins/apm/public/components/app/correlations/failed_transactions_correlations.tsx @@ -29,10 +29,7 @@ import type { Direction } from '@elastic/eui/src/services/sort/sort_direction'; import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n/react'; -import { - enableInspectEsQueries, - useUiTracker, -} from '../../../../../observability/public'; +import { useUiTracker } from '../../../../../observability/public'; import { asPercent } from '../../../../common/utils/formatters'; import { FailedTransactionsCorrelation } from '../../../../common/correlations/failed_transactions_correlations/types'; @@ -56,7 +53,6 @@ import { TransactionDistributionChart, TransactionDistributionChartData, } from '../../shared/charts/transaction_distribution_chart'; -import { CorrelationsLog } from './correlations_log'; import { CorrelationsEmptyStatePrompt } from './empty_state_prompt'; import { CrossClusterSearchCompatibilityWarning } from './cross_cluster_search_warning'; import { CorrelationsProgressControls } from './progress_controls'; @@ -75,12 +71,10 @@ export function FailedTransactionsCorrelations({ const transactionColors = useTransactionColors(); const { - core: { notifications, uiSettings }, + core: { notifications }, } = useApmPluginContext(); const trackApmEvent = useUiTracker({ app: 'apm' }); - const inspectEnabled = uiSettings.get(enableInspectEsQueries); - const { progress, response, startFetch, cancelFetch } = useFailedTransactionsCorrelations(); @@ -648,7 +642,6 @@ export function FailedTransactionsCorrelations({ )} {showCorrelationsEmptyStatePrompt && } - {inspectEnabled && } ); } diff --git a/x-pack/plugins/apm/public/components/app/correlations/latency_correlations.tsx b/x-pack/plugins/apm/public/components/app/correlations/latency_correlations.tsx index dad44baabac4c..53603c9158e43 100644 --- a/x-pack/plugins/apm/public/components/app/correlations/latency_correlations.tsx +++ b/x-pack/plugins/apm/public/components/app/correlations/latency_correlations.tsx @@ -25,10 +25,7 @@ import { EuiTableSortingType } from '@elastic/eui/src/components/basic_table/tab import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n/react'; -import { - enableInspectEsQueries, - useUiTracker, -} from '../../../../../observability/public'; +import { useUiTracker } from '../../../../../observability/public'; import { asPreciseDecimal } from '../../../../common/utils/formatters'; import { DEFAULT_PERCENTILE_THRESHOLD } from '../../../../common/correlations/constants'; @@ -48,7 +45,6 @@ import { CorrelationsTable } from './correlations_table'; import { LatencyCorrelationsHelpPopover } from './latency_correlations_help_popover'; import { isErrorMessage } from './utils/is_error_message'; import { getOverallHistogram } from './utils/get_overall_histogram'; -import { CorrelationsLog } from './correlations_log'; import { CorrelationsEmptyStatePrompt } from './empty_state_prompt'; import { CrossClusterSearchCompatibilityWarning } from './cross_cluster_search_warning'; import { CorrelationsProgressControls } from './progress_controls'; @@ -61,11 +57,9 @@ export function LatencyCorrelations({ onFilter }: { onFilter: () => void }) { const transactionColors = useTransactionColors(); const { - core: { notifications, uiSettings }, + core: { notifications }, } = useApmPluginContext(); - const displayLog = uiSettings.get(enableInspectEsQueries); - const { progress, response, startFetch, cancelFetch } = useLatencyCorrelations(); const progressNormalized = progress.loaded / progress.total; @@ -407,7 +401,6 @@ export function LatencyCorrelations({ onFilter }: { onFilter: () => void }) { )} {showCorrelationsEmptyStatePrompt && } - {displayLog && } ); } diff --git a/x-pack/plugins/apm/server/lib/correlations/queries/query_significant_correlations.ts b/x-pack/plugins/apm/server/lib/correlations/queries/query_significant_correlations.ts index d44d38c59fc6e..5abc29a73fae6 100644 --- a/x-pack/plugins/apm/server/lib/correlations/queries/query_significant_correlations.ts +++ b/x-pack/plugins/apm/server/lib/correlations/queries/query_significant_correlations.ts @@ -70,6 +70,8 @@ export const fetchSignificantCorrelations = async ( if (isLatencyCorrelation(item)) { latencyCorrelations.push(item); } else if ( + typeof item === 'object' && + item !== null && {}.hasOwnProperty.call(item, 'error') && paramsWithIndex?.index.includes(':') ) { From ba86ce1ea00fd0f7e1fbcf4c37fcd5e4f0dd67e9 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Mon, 18 Oct 2021 12:36:27 +0200 Subject: [PATCH 14/39] [ML] Refactor away from using generators. --- .../apm/common/correlations/constants.ts | 84 +++++++++++++++++ .../get_prioritized_field_value_pairs.test.ts | 0 .../get_prioritized_field_value_pairs.ts | 4 +- .../utils/has_prefix_to_include.test.ts | 0 .../utils/has_prefix_to_include.ts | 0 .../apm/common/correlations/utils/index.ts | 9 ++ .../correlations/use_latency_correlations.ts | 6 +- .../apm/server/lib/correlations/constants.ts | 90 ------------------- .../server/lib/correlations/queries/index.ts | 2 +- ... query_correlation_with_histogram.test.ts} | 86 ++++++++---------- .../query_correlation_with_histogram.ts | 65 ++++++++++++++ .../queries/query_field_candidates.test.ts | 2 +- .../queries/query_field_candidates.ts | 5 +- .../queries/query_field_value_pairs.ts | 3 +- .../queries/query_histograms_generator.ts | 78 ---------------- .../correlations/queries/query_p_values.ts | 47 +++++----- .../correlations/queries/query_percentiles.ts | 2 +- .../queries/query_significant_correlations.ts | 52 +++++------ .../utils/compute_expectations_and_ranges.ts | 3 +- .../server/lib/correlations/utils/index.ts | 2 +- .../utils/split_all_settled_promises.ts | 29 ++++++ 21 files changed, 291 insertions(+), 278 deletions(-) rename x-pack/plugins/apm/{server/lib/correlations/queries => common/correlations/utils}/get_prioritized_field_value_pairs.test.ts (100%) rename x-pack/plugins/apm/{server/lib/correlations/queries => common/correlations/utils}/get_prioritized_field_value_pairs.ts (88%) rename x-pack/plugins/apm/{server/lib => common}/correlations/utils/has_prefix_to_include.test.ts (100%) rename x-pack/plugins/apm/{server/lib => common}/correlations/utils/has_prefix_to_include.ts (100%) create mode 100644 x-pack/plugins/apm/common/correlations/utils/index.ts delete mode 100644 x-pack/plugins/apm/server/lib/correlations/constants.ts rename x-pack/plugins/apm/server/lib/correlations/queries/{query_histograms_generator.test.ts => query_correlation_with_histogram.test.ts} (66%) create mode 100644 x-pack/plugins/apm/server/lib/correlations/queries/query_correlation_with_histogram.ts delete mode 100644 x-pack/plugins/apm/server/lib/correlations/queries/query_histograms_generator.ts create mode 100644 x-pack/plugins/apm/server/lib/correlations/utils/split_all_settled_promises.ts diff --git a/x-pack/plugins/apm/common/correlations/constants.ts b/x-pack/plugins/apm/common/correlations/constants.ts index 4732f66e59c91..a4f5716bae04f 100644 --- a/x-pack/plugins/apm/common/correlations/constants.ts +++ b/x-pack/plugins/apm/common/correlations/constants.ts @@ -5,4 +5,88 @@ * 2.0. */ +/** + * Fields to exclude as potential field candidates + */ +export const FIELDS_TO_EXCLUDE_AS_CANDIDATE = new Set([ + // Exclude for all usage Contexts + 'parent.id', + 'trace.id', + 'transaction.id', + '@timestamp', + 'timestamp.us', + 'agent.ephemeral_id', + 'ecs.version', + 'event.ingested', + 'http.response.finished', + 'parent.id', + 'trace.id', + 'transaction.duration.us', + 'transaction.id', + 'process.pid', + 'process.ppid', + 'processor.event', + 'processor.name', + 'transaction.sampled', + 'transaction.span_count.dropped', + // Exclude for correlation on a Single Service + 'agent.name', + 'http.request.method', + 'service.framework.name', + 'service.language.name', + 'service.name', + 'service.runtime.name', + 'transaction.name', + 'transaction.type', +]); + +export const FIELD_PREFIX_TO_EXCLUDE_AS_CANDIDATE = ['observer.']; + +/** + * Fields to include/prioritize as potential field candidates + */ +export const FIELDS_TO_ADD_AS_CANDIDATE = new Set([ + 'service.version', + 'service.node.name', + 'service.framework.version', + 'service.language.version', + 'service.runtime.version', + 'kubernetes.pod.name', + 'kubernetes.pod.uid', + 'container.id', + 'source.ip', + 'client.ip', + 'host.ip', + 'service.environment', + 'process.args', + 'http.response.status_code', +]); +export const FIELD_PREFIX_TO_ADD_AS_CANDIDATE = [ + 'cloud.', + 'labels.', + 'user_agent.', +]; + +/** + * Other constants + */ +export const POPULATED_DOC_COUNT_SAMPLE_SIZE = 1000; + +export const PERCENTILES_STEP = 2; +export const TERMS_SIZE = 20; +export const SIGNIFICANT_FRACTION = 3; +export const SIGNIFICANT_VALUE_DIGITS = 3; + +export const CORRELATION_THRESHOLD = 0.3; +export const KS_TEST_THRESHOLD = 0.1; + +export const ERROR_CORRELATION_THRESHOLD = 0.02; + export const DEFAULT_PERCENTILE_THRESHOLD = 95; + +/** + * Field stats/top values sampling constants + */ + +export const SAMPLER_TOP_TERMS_THRESHOLD = 100000; +export const SAMPLER_TOP_TERMS_SHARD_SIZE = 5000; diff --git a/x-pack/plugins/apm/server/lib/correlations/queries/get_prioritized_field_value_pairs.test.ts b/x-pack/plugins/apm/common/correlations/utils/get_prioritized_field_value_pairs.test.ts similarity index 100% rename from x-pack/plugins/apm/server/lib/correlations/queries/get_prioritized_field_value_pairs.test.ts rename to x-pack/plugins/apm/common/correlations/utils/get_prioritized_field_value_pairs.test.ts diff --git a/x-pack/plugins/apm/server/lib/correlations/queries/get_prioritized_field_value_pairs.ts b/x-pack/plugins/apm/common/correlations/utils/get_prioritized_field_value_pairs.ts similarity index 88% rename from x-pack/plugins/apm/server/lib/correlations/queries/get_prioritized_field_value_pairs.ts rename to x-pack/plugins/apm/common/correlations/utils/get_prioritized_field_value_pairs.ts index 6e8fba9738730..4a0086ba02a6d 100644 --- a/x-pack/plugins/apm/server/lib/correlations/queries/get_prioritized_field_value_pairs.ts +++ b/x-pack/plugins/apm/common/correlations/utils/get_prioritized_field_value_pairs.ts @@ -6,9 +6,9 @@ */ import { FIELDS_TO_ADD_AS_CANDIDATE } from '../constants'; -import { hasPrefixToInclude } from '../utils'; +import { hasPrefixToInclude } from './has_prefix_to_include'; -import type { FieldValuePair } from '../../../../common/correlations/types'; +import type { FieldValuePair } from '../types'; export const getPrioritizedFieldValuePairs = ( fieldValuePairs: FieldValuePair[] diff --git a/x-pack/plugins/apm/server/lib/correlations/utils/has_prefix_to_include.test.ts b/x-pack/plugins/apm/common/correlations/utils/has_prefix_to_include.test.ts similarity index 100% rename from x-pack/plugins/apm/server/lib/correlations/utils/has_prefix_to_include.test.ts rename to x-pack/plugins/apm/common/correlations/utils/has_prefix_to_include.test.ts diff --git a/x-pack/plugins/apm/server/lib/correlations/utils/has_prefix_to_include.ts b/x-pack/plugins/apm/common/correlations/utils/has_prefix_to_include.ts similarity index 100% rename from x-pack/plugins/apm/server/lib/correlations/utils/has_prefix_to_include.ts rename to x-pack/plugins/apm/common/correlations/utils/has_prefix_to_include.ts diff --git a/x-pack/plugins/apm/common/correlations/utils/index.ts b/x-pack/plugins/apm/common/correlations/utils/index.ts new file mode 100644 index 0000000000000..eb83c8ae2ed01 --- /dev/null +++ b/x-pack/plugins/apm/common/correlations/utils/index.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { getPrioritizedFieldValuePairs } from './get_prioritized_field_value_pairs'; +export { hasPrefixToInclude } from './has_prefix_to_include'; diff --git a/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts b/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts index d5911f256c2f5..86332f063e106 100644 --- a/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts +++ b/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts @@ -15,6 +15,7 @@ import type { FieldValuePair, RawResponseBase, } from '../../../../common/correlations/types'; +import { getPrioritizedFieldValuePairs } from '../../../../common/correlations/utils'; import type { LatencyCorrelation, LatencyCorrelationsRawResponse, @@ -185,7 +186,10 @@ export function useLatencyCorrelations() { const fieldsToSample = new Set(); const latencyCorrelations: LatencyCorrelation[] = []; - const fieldValuePairChunks = chunk(fieldValuePairs, chunkSize); + const fieldValuePairChunks = chunk( + getPrioritizedFieldValuePairs(fieldValuePairs), + chunkSize + ); for (const fieldValuePairChunk of fieldValuePairChunks) { const significantCorrelations = await callApmApi({ diff --git a/x-pack/plugins/apm/server/lib/correlations/constants.ts b/x-pack/plugins/apm/server/lib/correlations/constants.ts deleted file mode 100644 index 5af1b21630720..0000000000000 --- a/x-pack/plugins/apm/server/lib/correlations/constants.ts +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -/** - * Fields to exclude as potential field candidates - */ -export const FIELDS_TO_EXCLUDE_AS_CANDIDATE = new Set([ - // Exclude for all usage Contexts - 'parent.id', - 'trace.id', - 'transaction.id', - '@timestamp', - 'timestamp.us', - 'agent.ephemeral_id', - 'ecs.version', - 'event.ingested', - 'http.response.finished', - 'parent.id', - 'trace.id', - 'transaction.duration.us', - 'transaction.id', - 'process.pid', - 'process.ppid', - 'processor.event', - 'processor.name', - 'transaction.sampled', - 'transaction.span_count.dropped', - // Exclude for correlation on a Single Service - 'agent.name', - 'http.request.method', - 'service.framework.name', - 'service.language.name', - 'service.name', - 'service.runtime.name', - 'transaction.name', - 'transaction.type', -]); - -export const FIELD_PREFIX_TO_EXCLUDE_AS_CANDIDATE = ['observer.']; - -/** - * Fields to include/prioritize as potential field candidates - */ -export const FIELDS_TO_ADD_AS_CANDIDATE = new Set([ - 'service.version', - 'service.node.name', - 'service.framework.version', - 'service.language.version', - 'service.runtime.version', - 'kubernetes.pod.name', - 'kubernetes.pod.uid', - 'container.id', - 'source.ip', - 'client.ip', - 'host.ip', - 'service.environment', - 'process.args', - 'http.response.status_code', -]); -export const FIELD_PREFIX_TO_ADD_AS_CANDIDATE = [ - 'cloud.', - 'labels.', - 'user_agent.', -]; - -/** - * Other constants - */ -export const POPULATED_DOC_COUNT_SAMPLE_SIZE = 1000; - -export const PERCENTILES_STEP = 2; -export const TERMS_SIZE = 20; -export const SIGNIFICANT_FRACTION = 3; -export const SIGNIFICANT_VALUE_DIGITS = 3; - -export const CORRELATION_THRESHOLD = 0.3; -export const KS_TEST_THRESHOLD = 0.1; - -export const ERROR_CORRELATION_THRESHOLD = 0.02; - -/** - * Field stats/top values sampling constants - */ - -export const SAMPLER_TOP_TERMS_THRESHOLD = 100000; -export const SAMPLER_TOP_TERMS_SHARD_SIZE = 5000; diff --git a/x-pack/plugins/apm/server/lib/correlations/queries/index.ts b/x-pack/plugins/apm/server/lib/correlations/queries/index.ts index f0c5a6a79a375..548127eb7647d 100644 --- a/x-pack/plugins/apm/server/lib/correlations/queries/index.ts +++ b/x-pack/plugins/apm/server/lib/correlations/queries/index.ts @@ -13,6 +13,6 @@ export { fetchTransactionDurationFieldValuePairs } from './query_field_value_pai export { fetchTransactionDurationFractions } from './query_fractions'; export { fetchTransactionDurationPercentiles } from './query_percentiles'; export { fetchTransactionDurationCorrelation } from './query_correlation'; -export { fetchTransactionDurationHistograms } from './query_histograms_generator'; +export { fetchTransactionDurationCorrelationWithHistogram } from './query_correlation_with_histogram'; export { fetchTransactionDurationHistogramRangeSteps } from './query_histogram_range_steps'; export { fetchTransactionDurationRanges } from './query_ranges'; diff --git a/x-pack/plugins/apm/server/lib/correlations/queries/query_histograms_generator.test.ts b/x-pack/plugins/apm/server/lib/correlations/queries/query_correlation_with_histogram.test.ts similarity index 66% rename from x-pack/plugins/apm/server/lib/correlations/queries/query_histograms_generator.test.ts rename to x-pack/plugins/apm/server/lib/correlations/queries/query_correlation_with_histogram.test.ts index dc5149fe0dd41..6e15b045a1e65 100644 --- a/x-pack/plugins/apm/server/lib/correlations/queries/query_histograms_generator.test.ts +++ b/x-pack/plugins/apm/server/lib/correlations/queries/query_correlation_with_histogram.test.ts @@ -9,9 +9,10 @@ import type { estypes } from '@elastic/elasticsearch'; import type { ElasticsearchClient } from 'src/core/server'; import { ENVIRONMENT_ALL } from '../../../../common/environment_filter_values'; -import { isLatencyCorrelation } from '../../../../common/correlations/latency_correlations/types'; -import { fetchTransactionDurationHistograms } from './query_histograms_generator'; +import { splitAllSettledPromises } from '../utils'; + +import { fetchTransactionDurationCorrelationWithHistogram } from './query_correlation_with_histogram'; const params = { index: 'apm-*', @@ -33,8 +34,8 @@ const fieldValuePairs = [ { fieldName: 'the-field-name-2', fieldValue: 'the-field-value-3' }, ]; -describe('query_histograms_generator', () => { - describe('fetchTransactionDurationHistograms', () => { +describe('query_correlation_with_histogram', () => { + describe('fetchTransactionDurationCorrelationWithHistogram', () => { it(`doesn't break on failing ES queries and adds messages to the log`, async () => { const esClientSearchMock = jest.fn( ( @@ -52,32 +53,26 @@ describe('query_histograms_generator', () => { search: esClientSearchMock, } as unknown as ElasticsearchClient; - let loadedHistograms = 0; - const items = []; - const errors = []; - - for await (const item of fetchTransactionDurationHistograms( - esClientMock, - params, - expectations, - ranges, - fractions, - histogramRangeSteps, - totalDocCount, - fieldValuePairs - )) { - if (isLatencyCorrelation(item)) { - items.push(item); - } else if (item?.error) { - errors.push(item); - } - loadedHistograms++; - } + const { fulfilled: items, rejected: errors } = splitAllSettledPromises( + await Promise.allSettled( + fieldValuePairs.map((fieldValuePair) => + fetchTransactionDurationCorrelationWithHistogram( + esClientMock, + params, + expectations, + ranges, + fractions, + histogramRangeSteps, + totalDocCount, + fieldValuePair + ) + ) + ) + ); expect(items.length).toEqual(0); - expect(loadedHistograms).toEqual(3); expect(esClientSearchMock).toHaveBeenCalledTimes(3); - expect(errors.map((e) => e.error.toString())).toEqual([ + expect(errors.map((e) => (e as Error).toString())).toEqual([ 'Error: fetchTransactionDurationCorrelation failed, did not return aggregations.', 'Error: fetchTransactionDurationCorrelation failed, did not return aggregations.', 'Error: fetchTransactionDurationCorrelation failed, did not return aggregations.', @@ -108,29 +103,26 @@ describe('query_histograms_generator', () => { search: esClientSearchMock, } as unknown as ElasticsearchClient; - let loadedHistograms = 0; - const items = []; - - for await (const item of fetchTransactionDurationHistograms( - esClientMock, - params, - expectations, - ranges, - fractions, - histogramRangeSteps, - totalDocCount, - fieldValuePairs - )) { - if (item !== undefined) { - items.push(item); - } - loadedHistograms++; - } + const { fulfilled: items, rejected: errors } = splitAllSettledPromises( + await Promise.allSettled( + fieldValuePairs.map((fieldValuePair) => + fetchTransactionDurationCorrelationWithHistogram( + esClientMock, + params, + expectations, + ranges, + fractions, + histogramRangeSteps, + totalDocCount, + fieldValuePair + ) + ) + ) + ); expect(items.length).toEqual(3); - expect(loadedHistograms).toEqual(3); expect(esClientSearchMock).toHaveBeenCalledTimes(6); - // expect(getLogMessages().length).toEqual(0); + expect(errors.length).toEqual(0); }); }); }); diff --git a/x-pack/plugins/apm/server/lib/correlations/queries/query_correlation_with_histogram.ts b/x-pack/plugins/apm/server/lib/correlations/queries/query_correlation_with_histogram.ts new file mode 100644 index 0000000000000..d031307ab6d5f --- /dev/null +++ b/x-pack/plugins/apm/server/lib/correlations/queries/query_correlation_with_histogram.ts @@ -0,0 +1,65 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { estypes } from '@elastic/elasticsearch'; + +import type { ElasticsearchClient } from 'src/core/server'; + +import type { + FieldValuePair, + SearchStrategyParams, +} from '../../../../common/correlations/types'; + +import type { LatencyCorrelation } from '../../../../common/correlations/latency_correlations/types'; +import { + CORRELATION_THRESHOLD, + KS_TEST_THRESHOLD, +} from '../../../../common/correlations/constants'; + +import { fetchTransactionDurationCorrelation } from './query_correlation'; +import { fetchTransactionDurationRanges } from './query_ranges'; + +export async function fetchTransactionDurationCorrelationWithHistogram( + esClient: ElasticsearchClient, + params: SearchStrategyParams, + expectations: number[], + ranges: estypes.AggregationsAggregationRange[], + fractions: number[], + histogramRangeSteps: number[], + totalDocCount: number, + fieldValuePair: FieldValuePair +): Promise { + const { correlation, ksTest } = await fetchTransactionDurationCorrelation( + esClient, + params, + expectations, + ranges, + fractions, + totalDocCount, + [fieldValuePair] + ); + + if ( + correlation !== null && + correlation > CORRELATION_THRESHOLD && + ksTest !== null && + ksTest < KS_TEST_THRESHOLD + ) { + const logHistogram = await fetchTransactionDurationRanges( + esClient, + params, + histogramRangeSteps, + [fieldValuePair] + ); + return { + ...fieldValuePair, + correlation, + ksTest, + histogram: logHistogram, + }; + } +} diff --git a/x-pack/plugins/apm/server/lib/correlations/queries/query_field_candidates.test.ts b/x-pack/plugins/apm/server/lib/correlations/queries/query_field_candidates.test.ts index bae42666e6db0..7124506a4000f 100644 --- a/x-pack/plugins/apm/server/lib/correlations/queries/query_field_candidates.test.ts +++ b/x-pack/plugins/apm/server/lib/correlations/queries/query_field_candidates.test.ts @@ -10,7 +10,7 @@ import type { estypes } from '@elastic/elasticsearch'; import type { ElasticsearchClient } from 'src/core/server'; import { ENVIRONMENT_ALL } from '../../../../common/environment_filter_values'; -import { hasPrefixToInclude } from '../utils'; +import { hasPrefixToInclude } from '../../../../common/correlations/utils'; import { fetchTransactionDurationFieldCandidates, diff --git a/x-pack/plugins/apm/server/lib/correlations/queries/query_field_candidates.ts b/x-pack/plugins/apm/server/lib/correlations/queries/query_field_candidates.ts index 676a2f82a9185..90bd6b867ab09 100644 --- a/x-pack/plugins/apm/server/lib/correlations/queries/query_field_candidates.ts +++ b/x-pack/plugins/apm/server/lib/correlations/queries/query_field_candidates.ts @@ -10,14 +10,13 @@ import type { estypes } from '@elastic/elasticsearch'; import type { ElasticsearchClient } from 'src/core/server'; import type { SearchStrategyParams } from '../../../../common/correlations/types'; - import { FIELD_PREFIX_TO_EXCLUDE_AS_CANDIDATE, FIELDS_TO_ADD_AS_CANDIDATE, FIELDS_TO_EXCLUDE_AS_CANDIDATE, POPULATED_DOC_COUNT_SAMPLE_SIZE, -} from '../constants'; -import { hasPrefixToInclude } from '../utils'; +} from '../../../../common/correlations/constants'; +import { hasPrefixToInclude } from '../../../../common/correlations/utils'; import { getQueryWithParams } from './get_query_with_params'; import { getRequestBase } from './get_request_base'; diff --git a/x-pack/plugins/apm/server/lib/correlations/queries/query_field_value_pairs.ts b/x-pack/plugins/apm/server/lib/correlations/queries/query_field_value_pairs.ts index a89005fc3d5ae..9e5cd26605a9d 100644 --- a/x-pack/plugins/apm/server/lib/correlations/queries/query_field_value_pairs.ts +++ b/x-pack/plugins/apm/server/lib/correlations/queries/query_field_value_pairs.ts @@ -13,8 +13,7 @@ import type { FieldValuePair, SearchStrategyParams, } from '../../../../common/correlations/types'; - -import { TERMS_SIZE } from '../constants'; +import { TERMS_SIZE } from '../../../../common/correlations/constants'; import { getQueryWithParams } from './get_query_with_params'; import { getRequestBase } from './get_request_base'; diff --git a/x-pack/plugins/apm/server/lib/correlations/queries/query_histograms_generator.ts b/x-pack/plugins/apm/server/lib/correlations/queries/query_histograms_generator.ts deleted file mode 100644 index 644b58b6e1179..0000000000000 --- a/x-pack/plugins/apm/server/lib/correlations/queries/query_histograms_generator.ts +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { estypes } from '@elastic/elasticsearch'; - -import type { ElasticsearchClient } from 'src/core/server'; - -import type { - FieldValuePair, - SearchStrategyParams, -} from '../../../../common/correlations/types'; - -import type { LatencyCorrelation } from '../../../../common/correlations/latency_correlations/types'; - -import { CORRELATION_THRESHOLD, KS_TEST_THRESHOLD } from '../constants'; - -import { getPrioritizedFieldValuePairs } from './get_prioritized_field_value_pairs'; -import { fetchTransactionDurationCorrelation } from './query_correlation'; -import { fetchTransactionDurationRanges } from './query_ranges'; - -export async function* fetchTransactionDurationHistograms( - esClient: ElasticsearchClient, - params: SearchStrategyParams, - expectations: number[], - ranges: estypes.AggregationsAggregationRange[], - fractions: number[], - histogramRangeSteps: number[], - totalDocCount: number, - fieldValuePairs: FieldValuePair[] -) { - for (const item of getPrioritizedFieldValuePairs(fieldValuePairs)) { - if (params === undefined || item === undefined) { - return; - } - - // If one of the fields have an error - // We don't want to stop the whole process - try { - const { correlation, ksTest } = await fetchTransactionDurationCorrelation( - esClient, - params, - expectations, - ranges, - fractions, - totalDocCount, - [item] - ); - - if ( - correlation !== null && - correlation > CORRELATION_THRESHOLD && - ksTest !== null && - ksTest < KS_TEST_THRESHOLD - ) { - const logHistogram = await fetchTransactionDurationRanges( - esClient, - params, - histogramRangeSteps, - [item] - ); - yield { - ...item, - correlation, - ksTest, - histogram: logHistogram, - } as LatencyCorrelation; - } else { - yield undefined; - } - } catch (e) { - yield { error: e }; - } - } -} diff --git a/x-pack/plugins/apm/server/lib/correlations/queries/query_p_values.ts b/x-pack/plugins/apm/server/lib/correlations/queries/query_p_values.ts index 91b3115ad2089..df7e8dd8932a8 100644 --- a/x-pack/plugins/apm/server/lib/correlations/queries/query_p_values.ts +++ b/x-pack/plugins/apm/server/lib/correlations/queries/query_p_values.ts @@ -9,8 +9,9 @@ import type { ElasticsearchClient } from 'src/core/server'; import type { SearchStrategyParams } from '../../../../common/correlations/types'; import type { FailedTransactionsCorrelation } from '../../../../common/correlations/failed_transactions_correlations/types'; +import { ERROR_CORRELATION_THRESHOLD } from '../../../../common/correlations/constants'; -import { ERROR_CORRELATION_THRESHOLD } from '../constants'; +import { splitAllSettledPromises } from '../utils'; import { fetchFailedTransactionsCorrelationPValues, @@ -22,40 +23,36 @@ export const fetchPValues = async ( paramsWithIndex: SearchStrategyParams, fieldCandidates: string[] ) => { - const failedTransactionsCorrelations: FailedTransactionsCorrelation[] = []; - const histogramRangeSteps = await fetchTransactionDurationHistogramRangeSteps( esClient, paramsWithIndex ); - const results = await Promise.allSettled( - fieldCandidates.map((fieldName) => - fetchFailedTransactionsCorrelationPValues( - esClient, - paramsWithIndex, - histogramRangeSteps, - fieldName + const { fulfilled, rejected } = splitAllSettledPromises( + await Promise.allSettled( + fieldCandidates.map((fieldName) => + fetchFailedTransactionsCorrelationPValues( + esClient, + paramsWithIndex, + histogramRangeSteps, + fieldName + ) ) ) ); - let ccsWarning = false; - - results.forEach((result, idx) => { - if (result.status === 'fulfilled') { - failedTransactionsCorrelations.push( - ...result.value.filter( - (record) => - record && - typeof record.pValue === 'number' && - record.pValue < ERROR_CORRELATION_THRESHOLD - ) + const failedTransactionsCorrelations: FailedTransactionsCorrelation[] = + fulfilled + .flat() + .filter( + (record) => + record && + typeof record.pValue === 'number' && + record.pValue < ERROR_CORRELATION_THRESHOLD ); - } else if (paramsWithIndex?.index.includes(':')) { - ccsWarning = true; - } - }); + + const ccsWarning = + rejected.length > 0 && paramsWithIndex?.index.includes(':'); return { failedTransactionsCorrelations, ccsWarning }; }; diff --git a/x-pack/plugins/apm/server/lib/correlations/queries/query_percentiles.ts b/x-pack/plugins/apm/server/lib/correlations/queries/query_percentiles.ts index da546315e812d..8d444b6736e11 100644 --- a/x-pack/plugins/apm/server/lib/correlations/queries/query_percentiles.ts +++ b/x-pack/plugins/apm/server/lib/correlations/queries/query_percentiles.ts @@ -10,6 +10,7 @@ import type { estypes } from '@elastic/elasticsearch'; import type { ElasticsearchClient } from 'src/core/server'; import { TRANSACTION_DURATION } from '../../../../common/elasticsearch_fieldnames'; +import { SIGNIFICANT_VALUE_DIGITS } from '../../../../common/correlations/constants'; import type { FieldValuePair, ResponseHit, @@ -18,7 +19,6 @@ import type { import { getQueryWithParams } from './get_query_with_params'; import { getRequestBase } from './get_request_base'; -import { SIGNIFICANT_VALUE_DIGITS } from '../constants'; export const getTransactionDurationPercentilesRequest = ( params: SearchStrategyParams, diff --git a/x-pack/plugins/apm/server/lib/correlations/queries/query_significant_correlations.ts b/x-pack/plugins/apm/server/lib/correlations/queries/query_significant_correlations.ts index 5abc29a73fae6..e35cc48722c53 100644 --- a/x-pack/plugins/apm/server/lib/correlations/queries/query_significant_correlations.ts +++ b/x-pack/plugins/apm/server/lib/correlations/queries/query_significant_correlations.ts @@ -19,12 +19,16 @@ import { } from '../../../../common/correlations/latency_correlations/types'; import { + computeExpectationsAndRanges, + splitAllSettledPromises, +} from '../utils'; + +import { + fetchTransactionDurationCorrelationWithHistogram, fetchTransactionDurationFractions, fetchTransactionDurationHistogramRangeSteps, - fetchTransactionDurationHistograms, fetchTransactionDurationPercentiles, } from './index'; -import { computeExpectationsAndRanges } from '../utils'; export const fetchSignificantCorrelations = async ( esClient: ElasticsearchClient, @@ -54,30 +58,28 @@ export const fetchSignificantCorrelations = async ( paramsWithIndex ); - const latencyCorrelations: LatencyCorrelation[] = []; - let ccsWarning = false; + const { fulfilled, rejected } = splitAllSettledPromises( + await Promise.allSettled( + fieldValuePairs.map((fieldValuePair) => + fetchTransactionDurationCorrelationWithHistogram( + esClient, + paramsWithIndex, + expectations, + ranges, + fractions, + histogramRangeSteps, + totalDocCount, + fieldValuePair + ) + ) + ) + ); + + const latencyCorrelations: LatencyCorrelation[] = + fulfilled.filter(isLatencyCorrelation); - for await (const item of fetchTransactionDurationHistograms( - esClient, - paramsWithIndex, - expectations, - ranges, - fractions, - histogramRangeSteps, - totalDocCount, - fieldValuePairs - )) { - if (isLatencyCorrelation(item)) { - latencyCorrelations.push(item); - } else if ( - typeof item === 'object' && - item !== null && - {}.hasOwnProperty.call(item, 'error') && - paramsWithIndex?.index.includes(':') - ) { - ccsWarning = true; - } - } + const ccsWarning = + rejected.length > 0 && paramsWithIndex?.index.includes(':'); return { latencyCorrelations, ccsWarning, totalDocCount }; }; diff --git a/x-pack/plugins/apm/server/lib/correlations/utils/compute_expectations_and_ranges.ts b/x-pack/plugins/apm/server/lib/correlations/utils/compute_expectations_and_ranges.ts index 9683af1fa341f..61b054a7835b3 100644 --- a/x-pack/plugins/apm/server/lib/correlations/utils/compute_expectations_and_ranges.ts +++ b/x-pack/plugins/apm/server/lib/correlations/utils/compute_expectations_and_ranges.ts @@ -6,7 +6,8 @@ */ import { estypes } from '@elastic/elasticsearch'; -import { PERCENTILES_STEP } from '../constants'; + +import { PERCENTILES_STEP } from '../../../../common/correlations/constants'; export const computeExpectationsAndRanges = ( percentiles: number[], diff --git a/x-pack/plugins/apm/server/lib/correlations/utils/index.ts b/x-pack/plugins/apm/server/lib/correlations/utils/index.ts index 727bc6cd787a0..f7c5abef939b9 100644 --- a/x-pack/plugins/apm/server/lib/correlations/utils/index.ts +++ b/x-pack/plugins/apm/server/lib/correlations/utils/index.ts @@ -6,4 +6,4 @@ */ export { computeExpectationsAndRanges } from './compute_expectations_and_ranges'; -export { hasPrefixToInclude } from './has_prefix_to_include'; +export { splitAllSettledPromises } from './split_all_settled_promises'; diff --git a/x-pack/plugins/apm/server/lib/correlations/utils/split_all_settled_promises.ts b/x-pack/plugins/apm/server/lib/correlations/utils/split_all_settled_promises.ts new file mode 100644 index 0000000000000..4e060477f024f --- /dev/null +++ b/x-pack/plugins/apm/server/lib/correlations/utils/split_all_settled_promises.ts @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +interface HandledPromises { + fulfilled: T[]; + rejected: unknown[]; +} + +export const splitAllSettledPromises = ( + promises: Array> +): HandledPromises => + promises.reduce( + (result, current) => { + if (current.status === 'fulfilled') { + result.fulfilled.push(current.value as T); + } else if (current.status === 'rejected') { + result.rejected.push(current.reason); + } + return result; + }, + { + fulfilled: [], + rejected: [], + } as HandledPromises + ); From 3368abad68e575c56cd18938ec81e9ac7a40fe33 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Tue, 19 Oct 2021 18:31:41 +0200 Subject: [PATCH 15/39] [ML] Remove references to log. --- .../correlations/failed_transactions_correlations/types.ts | 1 - .../apm/common/correlations/latency_correlations/types.ts | 1 - .../components/app/correlations/latency_correlations.test.tsx | 2 -- .../apm/public/hooks/use_latency_overall_distribution.ts | 2 +- .../server/lib/latency/get_overall_latency_distribution.ts | 4 +--- x-pack/plugins/apm/server/lib/latency/types.ts | 1 - 6 files changed, 2 insertions(+), 9 deletions(-) diff --git a/x-pack/plugins/apm/common/correlations/failed_transactions_correlations/types.ts b/x-pack/plugins/apm/common/correlations/failed_transactions_correlations/types.ts index 94081f8b9d2b6..ac344f67536d4 100644 --- a/x-pack/plugins/apm/common/correlations/failed_transactions_correlations/types.ts +++ b/x-pack/plugins/apm/common/correlations/failed_transactions_correlations/types.ts @@ -25,7 +25,6 @@ export type FailedTransactionsCorrelationsImpactThreshold = typeof FAILED_TRANSACTIONS_IMPACT_THRESHOLD[keyof typeof FAILED_TRANSACTIONS_IMPACT_THRESHOLD]; export interface FailedTransactionsCorrelationsRawResponse { - log: string[]; failedTransactionsCorrelations?: FailedTransactionsCorrelation[]; percentileThresholdValue?: number; overallHistogram?: HistogramItem[]; diff --git a/x-pack/plugins/apm/common/correlations/latency_correlations/types.ts b/x-pack/plugins/apm/common/correlations/latency_correlations/types.ts index c7906319c77f4..c36e1825e2c02 100644 --- a/x-pack/plugins/apm/common/correlations/latency_correlations/types.ts +++ b/x-pack/plugins/apm/common/correlations/latency_correlations/types.ts @@ -22,7 +22,6 @@ export const isLatencyCorrelation = (arg: unknown): arg is LatencyCorrelation => !Object.keys(arg).includes('error'); export interface LatencyCorrelationsRawResponse { - log: string[]; overallHistogram?: HistogramItem[]; percentileThresholdValue?: number; latencyCorrelations?: LatencyCorrelation[]; diff --git a/x-pack/plugins/apm/public/components/app/correlations/latency_correlations.test.tsx b/x-pack/plugins/apm/public/components/app/correlations/latency_correlations.test.tsx index cafbacae6babd..8fd257b7fc074 100644 --- a/x-pack/plugins/apm/public/components/app/correlations/latency_correlations.test.tsx +++ b/x-pack/plugins/apm/public/components/app/correlations/latency_correlations.test.tsx @@ -100,7 +100,6 @@ describe('correlations', () => { rawResponse: { ccsWarning: false, latencyCorrelations: [], - log: [], }, }} > @@ -122,7 +121,6 @@ describe('correlations', () => { rawResponse: { ccsWarning: false, latencyCorrelations: [], - log: [], }, }} > diff --git a/x-pack/plugins/apm/public/hooks/use_latency_overall_distribution.ts b/x-pack/plugins/apm/public/hooks/use_latency_overall_distribution.ts index 2669179d91991..3a9db0f7b75cb 100644 --- a/x-pack/plugins/apm/public/hooks/use_latency_overall_distribution.ts +++ b/x-pack/plugins/apm/public/hooks/use_latency_overall_distribution.ts @@ -27,7 +27,7 @@ export function useLatencyOverallDistribution() { const { start, end } = useTimeRange({ rangeFrom, rangeTo }); const { - data = { log: [] }, + data = {}, status, error, } = useFetcher( diff --git a/x-pack/plugins/apm/server/lib/latency/get_overall_latency_distribution.ts b/x-pack/plugins/apm/server/lib/latency/get_overall_latency_distribution.ts index aec946fb8c1df..07f470393e4c6 100644 --- a/x-pack/plugins/apm/server/lib/latency/get_overall_latency_distribution.ts +++ b/x-pack/plugins/apm/server/lib/latency/get_overall_latency_distribution.ts @@ -27,9 +27,7 @@ export async function getOverallLatencyDistribution( options: OverallLatencyDistributionOptions ) { return withApmSpan('get_overall_latency_distribution', async () => { - const overallLatencyDistribution: OverallLatencyDistributionResponse = { - log: [], - }; + const overallLatencyDistribution: OverallLatencyDistributionResponse = {}; const { setup, termFilters, ...rawParams } = options; const { apmEventClient } = setup; diff --git a/x-pack/plugins/apm/server/lib/latency/types.ts b/x-pack/plugins/apm/server/lib/latency/types.ts index ba735db2acebb..0179132949f9d 100644 --- a/x-pack/plugins/apm/server/lib/latency/types.ts +++ b/x-pack/plugins/apm/server/lib/latency/types.ts @@ -20,7 +20,6 @@ export interface OverallLatencyDistributionOptions } export interface OverallLatencyDistributionResponse { - log: string[]; percentileThresholdValue?: number; overallHistogram?: Array<{ key: number; From 430aa1235d021b4129dd32741d3bb34d6355260f Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Tue, 19 Oct 2021 18:46:46 +0200 Subject: [PATCH 16/39] [ML] Get rid of SearchStrategy references in type/var names. --- .../failed_transactions_correlations/types.ts | 1 + .../common/correlations/field_stats_types.ts | 4 ++-- .../latency_correlations/types.ts | 1 + .../plugins/apm/common/correlations/types.ts | 22 ++++--------------- .../latency_correlations.test.tsx | 5 +---- .../use_failed_transactions_correlations.ts | 9 ++++---- .../correlations/use_latency_correlations.ts | 13 +++++------ .../queries/field_stats/get_fields_stats.ts | 4 ++-- .../queries/get_query_with_params.ts | 4 ++-- .../correlations/queries/get_request_base.ts | 4 ++-- .../correlations/queries/query_correlation.ts | 6 ++--- .../query_correlation_with_histogram.ts | 4 ++-- .../queries/query_failure_correlation.ts | 6 ++--- .../queries/query_field_candidates.ts | 6 ++--- .../queries/query_field_value_pairs.ts | 8 +++---- .../correlations/queries/query_fractions.ts | 6 ++--- .../correlations/queries/query_histogram.ts | 6 ++--- .../queries/query_histogram_range_steps.ts | 6 ++--- .../correlations/queries/query_p_values.ts | 4 ++-- .../correlations/queries/query_percentiles.ts | 6 ++--- .../lib/correlations/queries/query_ranges.ts | 6 ++--- .../queries/query_significant_correlations.ts | 4 ++-- 22 files changed, 58 insertions(+), 77 deletions(-) diff --git a/x-pack/plugins/apm/common/correlations/failed_transactions_correlations/types.ts b/x-pack/plugins/apm/common/correlations/failed_transactions_correlations/types.ts index ac344f67536d4..9e567f599c1c9 100644 --- a/x-pack/plugins/apm/common/correlations/failed_transactions_correlations/types.ts +++ b/x-pack/plugins/apm/common/correlations/failed_transactions_correlations/types.ts @@ -25,6 +25,7 @@ export type FailedTransactionsCorrelationsImpactThreshold = typeof FAILED_TRANSACTIONS_IMPACT_THRESHOLD[keyof typeof FAILED_TRANSACTIONS_IMPACT_THRESHOLD]; export interface FailedTransactionsCorrelationsRawResponse { + ccsWarning: boolean; failedTransactionsCorrelations?: FailedTransactionsCorrelation[]; percentileThresholdValue?: number; overallHistogram?: HistogramItem[]; diff --git a/x-pack/plugins/apm/common/correlations/field_stats_types.ts b/x-pack/plugins/apm/common/correlations/field_stats_types.ts index d96bb4408f0e8..ed8a4eac44d7c 100644 --- a/x-pack/plugins/apm/common/correlations/field_stats_types.ts +++ b/x-pack/plugins/apm/common/correlations/field_stats_types.ts @@ -6,9 +6,9 @@ */ import { estypes } from '@elastic/elasticsearch'; -import { SearchStrategyParams } from './types'; +import { CorrelationsParams } from './types'; -export interface FieldStatsCommonRequestParams extends SearchStrategyParams { +export interface FieldStatsCommonRequestParams extends CorrelationsParams { samplerShardSize: number; } diff --git a/x-pack/plugins/apm/common/correlations/latency_correlations/types.ts b/x-pack/plugins/apm/common/correlations/latency_correlations/types.ts index c36e1825e2c02..bc90d955984a2 100644 --- a/x-pack/plugins/apm/common/correlations/latency_correlations/types.ts +++ b/x-pack/plugins/apm/common/correlations/latency_correlations/types.ts @@ -22,6 +22,7 @@ export const isLatencyCorrelation = (arg: unknown): arg is LatencyCorrelation => !Object.keys(arg).includes('error'); export interface LatencyCorrelationsRawResponse { + ccsWarning: boolean; overallHistogram?: HistogramItem[]; percentileThresholdValue?: number; latencyCorrelations?: LatencyCorrelation[]; diff --git a/x-pack/plugins/apm/common/correlations/types.ts b/x-pack/plugins/apm/common/correlations/types.ts index 3321826c49e46..402750b72b2ab 100644 --- a/x-pack/plugins/apm/common/correlations/types.ts +++ b/x-pack/plugins/apm/common/correlations/types.ts @@ -26,34 +26,20 @@ export interface ResponseHit { _source: ResponseHitSource; } -export interface RawResponseBase { - ccsWarning: boolean; -} - -export interface SearchStrategyClientParamsBase { +export interface CorrelationsClientParams { environment: string; kuery: string; serviceName?: string; transactionName?: string; transactionType?: string; -} - -export interface RawSearchStrategyClientParams - extends SearchStrategyClientParamsBase { - start?: string; - end?: string; -} - -export interface SearchStrategyClientParams - extends SearchStrategyClientParamsBase { start: number; end: number; } -export interface SearchStrategyServerParams { +export interface CorrelationsServerParams { index: string; includeFrozen?: boolean; } -export type SearchStrategyParams = SearchStrategyClientParams & - SearchStrategyServerParams; +export type CorrelationsParams = CorrelationsClientParams & + CorrelationsServerParams; diff --git a/x-pack/plugins/apm/public/components/app/correlations/latency_correlations.test.tsx b/x-pack/plugins/apm/public/components/app/correlations/latency_correlations.test.tsx index 8fd257b7fc074..d1a2ebf269d0a 100644 --- a/x-pack/plugins/apm/public/components/app/correlations/latency_correlations.test.tsx +++ b/x-pack/plugins/apm/public/components/app/correlations/latency_correlations.test.tsx @@ -19,7 +19,6 @@ import type { IKibanaSearchResponse } from 'src/plugins/data/public'; import { EuiThemeProvider } from 'src/plugins/kibana_react/common'; import { createKibanaReactContext } from 'src/plugins/kibana_react/public'; import type { LatencyCorrelationsRawResponse } from '../../../../common/correlations/latency_correlations/types'; -import type { RawResponseBase } from '../../../../common/correlations/types'; import { MockUrlParamsContextProvider } from '../../../context/url_params_context/mock_url_params_context_provider'; import { ApmPluginContextValue } from '../../../context/apm_plugin/apm_plugin_context'; import { @@ -35,9 +34,7 @@ function Wrapper({ dataSearchResponse, }: { children?: ReactNode; - dataSearchResponse: IKibanaSearchResponse< - LatencyCorrelationsRawResponse & RawResponseBase - >; + dataSearchResponse: IKibanaSearchResponse; }) { const mockDataSearch = jest.fn(() => of(dataSearchResponse)); diff --git a/x-pack/plugins/apm/public/components/app/correlations/use_failed_transactions_correlations.ts b/x-pack/plugins/apm/public/components/app/correlations/use_failed_transactions_correlations.ts index 9a1a0d4ef43c2..26bbef393b63c 100644 --- a/x-pack/plugins/apm/public/components/app/correlations/use_failed_transactions_correlations.ts +++ b/x-pack/plugins/apm/public/components/app/correlations/use_failed_transactions_correlations.ts @@ -13,7 +13,6 @@ import { IHttpFetchError } from 'src/core/public'; import { EVENT_OUTCOME } from '../../../../common/elasticsearch_fieldnames'; import { EventOutcome } from '../../../../common/event_outcome'; import { DEFAULT_PERCENTILE_THRESHOLD } from '../../../../common/correlations/constants'; -import type { RawResponseBase } from '../../../../common/correlations/types'; import type { FailedTransactionsCorrelation, FailedTransactionsCorrelationsRawResponse, @@ -26,9 +25,9 @@ import { useApmParams } from '../../../hooks/use_apm_params'; import { useTimeRange } from '../../../hooks/use_time_range'; import { callApmApi } from '../../../services/rest/createCallApmApi'; -type Response = FailedTransactionsCorrelationsRawResponse & RawResponseBase; +type Response = FailedTransactionsCorrelationsRawResponse; -interface SearchStrategyProgress { +interface CorrelationsProgress { error?: Error | IHttpFetchError; isRunning: boolean; loaded: number; @@ -46,7 +45,7 @@ const getInitialRawResponse = (): Response => ccsWarning: false, } as Response); -const getInitialProgress = (): SearchStrategyProgress => ({ +const getInitialProgress = (): CorrelationsProgress => ({ isRunning: false, loaded: 0, total: 100, @@ -77,7 +76,7 @@ export function useFailedTransactionsCorrelations() { ); const [fetchState, setFetchState] = useReducer( - getReducer(), + getReducer(), getInitialProgress() ); diff --git a/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts b/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts index 86332f063e106..0d5edd75c0d02 100644 --- a/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts +++ b/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts @@ -11,10 +11,7 @@ import { chunk } from 'lodash'; import { IHttpFetchError } from 'src/core/public'; import { DEFAULT_PERCENTILE_THRESHOLD } from '../../../../common/correlations/constants'; -import type { - FieldValuePair, - RawResponseBase, -} from '../../../../common/correlations/types'; +import type { FieldValuePair } from '../../../../common/correlations/types'; import { getPrioritizedFieldValuePairs } from '../../../../common/correlations/utils'; import type { LatencyCorrelation, @@ -28,9 +25,9 @@ import { useApmParams } from '../../../hooks/use_apm_params'; import { useTimeRange } from '../../../hooks/use_time_range'; import { callApmApi } from '../../../services/rest/createCallApmApi'; -type Response = LatencyCorrelationsRawResponse & RawResponseBase; +type Response = LatencyCorrelationsRawResponse; -interface SearchStrategyProgress { +interface CorrelationsProgress { error?: Error | IHttpFetchError; isRunning: boolean; loaded: number; @@ -48,7 +45,7 @@ const getInitialRawResponse = (): Response => ccsWarning: false, } as Response); -const getInitialProgress = (): SearchStrategyProgress => ({ +const getInitialProgress = (): CorrelationsProgress => ({ isRunning: false, loaded: 0, total: 100, @@ -79,7 +76,7 @@ export function useLatencyCorrelations() { ); const [fetchState, setFetchState] = useReducer( - getReducer(), + getReducer(), getInitialProgress() ); diff --git a/x-pack/plugins/apm/server/lib/correlations/queries/field_stats/get_fields_stats.ts b/x-pack/plugins/apm/server/lib/correlations/queries/field_stats/get_fields_stats.ts index a294c4588d0cb..8b41f7662679c 100644 --- a/x-pack/plugins/apm/server/lib/correlations/queries/field_stats/get_fields_stats.ts +++ b/x-pack/plugins/apm/server/lib/correlations/queries/field_stats/get_fields_stats.ts @@ -10,7 +10,7 @@ import { chunk } from 'lodash'; import { ES_FIELD_TYPES } from '@kbn/field-types'; import { FieldValuePair, - SearchStrategyParams, + CorrelationsParams, } from '../../../../../common/correlations/types'; import { FieldStats, @@ -23,7 +23,7 @@ import { fetchBooleanFieldStats } from './get_boolean_field_stats'; export const fetchFieldsStats = async ( esClient: ElasticsearchClient, - params: SearchStrategyParams, + params: CorrelationsParams, fieldsToSample: string[], termFilters?: FieldValuePair[] ): Promise<{ stats: FieldStats[]; errors: any[] }> => { diff --git a/x-pack/plugins/apm/server/lib/correlations/queries/get_query_with_params.ts b/x-pack/plugins/apm/server/lib/correlations/queries/get_query_with_params.ts index 22b8ce2353747..9d3d6d283de43 100644 --- a/x-pack/plugins/apm/server/lib/correlations/queries/get_query_with_params.ts +++ b/x-pack/plugins/apm/server/lib/correlations/queries/get_query_with_params.ts @@ -8,7 +8,7 @@ import type { estypes } from '@elastic/elasticsearch'; import type { FieldValuePair, - SearchStrategyParams, + CorrelationsParams, } from '../../../../common/correlations/types'; import { getCorrelationsFilters } from './get_filters'; @@ -17,7 +17,7 @@ export const getTermsQuery = ({ fieldName, fieldValue }: FieldValuePair) => { }; interface QueryParams { - params: SearchStrategyParams; + params: CorrelationsParams; termFilters?: FieldValuePair[]; } export const getQueryWithParams = ({ params, termFilters }: QueryParams) => { diff --git a/x-pack/plugins/apm/server/lib/correlations/queries/get_request_base.ts b/x-pack/plugins/apm/server/lib/correlations/queries/get_request_base.ts index 1323a2c088404..5ab4e3b26122d 100644 --- a/x-pack/plugins/apm/server/lib/correlations/queries/get_request_base.ts +++ b/x-pack/plugins/apm/server/lib/correlations/queries/get_request_base.ts @@ -5,12 +5,12 @@ * 2.0. */ -import type { SearchStrategyParams } from '../../../../common/correlations/types'; +import type { CorrelationsParams } from '../../../../common/correlations/types'; export const getRequestBase = ({ index, includeFrozen, -}: SearchStrategyParams) => ({ +}: CorrelationsParams) => ({ index, // matches APM's event client settings ignore_throttled: includeFrozen === undefined ? true : !includeFrozen, diff --git a/x-pack/plugins/apm/server/lib/correlations/queries/query_correlation.ts b/x-pack/plugins/apm/server/lib/correlations/queries/query_correlation.ts index 9ada1b47e574d..ba7d536b15fc0 100644 --- a/x-pack/plugins/apm/server/lib/correlations/queries/query_correlation.ts +++ b/x-pack/plugins/apm/server/lib/correlations/queries/query_correlation.ts @@ -13,7 +13,7 @@ import { TRANSACTION_DURATION } from '../../../../common/elasticsearch_fieldname import type { FieldValuePair, ResponseHit, - SearchStrategyParams, + CorrelationsParams, } from '../../../../common/correlations/types'; import { getQueryWithParams } from './get_query_with_params'; @@ -33,7 +33,7 @@ export interface BucketCorrelation { } export const getTransactionDurationCorrelationRequest = ( - params: SearchStrategyParams, + params: CorrelationsParams, expectations: number[], ranges: estypes.AggregationsAggregationRange[], fractions: number[], @@ -87,7 +87,7 @@ export const getTransactionDurationCorrelationRequest = ( export const fetchTransactionDurationCorrelation = async ( esClient: ElasticsearchClient, - params: SearchStrategyParams, + params: CorrelationsParams, expectations: number[], ranges: estypes.AggregationsAggregationRange[], fractions: number[], diff --git a/x-pack/plugins/apm/server/lib/correlations/queries/query_correlation_with_histogram.ts b/x-pack/plugins/apm/server/lib/correlations/queries/query_correlation_with_histogram.ts index d031307ab6d5f..217252081e52f 100644 --- a/x-pack/plugins/apm/server/lib/correlations/queries/query_correlation_with_histogram.ts +++ b/x-pack/plugins/apm/server/lib/correlations/queries/query_correlation_with_histogram.ts @@ -11,7 +11,7 @@ import type { ElasticsearchClient } from 'src/core/server'; import type { FieldValuePair, - SearchStrategyParams, + CorrelationsParams, } from '../../../../common/correlations/types'; import type { LatencyCorrelation } from '../../../../common/correlations/latency_correlations/types'; @@ -25,7 +25,7 @@ import { fetchTransactionDurationRanges } from './query_ranges'; export async function fetchTransactionDurationCorrelationWithHistogram( esClient: ElasticsearchClient, - params: SearchStrategyParams, + params: CorrelationsParams, expectations: number[], ranges: estypes.AggregationsAggregationRange[], fractions: number[], diff --git a/x-pack/plugins/apm/server/lib/correlations/queries/query_failure_correlation.ts b/x-pack/plugins/apm/server/lib/correlations/queries/query_failure_correlation.ts index f763b1a9abdb0..4b2d540e7a1c2 100644 --- a/x-pack/plugins/apm/server/lib/correlations/queries/query_failure_correlation.ts +++ b/x-pack/plugins/apm/server/lib/correlations/queries/query_failure_correlation.ts @@ -6,7 +6,7 @@ */ import { estypes } from '@elastic/elasticsearch'; import { ElasticsearchClient } from 'kibana/server'; -import { SearchStrategyParams } from '../../../../common/correlations/types'; +import { CorrelationsParams } from '../../../../common/correlations/types'; import { FailedTransactionsCorrelation } from '../../../../common/correlations/failed_transactions_correlations/types'; import { EVENT_OUTCOME } from '../../../../common/elasticsearch_fieldnames'; import { EventOutcome } from '../../../../common/event_outcome'; @@ -15,7 +15,7 @@ import { getQueryWithParams, getTermsQuery } from './get_query_with_params'; import { getRequestBase } from './get_request_base'; export const getFailureCorrelationRequest = ( - params: SearchStrategyParams, + params: CorrelationsParams, fieldName: string ): estypes.SearchRequest => { const query = getQueryWithParams({ @@ -66,7 +66,7 @@ export const getFailureCorrelationRequest = ( export const fetchFailedTransactionsCorrelationPValues = async ( esClient: ElasticsearchClient, - params: SearchStrategyParams, + params: CorrelationsParams, histogramRangeSteps: number[], fieldName: string ) => { diff --git a/x-pack/plugins/apm/server/lib/correlations/queries/query_field_candidates.ts b/x-pack/plugins/apm/server/lib/correlations/queries/query_field_candidates.ts index 90bd6b867ab09..7ae67433d0e8e 100644 --- a/x-pack/plugins/apm/server/lib/correlations/queries/query_field_candidates.ts +++ b/x-pack/plugins/apm/server/lib/correlations/queries/query_field_candidates.ts @@ -9,7 +9,7 @@ import type { estypes } from '@elastic/elasticsearch'; import type { ElasticsearchClient } from 'src/core/server'; -import type { SearchStrategyParams } from '../../../../common/correlations/types'; +import type { CorrelationsParams } from '../../../../common/correlations/types'; import { FIELD_PREFIX_TO_EXCLUDE_AS_CANDIDATE, FIELDS_TO_ADD_AS_CANDIDATE, @@ -31,7 +31,7 @@ export const shouldBeExcluded = (fieldName: string) => { }; export const getRandomDocsRequest = ( - params: SearchStrategyParams + params: CorrelationsParams ): estypes.SearchRequest => ({ ...getRequestBase(params), body: { @@ -50,7 +50,7 @@ export const getRandomDocsRequest = ( export const fetchTransactionDurationFieldCandidates = async ( esClient: ElasticsearchClient, - params: SearchStrategyParams + params: CorrelationsParams ): Promise<{ fieldCandidates: string[] }> => { const { index } = params; // Get all fields with keyword mapping diff --git a/x-pack/plugins/apm/server/lib/correlations/queries/query_field_value_pairs.ts b/x-pack/plugins/apm/server/lib/correlations/queries/query_field_value_pairs.ts index 9e5cd26605a9d..1f7333e4d07f8 100644 --- a/x-pack/plugins/apm/server/lib/correlations/queries/query_field_value_pairs.ts +++ b/x-pack/plugins/apm/server/lib/correlations/queries/query_field_value_pairs.ts @@ -11,7 +11,7 @@ import type { estypes } from '@elastic/elasticsearch'; import type { FieldValuePair, - SearchStrategyParams, + CorrelationsParams, } from '../../../../common/correlations/types'; import { TERMS_SIZE } from '../../../../common/correlations/constants'; @@ -19,7 +19,7 @@ import { getQueryWithParams } from './get_query_with_params'; import { getRequestBase } from './get_request_base'; export const getTermsAggRequest = ( - params: SearchStrategyParams, + params: CorrelationsParams, fieldName: string ): estypes.SearchRequest => ({ ...getRequestBase(params), @@ -39,7 +39,7 @@ export const getTermsAggRequest = ( const fetchTransactionDurationFieldTerms = async ( esClient: ElasticsearchClient, - params: SearchStrategyParams, + params: CorrelationsParams, fieldName: string ): Promise => { const resp = await esClient.search(getTermsAggRequest(params, fieldName)); @@ -68,7 +68,7 @@ const fetchTransactionDurationFieldTerms = async ( export const fetchTransactionDurationFieldValuePairs = async ( esClient: ElasticsearchClient, - params: SearchStrategyParams, + params: CorrelationsParams, fieldCandidates: string[] ): Promise => { const responses = await Promise.all( diff --git a/x-pack/plugins/apm/server/lib/correlations/queries/query_fractions.ts b/x-pack/plugins/apm/server/lib/correlations/queries/query_fractions.ts index 7e47b6edfb018..5b5c1cfe2b353 100644 --- a/x-pack/plugins/apm/server/lib/correlations/queries/query_fractions.ts +++ b/x-pack/plugins/apm/server/lib/correlations/queries/query_fractions.ts @@ -8,14 +8,14 @@ import { ElasticsearchClient } from 'kibana/server'; import { estypes } from '@elastic/elasticsearch'; -import { SearchStrategyParams } from '../../../../common/correlations/types'; +import { CorrelationsParams } from '../../../../common/correlations/types'; import { TRANSACTION_DURATION } from '../../../../common/elasticsearch_fieldnames'; import { getQueryWithParams } from './get_query_with_params'; import { getRequestBase } from './get_request_base'; export const getTransactionDurationRangesRequest = ( - params: SearchStrategyParams, + params: CorrelationsParams, ranges: estypes.AggregationsAggregationRange[] ): estypes.SearchRequest => ({ ...getRequestBase(params), @@ -38,7 +38,7 @@ export const getTransactionDurationRangesRequest = ( */ export const fetchTransactionDurationFractions = async ( esClient: ElasticsearchClient, - params: SearchStrategyParams, + params: CorrelationsParams, ranges: estypes.AggregationsAggregationRange[] ): Promise<{ fractions: number[]; totalDocCount: number }> => { const resp = await esClient.search( diff --git a/x-pack/plugins/apm/server/lib/correlations/queries/query_histogram.ts b/x-pack/plugins/apm/server/lib/correlations/queries/query_histogram.ts index f4c4e05a248a1..e817845345100 100644 --- a/x-pack/plugins/apm/server/lib/correlations/queries/query_histogram.ts +++ b/x-pack/plugins/apm/server/lib/correlations/queries/query_histogram.ts @@ -14,14 +14,14 @@ import type { FieldValuePair, HistogramItem, ResponseHit, - SearchStrategyParams, + CorrelationsParams, } from '../../../../common/correlations/types'; import { getQueryWithParams } from './get_query_with_params'; import { getRequestBase } from './get_request_base'; export const getTransactionDurationHistogramRequest = ( - params: SearchStrategyParams, + params: CorrelationsParams, interval: number, termFilters?: FieldValuePair[] ): estypes.SearchRequest => ({ @@ -39,7 +39,7 @@ export const getTransactionDurationHistogramRequest = ( export const fetchTransactionDurationHistogram = async ( esClient: ElasticsearchClient, - params: SearchStrategyParams, + params: CorrelationsParams, interval: number, termFilters?: FieldValuePair[] ): Promise => { diff --git a/x-pack/plugins/apm/server/lib/correlations/queries/query_histogram_range_steps.ts b/x-pack/plugins/apm/server/lib/correlations/queries/query_histogram_range_steps.ts index a2ce649b9a99a..98755e39c1209 100644 --- a/x-pack/plugins/apm/server/lib/correlations/queries/query_histogram_range_steps.ts +++ b/x-pack/plugins/apm/server/lib/correlations/queries/query_histogram_range_steps.ts @@ -12,7 +12,7 @@ import type { estypes } from '@elastic/elasticsearch'; import type { ElasticsearchClient } from 'src/core/server'; import { TRANSACTION_DURATION } from '../../../../common/elasticsearch_fieldnames'; -import type { SearchStrategyParams } from '../../../../common/correlations/types'; +import type { CorrelationsParams } from '../../../../common/correlations/types'; import { getQueryWithParams } from './get_query_with_params'; import { getRequestBase } from './get_request_base'; @@ -31,7 +31,7 @@ export const getHistogramRangeSteps = ( }; export const getHistogramIntervalRequest = ( - params: SearchStrategyParams + params: CorrelationsParams ): estypes.SearchRequest => ({ ...getRequestBase(params), body: { @@ -46,7 +46,7 @@ export const getHistogramIntervalRequest = ( export const fetchTransactionDurationHistogramRangeSteps = async ( esClient: ElasticsearchClient, - params: SearchStrategyParams + params: CorrelationsParams ): Promise => { const steps = 100; diff --git a/x-pack/plugins/apm/server/lib/correlations/queries/query_p_values.ts b/x-pack/plugins/apm/server/lib/correlations/queries/query_p_values.ts index df7e8dd8932a8..7c471aebd0f7a 100644 --- a/x-pack/plugins/apm/server/lib/correlations/queries/query_p_values.ts +++ b/x-pack/plugins/apm/server/lib/correlations/queries/query_p_values.ts @@ -7,7 +7,7 @@ import type { ElasticsearchClient } from 'src/core/server'; -import type { SearchStrategyParams } from '../../../../common/correlations/types'; +import type { CorrelationsParams } from '../../../../common/correlations/types'; import type { FailedTransactionsCorrelation } from '../../../../common/correlations/failed_transactions_correlations/types'; import { ERROR_CORRELATION_THRESHOLD } from '../../../../common/correlations/constants'; @@ -20,7 +20,7 @@ import { export const fetchPValues = async ( esClient: ElasticsearchClient, - paramsWithIndex: SearchStrategyParams, + paramsWithIndex: CorrelationsParams, fieldCandidates: string[] ) => { const histogramRangeSteps = await fetchTransactionDurationHistogramRangeSteps( diff --git a/x-pack/plugins/apm/server/lib/correlations/queries/query_percentiles.ts b/x-pack/plugins/apm/server/lib/correlations/queries/query_percentiles.ts index 8d444b6736e11..e64b9b77af56a 100644 --- a/x-pack/plugins/apm/server/lib/correlations/queries/query_percentiles.ts +++ b/x-pack/plugins/apm/server/lib/correlations/queries/query_percentiles.ts @@ -14,14 +14,14 @@ import { SIGNIFICANT_VALUE_DIGITS } from '../../../../common/correlations/consta import type { FieldValuePair, ResponseHit, - SearchStrategyParams, + CorrelationsParams, } from '../../../../common/correlations/types'; import { getQueryWithParams } from './get_query_with_params'; import { getRequestBase } from './get_request_base'; export const getTransactionDurationPercentilesRequest = ( - params: SearchStrategyParams, + params: CorrelationsParams, percents?: number[], termFilters?: FieldValuePair[] ): estypes.SearchRequest => { @@ -50,7 +50,7 @@ export const getTransactionDurationPercentilesRequest = ( export const fetchTransactionDurationPercentiles = async ( esClient: ElasticsearchClient, - params: SearchStrategyParams, + params: CorrelationsParams, percents?: number[], termFilters?: FieldValuePair[] ): Promise<{ totalDocs: number; percentiles: Record }> => { diff --git a/x-pack/plugins/apm/server/lib/correlations/queries/query_ranges.ts b/x-pack/plugins/apm/server/lib/correlations/queries/query_ranges.ts index 13ba83cdb8c36..94e1086293d45 100644 --- a/x-pack/plugins/apm/server/lib/correlations/queries/query_ranges.ts +++ b/x-pack/plugins/apm/server/lib/correlations/queries/query_ranges.ts @@ -13,14 +13,14 @@ import { TRANSACTION_DURATION } from '../../../../common/elasticsearch_fieldname import type { FieldValuePair, ResponseHit, - SearchStrategyParams, + CorrelationsParams, } from '../../../../common/correlations/types'; import { getQueryWithParams } from './get_query_with_params'; import { getRequestBase } from './get_request_base'; export const getTransactionDurationRangesRequest = ( - params: SearchStrategyParams, + params: CorrelationsParams, rangesSteps: number[], termFilters?: FieldValuePair[] ): estypes.SearchRequest => { @@ -57,7 +57,7 @@ export const getTransactionDurationRangesRequest = ( export const fetchTransactionDurationRanges = async ( esClient: ElasticsearchClient, - params: SearchStrategyParams, + params: CorrelationsParams, rangesSteps: number[], termFilters?: FieldValuePair[] ): Promise> => { diff --git a/x-pack/plugins/apm/server/lib/correlations/queries/query_significant_correlations.ts b/x-pack/plugins/apm/server/lib/correlations/queries/query_significant_correlations.ts index e35cc48722c53..869b24ed82b4a 100644 --- a/x-pack/plugins/apm/server/lib/correlations/queries/query_significant_correlations.ts +++ b/x-pack/plugins/apm/server/lib/correlations/queries/query_significant_correlations.ts @@ -11,7 +11,7 @@ import type { ElasticsearchClient } from 'src/core/server'; import type { FieldValuePair, - SearchStrategyParams, + CorrelationsParams, } from '../../../../common/correlations/types'; import { isLatencyCorrelation, @@ -32,7 +32,7 @@ import { export const fetchSignificantCorrelations = async ( esClient: ElasticsearchClient, - paramsWithIndex: SearchStrategyParams, + paramsWithIndex: CorrelationsParams, fieldValuePairs: FieldValuePair[] ) => { // Create an array of ranges [2, 4, 6, ..., 98] From aa80643de068722476fecc64504dd9dce39bb97a Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Tue, 19 Oct 2021 19:25:56 +0200 Subject: [PATCH 17/39] [ML] Deduplicate some code. --- .../use_failed_transactions_correlations.ts | 41 ++++------------ .../correlations/use_latency_correlations.ts | 41 ++++------------ .../correlations/utils/analysis_hook_utils.ts | 47 +++++++++++++++++++ 3 files changed, 63 insertions(+), 66 deletions(-) create mode 100644 x-pack/plugins/apm/public/components/app/correlations/utils/analysis_hook_utils.ts diff --git a/x-pack/plugins/apm/public/components/app/correlations/use_failed_transactions_correlations.ts b/x-pack/plugins/apm/public/components/app/correlations/use_failed_transactions_correlations.ts index 26bbef393b63c..990b64884fc25 100644 --- a/x-pack/plugins/apm/public/components/app/correlations/use_failed_transactions_correlations.ts +++ b/x-pack/plugins/apm/public/components/app/correlations/use_failed_transactions_correlations.ts @@ -8,8 +8,6 @@ import { useCallback, useEffect, useReducer, useRef } from 'react'; import { chunk } from 'lodash'; -import { IHttpFetchError } from 'src/core/public'; - import { EVENT_OUTCOME } from '../../../../common/elasticsearch_fieldnames'; import { EventOutcome } from '../../../../common/event_outcome'; import { DEFAULT_PERCENTILE_THRESHOLD } from '../../../../common/correlations/constants'; @@ -25,38 +23,15 @@ import { useApmParams } from '../../../hooks/use_apm_params'; import { useTimeRange } from '../../../hooks/use_time_range'; import { callApmApi } from '../../../services/rest/createCallApmApi'; -type Response = FailedTransactionsCorrelationsRawResponse; - -interface CorrelationsProgress { - error?: Error | IHttpFetchError; - isRunning: boolean; - loaded: number; - total: number; -} +import { + getInitialProgress, + getFailedTransactionsCorrelationsSortedByScore, + getInitialRawResponse, + getReducer, + CorrelationsProgress, +} from './utils/analysis_hook_utils'; -function getFailedTransactionsCorrelationsSortedByScore( - failedTransactionsCorrelations: FailedTransactionsCorrelation[] -) { - return failedTransactionsCorrelations.sort((a, b) => b.score - a.score); -} - -const getInitialRawResponse = (): Response => - ({ - ccsWarning: false, - } as Response); - -const getInitialProgress = (): CorrelationsProgress => ({ - isRunning: false, - loaded: 0, - total: 100, -}); - -const getReducer = - () => - (prev: T, update: Partial): T => ({ - ...prev, - ...update, - }); +type Response = FailedTransactionsCorrelationsRawResponse; export function useFailedTransactionsCorrelations() { const { serviceName, transactionType } = useApmServiceContext(); diff --git a/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts b/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts index 0d5edd75c0d02..492a8a79c9783 100644 --- a/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts +++ b/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts @@ -8,8 +8,6 @@ import { useCallback, useEffect, useReducer, useRef } from 'react'; import { chunk } from 'lodash'; -import { IHttpFetchError } from 'src/core/public'; - import { DEFAULT_PERCENTILE_THRESHOLD } from '../../../../common/correlations/constants'; import type { FieldValuePair } from '../../../../common/correlations/types'; import { getPrioritizedFieldValuePairs } from '../../../../common/correlations/utils'; @@ -25,38 +23,15 @@ import { useApmParams } from '../../../hooks/use_apm_params'; import { useTimeRange } from '../../../hooks/use_time_range'; import { callApmApi } from '../../../services/rest/createCallApmApi'; -type Response = LatencyCorrelationsRawResponse; - -interface CorrelationsProgress { - error?: Error | IHttpFetchError; - isRunning: boolean; - loaded: number; - total: number; -} +import { + getInitialProgress, + getLatencyCorrelationsSortedByCorrelation, + getInitialRawResponse, + getReducer, + CorrelationsProgress, +} from './utils/analysis_hook_utils'; -function getLatencyCorrelationsSortedByCorrelation( - latencyCorrelations: LatencyCorrelation[] -) { - return latencyCorrelations.sort((a, b) => b.correlation - a.correlation); -} - -const getInitialRawResponse = (): Response => - ({ - ccsWarning: false, - } as Response); - -const getInitialProgress = (): CorrelationsProgress => ({ - isRunning: false, - loaded: 0, - total: 100, -}); - -const getReducer = - () => - (prev: T, update: Partial): T => ({ - ...prev, - ...update, - }); +type Response = LatencyCorrelationsRawResponse; export function useLatencyCorrelations() { const { serviceName, transactionType } = useApmServiceContext(); diff --git a/x-pack/plugins/apm/public/components/app/correlations/utils/analysis_hook_utils.ts b/x-pack/plugins/apm/public/components/app/correlations/utils/analysis_hook_utils.ts new file mode 100644 index 0000000000000..aaba8f4509fb1 --- /dev/null +++ b/x-pack/plugins/apm/public/components/app/correlations/utils/analysis_hook_utils.ts @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { IHttpFetchError } from 'src/core/public'; + +import type { FailedTransactionsCorrelation } from '../../../../../common/correlations/failed_transactions_correlations/types'; +import type { LatencyCorrelation } from '../../../../../common/correlations/latency_correlations/types'; + +export interface CorrelationsProgress { + error?: Error | IHttpFetchError; + isRunning: boolean; + loaded: number; + total: number; +} + +export function getLatencyCorrelationsSortedByCorrelation( + latencyCorrelations: LatencyCorrelation[] +) { + return latencyCorrelations.sort((a, b) => b.correlation - a.correlation); +} + +export function getFailedTransactionsCorrelationsSortedByScore( + failedTransactionsCorrelations: FailedTransactionsCorrelation[] +) { + return failedTransactionsCorrelations.sort((a, b) => b.score - a.score); +} + +export const getInitialRawResponse = () => ({ + ccsWarning: false, +}); + +export const getInitialProgress = (): CorrelationsProgress => ({ + isRunning: false, + loaded: 0, + total: 100, +}); + +export const getReducer = + () => + (prev: T, update: Partial): T => ({ + ...prev, + ...update, + }); From 4f080bfd668dd08e9dfa1c859e27fe50f87bcc65 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Tue, 19 Oct 2021 20:53:10 +0200 Subject: [PATCH 18/39] [ML] Adds debouncing to correlation analysis. --- .../latency_correlations/types.ts | 7 ------ .../use_failed_transactions_correlations.ts | 25 +++++++++++++------ .../correlations/use_latency_correlations.ts | 25 +++++++++++++------ .../queries/query_significant_correlations.ts | 10 +++----- 4 files changed, 38 insertions(+), 29 deletions(-) diff --git a/x-pack/plugins/apm/common/correlations/latency_correlations/types.ts b/x-pack/plugins/apm/common/correlations/latency_correlations/types.ts index bc90d955984a2..e13e5c48b5452 100644 --- a/x-pack/plugins/apm/common/correlations/latency_correlations/types.ts +++ b/x-pack/plugins/apm/common/correlations/latency_correlations/types.ts @@ -14,13 +14,6 @@ export interface LatencyCorrelation extends FieldValuePair { ksTest: number; } -export const isLatencyCorrelation = (arg: unknown): arg is LatencyCorrelation => - typeof arg === 'object' && - arg !== null && - Object.keys(arg).length === 5 && - Object.keys(arg).includes('correlation') && - !Object.keys(arg).includes('error'); - export interface LatencyCorrelationsRawResponse { ccsWarning: boolean; overallHistogram?: HistogramItem[]; diff --git a/x-pack/plugins/apm/public/components/app/correlations/use_failed_transactions_correlations.ts b/x-pack/plugins/apm/public/components/app/correlations/use_failed_transactions_correlations.ts index 990b64884fc25..7ea75629d0b96 100644 --- a/x-pack/plugins/apm/public/components/app/correlations/use_failed_transactions_correlations.ts +++ b/x-pack/plugins/apm/public/components/app/correlations/use_failed_transactions_correlations.ts @@ -5,8 +5,8 @@ * 2.0. */ -import { useCallback, useEffect, useReducer, useRef } from 'react'; -import { chunk } from 'lodash'; +import { useCallback, useEffect, useMemo, useReducer, useRef } from 'react'; +import { chunk, debounce } from 'lodash'; import { EVENT_OUTCOME } from '../../../../common/elasticsearch_fieldnames'; import { EventOutcome } from '../../../../common/event_outcome'; @@ -45,26 +45,31 @@ export function useFailedTransactionsCorrelations() { const { start, end } = useTimeRange({ rangeFrom, rangeTo }); - const [rawResponse, setRawResponse] = useReducer( + const [rawResponse, setRawResponseRaw] = useReducer( getReducer(), getInitialRawResponse() ); + const setRawResponse = useMemo(() => debounce(setRawResponseRaw, 50), []); - const [fetchState, setFetchState] = useReducer( + const [fetchState, setFetchStateRaw] = useReducer( getReducer(), getInitialProgress() ); + const setFetchState = useMemo(() => debounce(setFetchStateRaw, 50), []); const isCancelledRef = useRef(false); const startFetch = useCallback(async () => { isCancelledRef.current = false; + setRawResponse(getInitialRawResponse()); setFetchState({ ...getInitialProgress(), isRunning: true, error: undefined, }); + setRawResponse.flush(); + setFetchState.flush(); const query = { serviceName, @@ -157,9 +162,9 @@ export function useFailedTransactionsCorrelations() { ...pValues.failedTransactionsCorrelations ); rawResponseUpdate.failedTransactionsCorrelations = - getFailedTransactionsCorrelationsSortedByScore( - failedTransactionsCorrelations - ); + getFailedTransactionsCorrelationsSortedByScore([ + ...failedTransactionsCorrelations, + ]); setRawResponse(rawResponseUpdate); } @@ -190,6 +195,8 @@ export function useFailedTransactionsCorrelations() { setFetchState({ loaded: 100, }); + setRawResponse.flush(); + setFetchState.flush(); } catch (e) { // const err = e as Error | IHttpFetchError; // const message = error.body?.message ?? error.response?.statusText; @@ -209,6 +216,8 @@ export function useFailedTransactionsCorrelations() { kuery, start, end, + setFetchState, + setRawResponse, ]); const cancelFetch = useCallback(() => { @@ -216,7 +225,7 @@ export function useFailedTransactionsCorrelations() { setFetchState({ isRunning: false, }); - }, []); + }, [setFetchState]); // auto-update useEffect(() => { diff --git a/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts b/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts index 492a8a79c9783..393bcd09b4f08 100644 --- a/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts +++ b/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts @@ -5,8 +5,8 @@ * 2.0. */ -import { useCallback, useEffect, useReducer, useRef } from 'react'; -import { chunk } from 'lodash'; +import { useCallback, useEffect, useMemo, useReducer, useRef } from 'react'; +import { chunk, debounce } from 'lodash'; import { DEFAULT_PERCENTILE_THRESHOLD } from '../../../../common/correlations/constants'; import type { FieldValuePair } from '../../../../common/correlations/types'; @@ -45,26 +45,31 @@ export function useLatencyCorrelations() { const { start, end } = useTimeRange({ rangeFrom, rangeTo }); - const [rawResponse, setRawResponse] = useReducer( + const [rawResponse, setRawResponseRaw] = useReducer( getReducer(), getInitialRawResponse() ); + const setRawResponse = useMemo(() => debounce(setRawResponseRaw, 50), []); - const [fetchState, setFetchState] = useReducer( + const [fetchState, setFetchStateRaw] = useReducer( getReducer(), getInitialProgress() ); + const setFetchState = useMemo(() => debounce(setFetchStateRaw, 50), []); const isCancelledRef = useRef(false); const startFetch = useCallback(async () => { isCancelledRef.current = false; + setRawResponse(getInitialRawResponse()); setFetchState({ ...getInitialProgress(), isRunning: true, error: undefined, }); + setRawResponse.flush(); + setFetchState.flush(); const query = { serviceName, @@ -139,11 +144,11 @@ export function useLatencyCorrelations() { return; } - chunkLoadCounter += chunkSize; setFetchState({ loaded: 10 + Math.round((chunkLoadCounter / fieldValuePairs.length) * 10), }); + chunkLoadCounter += chunkSize; } if (isCancelledRef.current) { @@ -180,7 +185,7 @@ export function useLatencyCorrelations() { ...significantCorrelations.latencyCorrelations ); rawResponseUpdate.latencyCorrelations = - getLatencyCorrelationsSortedByCorrelation(latencyCorrelations); + getLatencyCorrelationsSortedByCorrelation([...latencyCorrelations]); setRawResponse(rawResponseUpdate); } @@ -188,11 +193,11 @@ export function useLatencyCorrelations() { return; } - chunkLoadCounter += chunkSize; setFetchState({ loaded: 20 + Math.round((chunkLoadCounter / fieldValuePairs.length) * 80), }); + chunkLoadCounter += chunkSize; } const fieldStats = await callApmApi({ @@ -212,6 +217,8 @@ export function useLatencyCorrelations() { setFetchState({ loaded: 100, }); + setRawResponse.flush(); + setFetchState.flush(); } catch (e) { // const err = e as Error | IHttpFetchError; // const message = error.body?.message ?? error.response?.statusText; @@ -231,6 +238,8 @@ export function useLatencyCorrelations() { kuery, start, end, + setRawResponse, + setFetchState, ]); const cancelFetch = useCallback(() => { @@ -238,7 +247,7 @@ export function useLatencyCorrelations() { setFetchState({ isRunning: false, }); - }, []); + }, [setFetchState]); // auto-update useEffect(() => { diff --git a/x-pack/plugins/apm/server/lib/correlations/queries/query_significant_correlations.ts b/x-pack/plugins/apm/server/lib/correlations/queries/query_significant_correlations.ts index 869b24ed82b4a..b8cee29fca16f 100644 --- a/x-pack/plugins/apm/server/lib/correlations/queries/query_significant_correlations.ts +++ b/x-pack/plugins/apm/server/lib/correlations/queries/query_significant_correlations.ts @@ -13,10 +13,7 @@ import type { FieldValuePair, CorrelationsParams, } from '../../../../common/correlations/types'; -import { - isLatencyCorrelation, - LatencyCorrelation, -} from '../../../../common/correlations/latency_correlations/types'; +import { LatencyCorrelation } from '../../../../common/correlations/latency_correlations/types'; import { computeExpectationsAndRanges, @@ -75,8 +72,9 @@ export const fetchSignificantCorrelations = async ( ) ); - const latencyCorrelations: LatencyCorrelation[] = - fulfilled.filter(isLatencyCorrelation); + const latencyCorrelations: LatencyCorrelation[] = fulfilled.filter( + (d): d is LatencyCorrelation => d !== undefined + ); const ccsWarning = rejected.length > 0 && paramsWithIndex?.index.includes(':'); From 8868bb93b207925282478ef78c578c05f9a3f56c Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Tue, 19 Oct 2021 22:00:43 +0200 Subject: [PATCH 19/39] [ML] Fix types. --- ...use_transaction_distribution_chart_data.ts | 28 +++++++++---------- .../lib/correlations/queries/get_filters.ts | 4 +-- .../plugins/apm/server/lib/latency/types.ts | 4 +-- .../tests/correlations/failed_transactions.ts | 7 ++--- .../tests/correlations/latency.ts | 7 ++--- 5 files changed, 22 insertions(+), 28 deletions(-) diff --git a/x-pack/plugins/apm/public/components/app/transaction_details/distribution/use_transaction_distribution_chart_data.ts b/x-pack/plugins/apm/public/components/app/transaction_details/distribution/use_transaction_distribution_chart_data.ts index 650756b3fc538..7f005f662fc57 100644 --- a/x-pack/plugins/apm/public/components/app/transaction_details/distribution/use_transaction_distribution_chart_data.ts +++ b/x-pack/plugins/apm/public/components/app/transaction_details/distribution/use_transaction_distribution_chart_data.ts @@ -10,7 +10,6 @@ import { useEffect, useMemo } from 'react'; import { i18n } from '@kbn/i18n'; import { DEFAULT_PERCENTILE_THRESHOLD } from '../../../../../common/correlations/constants'; -import { RawSearchStrategyClientParams } from '../../../../../common/correlations/types'; import { EVENT_OUTCOME } from '../../../../../common/elasticsearch_fieldnames'; import { EventOutcome } from '../../../../../common/event_outcome'; @@ -25,11 +24,6 @@ import type { TransactionDistributionChartData } from '../../../shared/charts/tr import { isErrorMessage } from '../../correlations/utils/is_error_message'; -function hasRequiredParams(params: RawSearchStrategyClientParams) { - const { serviceName, environment, start, end } = params; - return serviceName && environment && start && end; -} - export const useTransactionDistributionChartData = () => { const { serviceName, transactionType } = useApmServiceContext(); @@ -68,14 +62,17 @@ export const useTransactionDistributionChartData = () => { ); const { - // TODO The default object has `log: []` to retain compatibility with the shared search strategies code. - // Remove once the other tabs are migrated away from search strategies. - data: overallLatencyData = { log: [] }, + data: overallLatencyData = {}, status: overallLatencyStatus, error: overallLatencyError, } = useFetcher( (callApmApi) => { - if (hasRequiredParams(params)) { + if ( + params.serviceName && + params.environment && + params.start && + params.end + ) { return callApmApi({ endpoint: 'POST /internal/apm/latency/overall_distribution', params: { @@ -114,12 +111,15 @@ export const useTransactionDistributionChartData = () => { Array.isArray(overallLatencyHistogram) && overallLatencyHistogram.length > 0; - // TODO The default object has `log: []` to retain compatibility with the shared search strategies code. - // Remove once the other tabs are migrated away from search strategies. - const { data: errorHistogramData = { log: [] }, error: errorHistogramError } = + const { data: errorHistogramData = {}, error: errorHistogramError } = useFetcher( (callApmApi) => { - if (hasRequiredParams(params)) { + if ( + params.serviceName && + params.environment && + params.start && + params.end + ) { return callApmApi({ endpoint: 'POST /internal/apm/latency/overall_distribution', params: { diff --git a/x-pack/plugins/apm/server/lib/correlations/queries/get_filters.ts b/x-pack/plugins/apm/server/lib/correlations/queries/get_filters.ts index b9ce529b1a05b..58ee5051d8863 100644 --- a/x-pack/plugins/apm/server/lib/correlations/queries/get_filters.ts +++ b/x-pack/plugins/apm/server/lib/correlations/queries/get_filters.ts @@ -15,7 +15,7 @@ import { PROCESSOR_EVENT, } from '../../../../common/elasticsearch_fieldnames'; import { ProcessorEvent } from '../../../../common/processor_event'; -import { SearchStrategyClientParams } from '../../../../common/correlations/types'; +import { CorrelationsClientParams } from '../../../../common/correlations/types'; export function getCorrelationsFilters({ environment, @@ -25,7 +25,7 @@ export function getCorrelationsFilters({ transactionName, start, end, -}: SearchStrategyClientParams) { +}: CorrelationsClientParams) { const correlationsFilters: ESFilter[] = [ { term: { [PROCESSOR_EVENT]: ProcessorEvent.transaction } }, ...rangeQuery(start, end), diff --git a/x-pack/plugins/apm/server/lib/latency/types.ts b/x-pack/plugins/apm/server/lib/latency/types.ts index 0179132949f9d..17c036f44f088 100644 --- a/x-pack/plugins/apm/server/lib/latency/types.ts +++ b/x-pack/plugins/apm/server/lib/latency/types.ts @@ -7,13 +7,13 @@ import type { FieldValuePair, - SearchStrategyClientParams, + CorrelationsClientParams, } from '../../../common/correlations/types'; import { Setup } from '../helpers/setup_request'; export interface OverallLatencyDistributionOptions - extends SearchStrategyClientParams { + extends CorrelationsClientParams { percentileThreshold: number; termFilters?: FieldValuePair[]; setup: Setup; diff --git a/x-pack/test/apm_api_integration/tests/correlations/failed_transactions.ts b/x-pack/test/apm_api_integration/tests/correlations/failed_transactions.ts index 492b9c7df1715..659cf85096138 100644 --- a/x-pack/test/apm_api_integration/tests/correlations/failed_transactions.ts +++ b/x-pack/test/apm_api_integration/tests/correlations/failed_transactions.ts @@ -9,7 +9,6 @@ import expect from '@kbn/expect'; import { FtrProviderContext } from '../../common/ftr_provider_context'; import { registry } from '../../common/registry'; -import type { RawResponseBase } from '../../../../plugins/apm/common/correlations/types'; import type { FailedTransactionsCorrelationsRawResponse } from '../../../../plugins/apm/common/correlations/failed_transactions_correlations/types'; import { EVENT_OUTCOME } from '../../../../plugins/apm/common/elasticsearch_fieldnames'; import { EventOutcome } from '../../../../plugins/apm/common/event_outcome'; @@ -87,13 +86,12 @@ export default function ApiTest({ getService }: FtrProviderContext) { `Expected status to be '200', got '${failedTransactionsCorrelationsResponse.status}'` ); - const finalRawResponse: FailedTransactionsCorrelationsRawResponse & RawResponseBase = { + const finalRawResponse: FailedTransactionsCorrelationsRawResponse = { ccsWarning: failedTransactionsCorrelationsResponse.body?.ccsWarning, percentileThresholdValue: overallDistributionResponse.body?.percentileThresholdValue, overallHistogram: overallDistributionResponse.body?.overallHistogram, failedTransactionsCorrelations: failedTransactionsCorrelationsResponse.body?.failedTransactionsCorrelations, - log: [], }; expect(finalRawResponse?.failedTransactionsCorrelations?.length).to.eql( @@ -173,14 +171,13 @@ export default function ApiTest({ getService }: FtrProviderContext) { `Expected status to be '200', got '${failedTransactionsCorrelationsResponse.status}'` ); - const finalRawResponse: FailedTransactionsCorrelationsRawResponse & RawResponseBase = { + const finalRawResponse: FailedTransactionsCorrelationsRawResponse = { ccsWarning: failedTransactionsCorrelationsResponse.body?.ccsWarning, percentileThresholdValue: overallDistributionResponse.body?.percentileThresholdValue, overallHistogram: overallDistributionResponse.body?.overallHistogram, errorHistogram: errorDistributionResponse.body?.overallHistogram, failedTransactionsCorrelations: failedTransactionsCorrelationsResponse.body?.failedTransactionsCorrelations, - log: [], }; expect(finalRawResponse?.percentileThresholdValue).to.be(1309695.875); diff --git a/x-pack/test/apm_api_integration/tests/correlations/latency.ts b/x-pack/test/apm_api_integration/tests/correlations/latency.ts index 2151a9f8244ba..a9c59fa79f3ec 100644 --- a/x-pack/test/apm_api_integration/tests/correlations/latency.ts +++ b/x-pack/test/apm_api_integration/tests/correlations/latency.ts @@ -9,7 +9,6 @@ import expect from '@kbn/expect'; import { FtrProviderContext } from '../../common/ftr_provider_context'; import { registry } from '../../common/registry'; -import type { RawResponseBase } from '../../../../plugins/apm/common/correlations/types'; import type { LatencyCorrelationsRawResponse } from '../../../../plugins/apm/common/correlations/latency_correlations/types'; // These tests go through the full sequence of queries required @@ -87,12 +86,11 @@ export default function ApiTest({ getService }: FtrProviderContext) { `Expected status to be '200', got '${significantCorrelationsResponse.status}'` ); - const finalRawResponse: LatencyCorrelationsRawResponse & RawResponseBase = { + const finalRawResponse: LatencyCorrelationsRawResponse = { ccsWarning: significantCorrelationsResponse.body?.ccsWarning, percentileThresholdValue: overallDistributionResponse.body?.percentileThresholdValue, overallHistogram: overallDistributionResponse.body?.overallHistogram, latencyCorrelations: significantCorrelationsResponse.body?.latencyCorrelations, - log: [], }; expect(finalRawResponse?.percentileThresholdValue).to.be(undefined); @@ -183,12 +181,11 @@ export default function ApiTest({ getService }: FtrProviderContext) { `Expected 1244 total doc count, got ${significantCorrelationsResponse.body?.totalDocCount}.` ); - const finalRawResponse: LatencyCorrelationsRawResponse & RawResponseBase = { + const finalRawResponse: LatencyCorrelationsRawResponse = { ccsWarning: significantCorrelationsResponse.body?.ccsWarning, percentileThresholdValue: overallDistributionResponse.body?.percentileThresholdValue, overallHistogram: overallDistributionResponse.body?.overallHistogram, latencyCorrelations: significantCorrelationsResponse.body?.latencyCorrelations, - log: [], }; // Fetched 95th percentile value of 1309695.875 based on 1244 documents. From 5a3b9278d71b9f7d5069013367b78c6b24138ba4 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Wed, 20 Oct 2021 13:08:00 +0200 Subject: [PATCH 20/39] [ML] Cleanup. Fix progress. --- .../apm/common/correlations/constants.ts | 7 - .../failed_transactions_correlations/types.ts | 2 +- .../latency_correlations/types.ts | 2 +- .../failed_transactions_correlations.tsx | 5 +- .../latency_correlations.test.tsx | 4 +- .../app/correlations/latency_correlations.tsx | 6 +- .../use_failed_transactions_correlations.ts | 158 +++++++---------- .../app/correlations/use_fetch_params.ts | 48 +++++ .../correlations/use_latency_correlations.ts | 166 ++++++++---------- .../correlations/utils/analysis_hook_utils.ts | 7 +- .../utils/get_overall_histogram.test.ts | 10 +- .../utils/get_overall_histogram.ts | 4 +- ...use_transaction_distribution_chart_data.ts | 39 +--- .../hooks/use_latency_overall_distribution.ts | 73 -------- .../tests/correlations/failed_transactions.ts | 6 +- .../tests/correlations/latency.ts | 6 +- 16 files changed, 208 insertions(+), 335 deletions(-) create mode 100644 x-pack/plugins/apm/public/components/app/correlations/use_fetch_params.ts delete mode 100644 x-pack/plugins/apm/public/hooks/use_latency_overall_distribution.ts diff --git a/x-pack/plugins/apm/common/correlations/constants.ts b/x-pack/plugins/apm/common/correlations/constants.ts index a4f5716bae04f..346e36b13ddf4 100644 --- a/x-pack/plugins/apm/common/correlations/constants.ts +++ b/x-pack/plugins/apm/common/correlations/constants.ts @@ -83,10 +83,3 @@ export const KS_TEST_THRESHOLD = 0.1; export const ERROR_CORRELATION_THRESHOLD = 0.02; export const DEFAULT_PERCENTILE_THRESHOLD = 95; - -/** - * Field stats/top values sampling constants - */ - -export const SAMPLER_TOP_TERMS_THRESHOLD = 100000; -export const SAMPLER_TOP_TERMS_SHARD_SIZE = 5000; diff --git a/x-pack/plugins/apm/common/correlations/failed_transactions_correlations/types.ts b/x-pack/plugins/apm/common/correlations/failed_transactions_correlations/types.ts index 9e567f599c1c9..8b09d45c1e1b6 100644 --- a/x-pack/plugins/apm/common/correlations/failed_transactions_correlations/types.ts +++ b/x-pack/plugins/apm/common/correlations/failed_transactions_correlations/types.ts @@ -24,7 +24,7 @@ export interface FailedTransactionsCorrelation extends FieldValuePair { export type FailedTransactionsCorrelationsImpactThreshold = typeof FAILED_TRANSACTIONS_IMPACT_THRESHOLD[keyof typeof FAILED_TRANSACTIONS_IMPACT_THRESHOLD]; -export interface FailedTransactionsCorrelationsRawResponse { +export interface FailedTransactionsCorrelationsResponse { ccsWarning: boolean; failedTransactionsCorrelations?: FailedTransactionsCorrelation[]; percentileThresholdValue?: number; diff --git a/x-pack/plugins/apm/common/correlations/latency_correlations/types.ts b/x-pack/plugins/apm/common/correlations/latency_correlations/types.ts index e13e5c48b5452..23c91554b6547 100644 --- a/x-pack/plugins/apm/common/correlations/latency_correlations/types.ts +++ b/x-pack/plugins/apm/common/correlations/latency_correlations/types.ts @@ -14,7 +14,7 @@ export interface LatencyCorrelation extends FieldValuePair { ksTest: number; } -export interface LatencyCorrelationsRawResponse { +export interface LatencyCorrelationsResponse { ccsWarning: boolean; overallHistogram?: HistogramItem[]; percentileThresholdValue?: number; diff --git a/x-pack/plugins/apm/public/components/app/correlations/failed_transactions_correlations.tsx b/x-pack/plugins/apm/public/components/app/correlations/failed_transactions_correlations.tsx index 9021bad1c62bd..56e66295eaad2 100644 --- a/x-pack/plugins/apm/public/components/app/correlations/failed_transactions_correlations.tsx +++ b/x-pack/plugins/apm/public/components/app/correlations/failed_transactions_correlations.tsx @@ -85,7 +85,6 @@ export function FailedTransactionsCorrelations({ }, {} as Record); }, [response?.fieldStats]); - const progressNormalized = progress.loaded / progress.total; const { overallHistogram, hasData, status } = getOverallHistogram( response, progress.isRunning @@ -427,7 +426,7 @@ export function FailedTransactionsCorrelations({ const showCorrelationsEmptyStatePrompt = correlationTerms.length < 1 && - (progressNormalized === 1 || !progress.isRunning); + (progress.loaded === 1 || !progress.isRunning); const transactionDistributionChartData: TransactionDistributionChartData[] = []; @@ -609,7 +608,7 @@ export function FailedTransactionsCorrelations({ ; + dataSearchResponse: IKibanaSearchResponse; }) { const mockDataSearch = jest.fn(() => of(dataSearchResponse)); diff --git a/x-pack/plugins/apm/public/components/app/correlations/latency_correlations.tsx b/x-pack/plugins/apm/public/components/app/correlations/latency_correlations.tsx index 53603c9158e43..964896bd04729 100644 --- a/x-pack/plugins/apm/public/components/app/correlations/latency_correlations.tsx +++ b/x-pack/plugins/apm/public/components/app/correlations/latency_correlations.tsx @@ -62,7 +62,6 @@ export function LatencyCorrelations({ onFilter }: { onFilter: () => void }) { const { progress, response, startFetch, cancelFetch } = useLatencyCorrelations(); - const progressNormalized = progress.loaded / progress.total; const { overallHistogram, hasData, status } = getOverallHistogram( response, progress.isRunning @@ -274,8 +273,7 @@ export function LatencyCorrelations({ onFilter }: { onFilter: () => void }) { const showCorrelationsTable = progress.isRunning || histogramTerms.length > 0; const showCorrelationsEmptyStatePrompt = - histogramTerms.length < 1 && - (progressNormalized === 1 || !progress.isRunning); + histogramTerms.length < 1 && (progress.loaded === 1 || !progress.isRunning); const transactionDistributionChartData: TransactionDistributionChartData[] = []; @@ -368,7 +366,7 @@ export function LatencyCorrelations({ onFilter }: { onFilter: () => void }) { (), - getInitialRawResponse() + const [response, setResponseUnDebounced] = useReducer( + getReducer(), + getInitialResponse() ); - const setRawResponse = useMemo(() => debounce(setRawResponseRaw, 50), []); - - const [fetchState, setFetchStateRaw] = useReducer( - getReducer(), - getInitialProgress() - ); - const setFetchState = useMemo(() => debounce(setFetchStateRaw, 50), []); + const setResponse = useMemo(() => debounce(setResponseUnDebounced, 50), []); + // We're using a ref here because otherwise the startFetch function might have + // a stale value for checking if the task has been cancelled. const isCancelledRef = useRef(false); const startFetch = useCallback(async () => { isCancelledRef.current = false; - setRawResponse(getInitialRawResponse()); - setFetchState({ - ...getInitialProgress(), + setResponse({ + ...getInitialResponse(), isRunning: true, + // explicitly set these to undefined to override a possible previous state. error: undefined, + failedTransactionsCorrelations: undefined, + percentileThresholdValue: undefined, + overallHistogram: undefined, + errorHistogram: undefined, + fieldStats: undefined, }); - setRawResponse.flush(); - setFetchState.flush(); - - const query = { - serviceName, - transactionName, - transactionType, - kuery, - environment, - start, - end, - }; + setResponse.flush(); try { - const rawResponseUpdate = (await callApmApi({ + const responseUpdate = (await callApmApi({ endpoint: 'POST /internal/apm/latency/overall_distribution', signal: null, params: { body: { - ...query, + ...fetchParams, percentileThreshold: DEFAULT_PERCENTILE_THRESHOLD, }, }, @@ -98,7 +74,7 @@ export function useFailedTransactionsCorrelations() { signal: null, params: { body: { - ...query, + ...fetchParams, percentileThreshold: DEFAULT_PERCENTILE_THRESHOLD, termFilters: [ { fieldName: EVENT_OUTCOME, fieldValue: EventOutcome.failure }, @@ -111,19 +87,18 @@ export function useFailedTransactionsCorrelations() { return; } - setRawResponse({ - ...rawResponseUpdate, + setResponse({ + ...responseUpdate, errorHistogram, + loaded: 0.05, }); - setFetchState({ - loaded: 5, - }); + setResponse.flush(); const { fieldCandidates: candidates } = await callApmApi({ endpoint: 'GET /internal/apm/correlations/field_candidates', signal: null, params: { - query, + query: fetchParams, }, }); @@ -133,15 +108,15 @@ export function useFailedTransactionsCorrelations() { const fieldCandidates = candidates.filter((t) => !(t === EVENT_OUTCOME)); - setFetchState({ - loaded: 10, + setResponse({ + loaded: 0.1, }); const failedTransactionsCorrelations: FailedTransactionsCorrelation[] = []; const fieldsToSample = new Set(); const chunkSize = 10; - let loadCounter = 0; + let chunkLoadCounter = 0; const fieldCandidatesChunks = chunk(fieldCandidates, chunkSize); @@ -150,7 +125,7 @@ export function useFailedTransactionsCorrelations() { endpoint: 'POST /internal/apm/correlations/p_values', signal: null, params: { - body: { ...query, fieldCandidates: fieldCandidatesChunk }, + body: { ...fetchParams, fieldCandidates: fieldCandidatesChunk }, }, }); @@ -161,21 +136,26 @@ export function useFailedTransactionsCorrelations() { failedTransactionsCorrelations.push( ...pValues.failedTransactionsCorrelations ); - rawResponseUpdate.failedTransactionsCorrelations = + responseUpdate.failedTransactionsCorrelations = getFailedTransactionsCorrelationsSortedByScore([ ...failedTransactionsCorrelations, ]); - setRawResponse(rawResponseUpdate); + setResponse({ + ...responseUpdate, + loaded: + 0.2 + + Math.round( + (chunkLoadCounter / fieldCandidatesChunks.length) * 80 + ) / + 100, + }); } if (isCancelledRef.current) { return; } - loadCounter += chunkSize; - setFetchState({ - loaded: 20 + Math.round((loadCounter / fieldCandidates.length) * 80), - }); + chunkLoadCounter++; } const fieldStats = await callApmApi({ @@ -183,49 +163,33 @@ export function useFailedTransactionsCorrelations() { signal: null, params: { body: { - ...query, + ...fetchParams, fieldsToSample: [...fieldsToSample], }, }, }); - rawResponseUpdate.fieldStats = fieldStats.stats; - setRawResponse(rawResponseUpdate); - - setFetchState({ - loaded: 100, - }); - setRawResponse.flush(); - setFetchState.flush(); + responseUpdate.fieldStats = fieldStats.stats; + setResponse({ ...responseUpdate, loaded: 1, isRunning: false }); + setResponse.flush(); } catch (e) { + // TODO Improve error handling // const err = e as Error | IHttpFetchError; // const message = error.body?.message ?? error.response?.statusText; - setFetchState({ + setResponse({ error: e as Error, + isRunning: false, }); + setResponse.flush(); } - - setFetchState({ - isRunning: false, - }); - }, [ - environment, - serviceName, - transactionName, - transactionType, - kuery, - start, - end, - setFetchState, - setRawResponse, - ]); + }, [fetchParams, setResponse]); const cancelFetch = useCallback(() => { isCancelledRef.current = true; - setFetchState({ + setResponse({ isRunning: false, }); - }, [setFetchState]); + }, [setResponse]); // auto-update useEffect(() => { @@ -233,9 +197,19 @@ export function useFailedTransactionsCorrelations() { return cancelFetch; }, [startFetch, cancelFetch]); + const { error, loaded, isRunning, ...returnedResponse } = response; + const progress = useMemo( + () => ({ + error, + loaded, + isRunning, + }), + [error, loaded, isRunning] + ); + return { - progress: fetchState, - response: rawResponse, + progress, + response: returnedResponse, startFetch, cancelFetch, }; diff --git a/x-pack/plugins/apm/public/components/app/correlations/use_fetch_params.ts b/x-pack/plugins/apm/public/components/app/correlations/use_fetch_params.ts new file mode 100644 index 0000000000000..8c7dea16af456 --- /dev/null +++ b/x-pack/plugins/apm/public/components/app/correlations/use_fetch_params.ts @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { useMemo } from 'react'; + +import { useApmServiceContext } from '../../../context/apm_service/use_apm_service_context'; +import { useUrlParams } from '../../../context/url_params_context/use_url_params'; + +import { useApmParams } from '../../../hooks/use_apm_params'; +import { useTimeRange } from '../../../hooks/use_time_range'; + +export const useFetchParams = () => { + const { serviceName, transactionType } = useApmServiceContext(); + + const { urlParams } = useUrlParams(); + const { transactionName } = urlParams; + + const { + query: { kuery, environment, rangeFrom, rangeTo }, + } = useApmParams('/services/{serviceName}/transactions/view'); + + const { start, end } = useTimeRange({ rangeFrom, rangeTo }); + + return useMemo( + () => ({ + serviceName, + transactionName, + transactionType, + kuery, + environment, + start, + end, + }), + [ + serviceName, + transactionName, + transactionType, + kuery, + environment, + start, + end, + ] + ); +}; diff --git a/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts b/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts index 393bcd09b4f08..eba8b4f90b7f0 100644 --- a/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts +++ b/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts @@ -13,81 +13,56 @@ import type { FieldValuePair } from '../../../../common/correlations/types'; import { getPrioritizedFieldValuePairs } from '../../../../common/correlations/utils'; import type { LatencyCorrelation, - LatencyCorrelationsRawResponse, + LatencyCorrelationsResponse, } from '../../../../common/correlations/latency_correlations/types'; -import { useApmServiceContext } from '../../../context/apm_service/use_apm_service_context'; -import { useUrlParams } from '../../../context/url_params_context/use_url_params'; - -import { useApmParams } from '../../../hooks/use_apm_params'; -import { useTimeRange } from '../../../hooks/use_time_range'; import { callApmApi } from '../../../services/rest/createCallApmApi'; import { - getInitialProgress, + getInitialResponse, getLatencyCorrelationsSortedByCorrelation, - getInitialRawResponse, getReducer, CorrelationsProgress, } from './utils/analysis_hook_utils'; +import { useFetchParams } from './use_fetch_params'; -type Response = LatencyCorrelationsRawResponse; +type Response = LatencyCorrelationsResponse; export function useLatencyCorrelations() { - const { serviceName, transactionType } = useApmServiceContext(); - - const { urlParams } = useUrlParams(); - const { transactionName } = urlParams; - - const { - query: { kuery, environment, rangeFrom, rangeTo }, - } = useApmParams('/services/{serviceName}/transactions/view'); + const fetchParams = useFetchParams(); - const { start, end } = useTimeRange({ rangeFrom, rangeTo }); - - const [rawResponse, setRawResponseRaw] = useReducer( - getReducer(), - getInitialRawResponse() + const [response, setResponseUnDebounced] = useReducer( + getReducer(), + getInitialResponse() ); - const setRawResponse = useMemo(() => debounce(setRawResponseRaw, 50), []); - - const [fetchState, setFetchStateRaw] = useReducer( - getReducer(), - getInitialProgress() - ); - const setFetchState = useMemo(() => debounce(setFetchStateRaw, 50), []); + const setResponse = useMemo(() => debounce(setResponseUnDebounced, 100), []); + // We're using a ref here because otherwise the startFetch function might have + // a stale value for checking if the task has been cancelled. const isCancelledRef = useRef(false); const startFetch = useCallback(async () => { isCancelledRef.current = false; - setRawResponse(getInitialRawResponse()); - setFetchState({ - ...getInitialProgress(), + setResponse({ + ...getInitialResponse(), isRunning: true, + // explicitly set these to undefined to override a possible previous state. error: undefined, + latencyCorrelations: undefined, + percentileThresholdValue: undefined, + overallHistogram: undefined, + fieldStats: undefined, }); - setRawResponse.flush(); - setFetchState.flush(); - - const query = { - serviceName, - transactionName, - transactionType, - kuery, - environment, - start, - end, - }; + setResponse.flush(); try { - const rawResponseUpdate = (await callApmApi({ + const responseUpdate = (await callApmApi({ endpoint: 'POST /internal/apm/latency/overall_distribution', signal: null, params: { body: { - ...query, + ...fetchParams, percentileThreshold: DEFAULT_PERCENTILE_THRESHOLD + '', }, }, @@ -97,16 +72,17 @@ export function useLatencyCorrelations() { return; } - setRawResponse(rawResponseUpdate); - setFetchState({ - loaded: 5, + setResponse({ + ...responseUpdate, + loaded: 0.05, }); + setResponse.flush(); const { fieldCandidates } = await callApmApi({ endpoint: 'GET /internal/apm/correlations/field_candidates', signal: null, params: { - query, + query: fetchParams, }, }); @@ -114,8 +90,8 @@ export function useLatencyCorrelations() { return; } - setFetchState({ - loaded: 10, + setResponse({ + loaded: 0.1, }); const chunkSize = 10; @@ -130,7 +106,7 @@ export function useLatencyCorrelations() { signal: null, params: { query: { - ...query, + ...fetchParams, fieldCandidates: fieldCandidateChunk, }, }, @@ -144,21 +120,19 @@ export function useLatencyCorrelations() { return; } - setFetchState({ + setResponse({ loaded: - 10 + Math.round((chunkLoadCounter / fieldValuePairs.length) * 10), + 0.1 + + Math.round((chunkLoadCounter / fieldCandidateChunks.length) * 30) / + 100, }); - chunkLoadCounter += chunkSize; + chunkLoadCounter++; } if (isCancelledRef.current) { return; } - setFetchState({ - loaded: 20, - }); - chunkLoadCounter = 0; const fieldsToSample = new Set(); @@ -173,7 +147,7 @@ export function useLatencyCorrelations() { endpoint: 'POST /internal/apm/correlations/significant_correlations', signal: null, params: { - body: { ...query, fieldValuePairs: fieldValuePairChunk }, + body: { ...fetchParams, fieldValuePairs: fieldValuePairChunk }, }, }); @@ -184,20 +158,24 @@ export function useLatencyCorrelations() { latencyCorrelations.push( ...significantCorrelations.latencyCorrelations ); - rawResponseUpdate.latencyCorrelations = + responseUpdate.latencyCorrelations = getLatencyCorrelationsSortedByCorrelation([...latencyCorrelations]); - setRawResponse(rawResponseUpdate); + setResponse({ + ...responseUpdate, + loaded: + 0.4 + + Math.round( + (chunkLoadCounter / fieldValuePairChunks.length) * 60 + ) / + 100, + }); } if (isCancelledRef.current) { return; } - setFetchState({ - loaded: - 20 + Math.round((chunkLoadCounter / fieldValuePairs.length) * 80), - }); - chunkLoadCounter += chunkSize; + chunkLoadCounter++; } const fieldStats = await callApmApi({ @@ -205,49 +183,33 @@ export function useLatencyCorrelations() { signal: null, params: { body: { - ...query, + ...fetchParams, fieldsToSample: [...fieldsToSample], }, }, }); - rawResponseUpdate.fieldStats = fieldStats.stats; - setRawResponse(rawResponseUpdate); - - setFetchState({ - loaded: 100, - }); - setRawResponse.flush(); - setFetchState.flush(); + responseUpdate.fieldStats = fieldStats.stats; + setResponse({ ...responseUpdate, loaded: 1, isRunning: false }); + setResponse.flush(); } catch (e) { + // TODO Improve error handling // const err = e as Error | IHttpFetchError; // const message = error.body?.message ?? error.response?.statusText; - setFetchState({ + setResponse({ error: e as Error, + isRunning: false, }); + setResponse.flush(); } - - setFetchState({ - isRunning: false, - }); - }, [ - environment, - serviceName, - transactionName, - transactionType, - kuery, - start, - end, - setRawResponse, - setFetchState, - ]); + }, [fetchParams, setResponse]); const cancelFetch = useCallback(() => { isCancelledRef.current = true; - setFetchState({ + setResponse({ isRunning: false, }); - }, [setFetchState]); + }, [setResponse]); // auto-update useEffect(() => { @@ -255,9 +217,19 @@ export function useLatencyCorrelations() { return cancelFetch; }, [startFetch, cancelFetch]); + const { error, loaded, isRunning, ...returnedResponse } = response; + const progress = useMemo( + () => ({ + error, + loaded, + isRunning, + }), + [error, loaded, isRunning] + ); + return { - progress: fetchState, - response: rawResponse, + progress, + response: returnedResponse, startFetch, cancelFetch, }; diff --git a/x-pack/plugins/apm/public/components/app/correlations/utils/analysis_hook_utils.ts b/x-pack/plugins/apm/public/components/app/correlations/utils/analysis_hook_utils.ts index aaba8f4509fb1..51f4a0a48cbe8 100644 --- a/x-pack/plugins/apm/public/components/app/correlations/utils/analysis_hook_utils.ts +++ b/x-pack/plugins/apm/public/components/app/correlations/utils/analysis_hook_utils.ts @@ -14,7 +14,6 @@ export interface CorrelationsProgress { error?: Error | IHttpFetchError; isRunning: boolean; loaded: number; - total: number; } export function getLatencyCorrelationsSortedByCorrelation( @@ -29,14 +28,10 @@ export function getFailedTransactionsCorrelationsSortedByScore( return failedTransactionsCorrelations.sort((a, b) => b.score - a.score); } -export const getInitialRawResponse = () => ({ +export const getInitialResponse = () => ({ ccsWarning: false, -}); - -export const getInitialProgress = (): CorrelationsProgress => ({ isRunning: false, loaded: 0, - total: 100, }); export const getReducer = diff --git a/x-pack/plugins/apm/public/components/app/correlations/utils/get_overall_histogram.test.ts b/x-pack/plugins/apm/public/components/app/correlations/utils/get_overall_histogram.test.ts index abf2bd9bec563..b76777b660d8f 100644 --- a/x-pack/plugins/apm/public/components/app/correlations/utils/get_overall_histogram.test.ts +++ b/x-pack/plugins/apm/public/components/app/correlations/utils/get_overall_histogram.test.ts @@ -5,14 +5,14 @@ * 2.0. */ -import type { LatencyCorrelationsRawResponse } from '../../../../../common/correlations/latency_correlations/types'; +import type { LatencyCorrelationsResponse } from '../../../../../common/correlations/latency_correlations/types'; import { getOverallHistogram } from './get_overall_histogram'; describe('getOverallHistogram', () => { it('returns "loading" when undefined and running', () => { const { overallHistogram, hasData, status } = getOverallHistogram( - {} as LatencyCorrelationsRawResponse, + {} as LatencyCorrelationsResponse, true ); expect(overallHistogram).toStrictEqual(undefined); @@ -22,7 +22,7 @@ describe('getOverallHistogram', () => { it('returns "success" when undefined and not running', () => { const { overallHistogram, hasData, status } = getOverallHistogram( - {} as LatencyCorrelationsRawResponse, + {} as LatencyCorrelationsResponse, false ); expect(overallHistogram).toStrictEqual([]); @@ -34,7 +34,7 @@ describe('getOverallHistogram', () => { const { overallHistogram, hasData, status } = getOverallHistogram( { overallHistogram: [{ key: 1, doc_count: 1234 }], - } as LatencyCorrelationsRawResponse, + } as LatencyCorrelationsResponse, true ); expect(overallHistogram).toStrictEqual([{ key: 1, doc_count: 1234 }]); @@ -46,7 +46,7 @@ describe('getOverallHistogram', () => { const { overallHistogram, hasData, status } = getOverallHistogram( { overallHistogram: [{ key: 1, doc_count: 1234 }], - } as LatencyCorrelationsRawResponse, + } as LatencyCorrelationsResponse, false ); expect(overallHistogram).toStrictEqual([{ key: 1, doc_count: 1234 }]); diff --git a/x-pack/plugins/apm/public/components/app/correlations/utils/get_overall_histogram.ts b/x-pack/plugins/apm/public/components/app/correlations/utils/get_overall_histogram.ts index 36b9f2e1188c3..3a6a2704b3984 100644 --- a/x-pack/plugins/apm/public/components/app/correlations/utils/get_overall_histogram.ts +++ b/x-pack/plugins/apm/public/components/app/correlations/utils/get_overall_histogram.ts @@ -5,7 +5,7 @@ * 2.0. */ -import type { LatencyCorrelationsRawResponse } from '../../../../../common/correlations/latency_correlations/types'; +import type { LatencyCorrelationsResponse } from '../../../../../common/correlations/latency_correlations/types'; import { FETCH_STATUS } from '../../../../hooks/use_fetcher'; @@ -13,7 +13,7 @@ import { FETCH_STATUS } from '../../../../hooks/use_fetcher'; // of fetching more data such as correlation results. That's why we have to determine // the `status` of the data for the latency chart separately. export function getOverallHistogram( - data: LatencyCorrelationsRawResponse, + data: LatencyCorrelationsResponse, isRunning: boolean ) { const overallHistogram = diff --git a/x-pack/plugins/apm/public/components/app/transaction_details/distribution/use_transaction_distribution_chart_data.ts b/x-pack/plugins/apm/public/components/app/transaction_details/distribution/use_transaction_distribution_chart_data.ts index 7f005f662fc57..c5f56476a0ad8 100644 --- a/x-pack/plugins/apm/public/components/app/transaction_details/distribution/use_transaction_distribution_chart_data.ts +++ b/x-pack/plugins/apm/public/components/app/transaction_details/distribution/use_transaction_distribution_chart_data.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { useEffect, useMemo } from 'react'; +import { useEffect } from 'react'; import { i18n } from '@kbn/i18n'; @@ -14,53 +14,20 @@ import { EVENT_OUTCOME } from '../../../../../common/elasticsearch_fieldnames'; import { EventOutcome } from '../../../../../common/event_outcome'; import { useApmPluginContext } from '../../../../context/apm_plugin/use_apm_plugin_context'; -import { useApmServiceContext } from '../../../../context/apm_service/use_apm_service_context'; -import { useUrlParams } from '../../../../context/url_params_context/use_url_params'; -import { useApmParams } from '../../../../hooks/use_apm_params'; import { useFetcher, FETCH_STATUS } from '../../../../hooks/use_fetcher'; -import { useTimeRange } from '../../../../hooks/use_time_range'; import type { TransactionDistributionChartData } from '../../../shared/charts/transaction_distribution_chart'; import { isErrorMessage } from '../../correlations/utils/is_error_message'; +import { useFetchParams } from '../../correlations/use_fetch_params'; export const useTransactionDistributionChartData = () => { - const { serviceName, transactionType } = useApmServiceContext(); + const params = useFetchParams(); const { core: { notifications }, } = useApmPluginContext(); - const { urlParams } = useUrlParams(); - const { transactionName } = urlParams; - - const { - query: { kuery, environment, rangeFrom, rangeTo }, - } = useApmParams('/services/{serviceName}/transactions/view'); - - const { start, end } = useTimeRange({ rangeFrom, rangeTo }); - - const params = useMemo( - () => ({ - serviceName, - transactionName, - transactionType, - kuery, - environment, - start, - end, - }), - [ - serviceName, - transactionName, - transactionType, - kuery, - environment, - start, - end, - ] - ); - const { data: overallLatencyData = {}, status: overallLatencyStatus, diff --git a/x-pack/plugins/apm/public/hooks/use_latency_overall_distribution.ts b/x-pack/plugins/apm/public/hooks/use_latency_overall_distribution.ts deleted file mode 100644 index 3a9db0f7b75cb..0000000000000 --- a/x-pack/plugins/apm/public/hooks/use_latency_overall_distribution.ts +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { DEFAULT_PERCENTILE_THRESHOLD } from '../../common/correlations/constants'; - -import { useApmServiceContext } from '../context/apm_service/use_apm_service_context'; -import { useUrlParams } from '../context/url_params_context/use_url_params'; - -import { useApmParams } from './use_apm_params'; -import { useFetcher, FETCH_STATUS } from './use_fetcher'; -import { useTimeRange } from './use_time_range'; - -export function useLatencyOverallDistribution() { - const { serviceName, transactionType } = useApmServiceContext(); - - const { urlParams } = useUrlParams(); - const { transactionName } = urlParams; - - const { - query: { kuery, environment, rangeFrom, rangeTo }, - } = useApmParams('/services/{serviceName}/transactions/view'); - - const { start, end } = useTimeRange({ rangeFrom, rangeTo }); - - const { - data = {}, - status, - error, - } = useFetcher( - (callApmApi) => { - if (serviceName && environment && start && end) { - return callApmApi({ - endpoint: 'POST /internal/apm/latency/overall_distribution', - params: { - body: { - serviceName, - transactionName, - transactionType, - kuery, - environment, - start, - end, - percentileThreshold: DEFAULT_PERCENTILE_THRESHOLD, - }, - }, - }); - } - }, - [ - serviceName, - transactionName, - transactionType, - kuery, - environment, - start, - end, - ] - ); - - const { percentileThresholdValue } = data; - const overallHistogram = - data.overallHistogram === undefined && status !== FETCH_STATUS.LOADING - ? [] - : data.overallHistogram; - const hasData = - Array.isArray(overallHistogram) && overallHistogram.length > 0; - - return { error, hasData, overallHistogram, percentileThresholdValue, status }; -} diff --git a/x-pack/test/apm_api_integration/tests/correlations/failed_transactions.ts b/x-pack/test/apm_api_integration/tests/correlations/failed_transactions.ts index 659cf85096138..8d8410495bbd9 100644 --- a/x-pack/test/apm_api_integration/tests/correlations/failed_transactions.ts +++ b/x-pack/test/apm_api_integration/tests/correlations/failed_transactions.ts @@ -9,7 +9,7 @@ import expect from '@kbn/expect'; import { FtrProviderContext } from '../../common/ftr_provider_context'; import { registry } from '../../common/registry'; -import type { FailedTransactionsCorrelationsRawResponse } from '../../../../plugins/apm/common/correlations/failed_transactions_correlations/types'; +import type { FailedTransactionsCorrelationsResponse } from '../../../../plugins/apm/common/correlations/failed_transactions_correlations/types'; import { EVENT_OUTCOME } from '../../../../plugins/apm/common/elasticsearch_fieldnames'; import { EventOutcome } from '../../../../plugins/apm/common/event_outcome'; @@ -86,7 +86,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { `Expected status to be '200', got '${failedTransactionsCorrelationsResponse.status}'` ); - const finalRawResponse: FailedTransactionsCorrelationsRawResponse = { + const finalRawResponse: FailedTransactionsCorrelationsResponse = { ccsWarning: failedTransactionsCorrelationsResponse.body?.ccsWarning, percentileThresholdValue: overallDistributionResponse.body?.percentileThresholdValue, overallHistogram: overallDistributionResponse.body?.overallHistogram, @@ -171,7 +171,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { `Expected status to be '200', got '${failedTransactionsCorrelationsResponse.status}'` ); - const finalRawResponse: FailedTransactionsCorrelationsRawResponse = { + const finalRawResponse: FailedTransactionsCorrelationsResponse = { ccsWarning: failedTransactionsCorrelationsResponse.body?.ccsWarning, percentileThresholdValue: overallDistributionResponse.body?.percentileThresholdValue, overallHistogram: overallDistributionResponse.body?.overallHistogram, diff --git a/x-pack/test/apm_api_integration/tests/correlations/latency.ts b/x-pack/test/apm_api_integration/tests/correlations/latency.ts index a9c59fa79f3ec..bb5cfa0fb7cee 100644 --- a/x-pack/test/apm_api_integration/tests/correlations/latency.ts +++ b/x-pack/test/apm_api_integration/tests/correlations/latency.ts @@ -9,7 +9,7 @@ import expect from '@kbn/expect'; import { FtrProviderContext } from '../../common/ftr_provider_context'; import { registry } from '../../common/registry'; -import type { LatencyCorrelationsRawResponse } from '../../../../plugins/apm/common/correlations/latency_correlations/types'; +import type { LatencyCorrelationsResponse } from '../../../../plugins/apm/common/correlations/latency_correlations/types'; // These tests go through the full sequence of queries required // to get the final results for a latency correlation analysis. @@ -86,7 +86,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { `Expected status to be '200', got '${significantCorrelationsResponse.status}'` ); - const finalRawResponse: LatencyCorrelationsRawResponse = { + const finalRawResponse: LatencyCorrelationsResponse = { ccsWarning: significantCorrelationsResponse.body?.ccsWarning, percentileThresholdValue: overallDistributionResponse.body?.percentileThresholdValue, overallHistogram: overallDistributionResponse.body?.overallHistogram, @@ -181,7 +181,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { `Expected 1244 total doc count, got ${significantCorrelationsResponse.body?.totalDocCount}.` ); - const finalRawResponse: LatencyCorrelationsRawResponse = { + const finalRawResponse: LatencyCorrelationsResponse = { ccsWarning: significantCorrelationsResponse.body?.ccsWarning, percentileThresholdValue: overallDistributionResponse.body?.percentileThresholdValue, overallHistogram: overallDistributionResponse.body?.overallHistogram, From 9ccd53b4f6f81490553b8e79c645699fe5d39cae Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Wed, 20 Oct 2021 15:33:09 +0200 Subject: [PATCH 21/39] [ML] Add comments. --- .../app/correlations/use_failed_transactions_correlations.ts | 2 ++ .../components/app/correlations/use_latency_correlations.ts | 2 ++ 2 files changed, 4 insertions(+) diff --git a/x-pack/plugins/apm/public/components/app/correlations/use_failed_transactions_correlations.ts b/x-pack/plugins/apm/public/components/app/correlations/use_failed_transactions_correlations.ts index bc55927c98683..cd790138c446b 100644 --- a/x-pack/plugins/apm/public/components/app/correlations/use_failed_transactions_correlations.ts +++ b/x-pack/plugins/apm/public/components/app/correlations/use_failed_transactions_correlations.ts @@ -31,6 +31,8 @@ type Response = FailedTransactionsCorrelationsResponse; export function useFailedTransactionsCorrelations() { const fetchParams = useFetchParams(); + // This use of useReducer (the dispatch function won't get reinstantiated + // on every update) and debounce avoids flooding consuming components with updates. const [response, setResponseUnDebounced] = useReducer( getReducer(), getInitialResponse() diff --git a/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts b/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts index eba8b4f90b7f0..83f548697ff5a 100644 --- a/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts +++ b/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts @@ -31,6 +31,8 @@ type Response = LatencyCorrelationsResponse; export function useLatencyCorrelations() { const fetchParams = useFetchParams(); + // This use of useReducer (the dispatch function won't get reinstantiated + // on every update) and debounce avoids flooding consuming components with updates. const [response, setResponseUnDebounced] = useReducer( getReducer(), getInitialResponse() From d15d18e22dc90edd6a35a9a5290cd1d71f3d55e0 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Wed, 20 Oct 2021 18:03:31 +0200 Subject: [PATCH 22/39] [ML] Fix chart labels. --- .../failed_transactions_correlations.tsx | 12 ++++++------ .../app/transaction_details/distribution/index.tsx | 8 ++++---- .../use_transaction_distribution_chart_data.ts | 4 ++-- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/x-pack/plugins/apm/public/components/app/correlations/failed_transactions_correlations.tsx b/x-pack/plugins/apm/public/components/app/correlations/failed_transactions_correlations.tsx index 56e66295eaad2..56ca110b5e7b8 100644 --- a/x-pack/plugins/apm/public/components/app/correlations/failed_transactions_correlations.tsx +++ b/x-pack/plugins/apm/public/components/app/correlations/failed_transactions_correlations.tsx @@ -444,8 +444,8 @@ export function FailedTransactionsCorrelations({ if (Array.isArray(response.errorHistogram)) { transactionDistributionChartData.push({ id: i18n.translate( - 'xpack.apm.transactionDistribution.chart.allFailedTransactionsLabel', - { defaultMessage: 'All failed transactions' } + 'xpack.apm.transactionDistribution.chart.failedTransactionsLabel', + { defaultMessage: 'Failed transactions' } ), histogram: response.errorHistogram, }); @@ -512,7 +512,7 @@ export function FailedTransactionsCorrelations({ , allTransactions: ( @@ -523,13 +523,13 @@ export function FailedTransactionsCorrelations({ /> ), - allFailedTransactions: ( + failedTransactions: ( ), diff --git a/x-pack/plugins/apm/public/components/app/transaction_details/distribution/index.tsx b/x-pack/plugins/apm/public/components/app/transaction_details/distribution/index.tsx index 0b58db492b472..1f5fdc6ec5170 100644 --- a/x-pack/plugins/apm/public/components/app/transaction_details/distribution/index.tsx +++ b/x-pack/plugins/apm/public/components/app/transaction_details/distribution/index.tsx @@ -165,7 +165,7 @@ export function TransactionDistribution({ @@ -175,13 +175,13 @@ export function TransactionDistribution({ /> ), - allFailedTransactions: ( + failedTransactions: ( ), diff --git a/x-pack/plugins/apm/public/components/app/transaction_details/distribution/use_transaction_distribution_chart_data.ts b/x-pack/plugins/apm/public/components/app/transaction_details/distribution/use_transaction_distribution_chart_data.ts index c5f56476a0ad8..a02fc7fe6665f 100644 --- a/x-pack/plugins/apm/public/components/app/transaction_details/distribution/use_transaction_distribution_chart_data.ts +++ b/x-pack/plugins/apm/public/components/app/transaction_details/distribution/use_transaction_distribution_chart_data.ts @@ -138,8 +138,8 @@ export const useTransactionDistributionChartData = () => { if (Array.isArray(errorHistogramData.overallHistogram)) { transactionDistributionChartData.push({ id: i18n.translate( - 'xpack.apm.transactionDistribution.chart.allFailedTransactionsLabel', - { defaultMessage: 'All failed transactions' } + 'xpack.apm.transactionDistribution.chart.failedTransactionsLabel', + { defaultMessage: 'Failed transactions' } ), histogram: errorHistogramData.overallHistogram, }); From 8bb304bd0fa0884b1507b0f383469c4ca848df74 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Wed, 20 Oct 2021 18:04:28 +0200 Subject: [PATCH 23/39] [ML] Add platinum license guard to correlations API endpoints. --- .../plugins/apm/server/routes/correlations.ts | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/x-pack/plugins/apm/server/routes/correlations.ts b/x-pack/plugins/apm/server/routes/correlations.ts index e64e802187e45..5c0c285f34229 100644 --- a/x-pack/plugins/apm/server/routes/correlations.ts +++ b/x-pack/plugins/apm/server/routes/correlations.ts @@ -6,9 +6,13 @@ */ import * as t from 'io-ts'; +import Boom from '@hapi/boom'; +import { i18n } from '@kbn/i18n'; import { toNumberRt } from '@kbn/io-ts-utils'; +import { isActivePlatinumLicense } from '../../common/license_check'; + import { setupRequest } from '../lib/helpers/setup_request'; import { fetchPValues, @@ -24,6 +28,11 @@ import { createApmServerRoute } from './create_apm_server_route'; import { createApmServerRouteRepository } from './create_apm_server_route_repository'; import { environmentRt, kueryRt, rangeRt } from './default_api_types'; +const INVALID_LICENSE = i18n.translate('xpack.apm.correlations.license.text', { + defaultMessage: + 'To use the correlations API, you must be subscribed to an Elastic Platinum license.', +}); + const fieldCandidatesRoute = createApmServerRoute({ endpoint: 'GET /internal/apm/correlations/field_candidates', params: t.type({ @@ -40,6 +49,11 @@ const fieldCandidatesRoute = createApmServerRoute({ }), options: { tags: ['access:apm'] }, handler: async (resources) => { + const { context } = resources; + if (!isActivePlatinumLicense(context.licensing.license)) { + throw Boom.forbidden(INVALID_LICENSE); + } + const { indices } = await setupRequest(resources); const esClient = resources.context.core.elasticsearch.client.asCurrentUser; @@ -73,6 +87,11 @@ const fieldStatsRoute = createApmServerRoute({ }), options: { tags: ['access:apm'] }, handler: async (resources) => { + const { context } = resources; + if (!isActivePlatinumLicense(context.licensing.license)) { + throw Boom.forbidden(INVALID_LICENSE); + } + const { indices } = await setupRequest(resources); const esClient = resources.context.core.elasticsearch.client.asCurrentUser; @@ -112,6 +131,11 @@ const fieldValuePairsRoute = createApmServerRoute({ }), options: { tags: ['access:apm'] }, handler: async (resources) => { + const { context } = resources; + if (!isActivePlatinumLicense(context.licensing.license)) { + throw Boom.forbidden(INVALID_LICENSE); + } + const { indices } = await setupRequest(resources); const esClient = resources.context.core.elasticsearch.client.asCurrentUser; @@ -154,6 +178,11 @@ const significantCorrelationsRoute = createApmServerRoute({ }), options: { tags: ['access:apm'] }, handler: async (resources) => { + const { context } = resources; + if (!isActivePlatinumLicense(context.licensing.license)) { + throw Boom.forbidden(INVALID_LICENSE); + } + const { indices } = await setupRequest(resources); const esClient = resources.context.core.elasticsearch.client.asCurrentUser; @@ -195,6 +224,11 @@ const pValuesRoute = createApmServerRoute({ }), options: { tags: ['access:apm'] }, handler: async (resources) => { + const { context } = resources; + if (!isActivePlatinumLicense(context.licensing.license)) { + throw Boom.forbidden(INVALID_LICENSE); + } + const { indices } = await setupRequest(resources); const esClient = resources.context.core.elasticsearch.client.asCurrentUser; From da37fd36b1e4196e0b5e0d0db5224c62713fd7ba Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Fri, 29 Oct 2021 14:44:01 +0200 Subject: [PATCH 24/39] [ML] Move progress parts to constants. --- .../apm/common/correlations/constants.ts | 1 + .../use_failed_transactions_correlations.ts | 39 +++++++++----- .../correlations/use_latency_correlations.ts | 53 ++++++++++++------- 3 files changed, 61 insertions(+), 32 deletions(-) diff --git a/x-pack/plugins/apm/common/correlations/constants.ts b/x-pack/plugins/apm/common/correlations/constants.ts index 346e36b13ddf4..11b9a9a109dbf 100644 --- a/x-pack/plugins/apm/common/correlations/constants.ts +++ b/x-pack/plugins/apm/common/correlations/constants.ts @@ -83,3 +83,4 @@ export const KS_TEST_THRESHOLD = 0.1; export const ERROR_CORRELATION_THRESHOLD = 0.02; export const DEFAULT_PERCENTILE_THRESHOLD = 95; +export const DEBOUNCE_INTERVAL = 100; diff --git a/x-pack/plugins/apm/public/components/app/correlations/use_failed_transactions_correlations.ts b/x-pack/plugins/apm/public/components/app/correlations/use_failed_transactions_correlations.ts index cd790138c446b..789702ac121f8 100644 --- a/x-pack/plugins/apm/public/components/app/correlations/use_failed_transactions_correlations.ts +++ b/x-pack/plugins/apm/public/components/app/correlations/use_failed_transactions_correlations.ts @@ -10,7 +10,10 @@ import { chunk, debounce } from 'lodash'; import { EVENT_OUTCOME } from '../../../../common/elasticsearch_fieldnames'; import { EventOutcome } from '../../../../common/event_outcome'; -import { DEFAULT_PERCENTILE_THRESHOLD } from '../../../../common/correlations/constants'; +import { + DEBOUNCE_INTERVAL, + DEFAULT_PERCENTILE_THRESHOLD, +} from '../../../../common/correlations/constants'; import type { FailedTransactionsCorrelation, FailedTransactionsCorrelationsResponse, @@ -28,6 +31,12 @@ import { useFetchParams } from './use_fetch_params'; type Response = FailedTransactionsCorrelationsResponse; +// Overall progress is a float from 0 to 1. +const LOADED_OVERALL_HISTOGRAM = 0.05; +const LOADED_FIELD_CANDIDATES = LOADED_OVERALL_HISTOGRAM + 0.05; +const LOADED_DONE = 1; +const PROGRESS_STEP_P_VALUES = 0.8; + export function useFailedTransactionsCorrelations() { const fetchParams = useFetchParams(); @@ -37,7 +46,10 @@ export function useFailedTransactionsCorrelations() { getReducer(), getInitialResponse() ); - const setResponse = useMemo(() => debounce(setResponseUnDebounced, 50), []); + const setResponse = useMemo( + () => debounce(setResponseUnDebounced, DEBOUNCE_INTERVAL), + [] + ); // We're using a ref here because otherwise the startFetch function might have // a stale value for checking if the task has been cancelled. @@ -92,7 +104,7 @@ export function useFailedTransactionsCorrelations() { setResponse({ ...responseUpdate, errorHistogram, - loaded: 0.05, + loaded: LOADED_OVERALL_HISTOGRAM, }); setResponse.flush(); @@ -111,7 +123,7 @@ export function useFailedTransactionsCorrelations() { const fieldCandidates = candidates.filter((t) => !(t === EVENT_OUTCOME)); setResponse({ - loaded: 0.1, + loaded: LOADED_FIELD_CANDIDATES, }); const failedTransactionsCorrelations: FailedTransactionsCorrelation[] = @@ -142,17 +154,16 @@ export function useFailedTransactionsCorrelations() { getFailedTransactionsCorrelationsSortedByScore([ ...failedTransactionsCorrelations, ]); - setResponse({ - ...responseUpdate, - loaded: - 0.2 + - Math.round( - (chunkLoadCounter / fieldCandidatesChunks.length) * 80 - ) / - 100, - }); } + setResponse({ + ...responseUpdate, + loaded: + LOADED_FIELD_CANDIDATES + + (chunkLoadCounter / fieldCandidatesChunks.length) * + PROGRESS_STEP_P_VALUES, + }); + if (isCancelledRef.current) { return; } @@ -172,7 +183,7 @@ export function useFailedTransactionsCorrelations() { }); responseUpdate.fieldStats = fieldStats.stats; - setResponse({ ...responseUpdate, loaded: 1, isRunning: false }); + setResponse({ ...responseUpdate, loaded: LOADED_DONE, isRunning: false }); setResponse.flush(); } catch (e) { // TODO Improve error handling diff --git a/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts b/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts index 83f548697ff5a..cde1526cde9e7 100644 --- a/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts +++ b/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts @@ -8,7 +8,10 @@ import { useCallback, useEffect, useMemo, useReducer, useRef } from 'react'; import { chunk, debounce } from 'lodash'; -import { DEFAULT_PERCENTILE_THRESHOLD } from '../../../../common/correlations/constants'; +import { + DEBOUNCE_INTERVAL, + DEFAULT_PERCENTILE_THRESHOLD, +} from '../../../../common/correlations/constants'; import type { FieldValuePair } from '../../../../common/correlations/types'; import { getPrioritizedFieldValuePairs } from '../../../../common/correlations/utils'; import type { @@ -28,6 +31,14 @@ import { useFetchParams } from './use_fetch_params'; type Response = LatencyCorrelationsResponse; +// Overall progress is a float from 0 to 1. +const LOADED_OVERALL_HISTOGRAM = 0.05; +const LOADED_FIELD_CANDIDATES = LOADED_OVERALL_HISTOGRAM + 0.05; +const LOADED_FIELD_VALUE_PAIRS = LOADED_FIELD_CANDIDATES + 0.3; +const LOADED_DONE = 1; +const PROGRESS_STEP_FIELD_VALUE_PAIRS = 0.3; +const PROGRESS_STEP_CORRELATIONS = 0.6; + export function useLatencyCorrelations() { const fetchParams = useFetchParams(); @@ -37,7 +48,10 @@ export function useLatencyCorrelations() { getReducer(), getInitialResponse() ); - const setResponse = useMemo(() => debounce(setResponseUnDebounced, 100), []); + const setResponse = useMemo( + () => debounce(setResponseUnDebounced, DEBOUNCE_INTERVAL), + [] + ); // We're using a ref here because otherwise the startFetch function might have // a stale value for checking if the task has been cancelled. @@ -76,7 +90,7 @@ export function useLatencyCorrelations() { setResponse({ ...responseUpdate, - loaded: 0.05, + loaded: LOADED_OVERALL_HISTOGRAM, }); setResponse.flush(); @@ -93,7 +107,7 @@ export function useLatencyCorrelations() { } setResponse({ - loaded: 0.1, + loaded: LOADED_FIELD_CANDIDATES, }); const chunkSize = 10; @@ -124,9 +138,9 @@ export function useLatencyCorrelations() { setResponse({ loaded: - 0.1 + - Math.round((chunkLoadCounter / fieldCandidateChunks.length) * 30) / - 100, + LOADED_FIELD_CANDIDATES + + (chunkLoadCounter / fieldCandidateChunks.length) * + PROGRESS_STEP_FIELD_VALUE_PAIRS, }); chunkLoadCounter++; } @@ -162,17 +176,16 @@ export function useLatencyCorrelations() { ); responseUpdate.latencyCorrelations = getLatencyCorrelationsSortedByCorrelation([...latencyCorrelations]); - setResponse({ - ...responseUpdate, - loaded: - 0.4 + - Math.round( - (chunkLoadCounter / fieldValuePairChunks.length) * 60 - ) / - 100, - }); } + setResponse({ + ...responseUpdate, + loaded: + LOADED_FIELD_VALUE_PAIRS + + (chunkLoadCounter / fieldValuePairChunks.length) * + PROGRESS_STEP_CORRELATIONS, + }); + if (isCancelledRef.current) { return; } @@ -192,7 +205,11 @@ export function useLatencyCorrelations() { }); responseUpdate.fieldStats = fieldStats.stats; - setResponse({ ...responseUpdate, loaded: 1, isRunning: false }); + setResponse({ + ...responseUpdate, + loaded: LOADED_DONE, + isRunning: false, + }); setResponse.flush(); } catch (e) { // TODO Improve error handling @@ -223,7 +240,7 @@ export function useLatencyCorrelations() { const progress = useMemo( () => ({ error, - loaded, + loaded: Math.round(loaded * 100) / 100, isRunning, }), [error, loaded, isRunning] From cbeb5ef0a2bdb5b8ec7f2212f1cf8d741a43e96a Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Fri, 29 Oct 2021 14:58:11 +0200 Subject: [PATCH 25/39] [ML] Adds comments to explain the purpose of responseUpdate. --- .../app/correlations/use_failed_transactions_correlations.ts | 4 ++++ .../components/app/correlations/use_latency_correlations.ts | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/x-pack/plugins/apm/public/components/app/correlations/use_failed_transactions_correlations.ts b/x-pack/plugins/apm/public/components/app/correlations/use_failed_transactions_correlations.ts index 789702ac121f8..50983ae0d1bfb 100644 --- a/x-pack/plugins/apm/public/components/app/correlations/use_failed_transactions_correlations.ts +++ b/x-pack/plugins/apm/public/components/app/correlations/use_failed_transactions_correlations.ts @@ -72,6 +72,10 @@ export function useFailedTransactionsCorrelations() { setResponse.flush(); try { + // Initial call to fetch the overall distribution for the log-log plot. + // `responseUpdate` will be enriched with additional data with subsequent + // calls to fetch error histograms, field candidates, field value pairs, correlation results + // and histogram data for statistically significant results. const responseUpdate = (await callApmApi({ endpoint: 'POST /internal/apm/latency/overall_distribution', signal: null, diff --git a/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts b/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts index cde1526cde9e7..7a43c61aef4eb 100644 --- a/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts +++ b/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts @@ -73,6 +73,10 @@ export function useLatencyCorrelations() { setResponse.flush(); try { + // Initial call to fetch the overall distribution for the log-log plot. + // `responseUpdate` will be enriched with additional data with subsequent + // calls to fetch field candidates, field value pairs, correlation results + // and histogram data for statistically significant results. const responseUpdate = (await callApmApi({ endpoint: 'POST /internal/apm/latency/overall_distribution', signal: null, From a0992efcae2b4d4f11ae320ff0fe68e9f5e4611e Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Fri, 29 Oct 2021 15:00:26 +0200 Subject: [PATCH 26/39] [ML] Remove unnecessary cast to string. --- .../components/app/correlations/use_latency_correlations.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts b/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts index 7a43c61aef4eb..dc37021b9b949 100644 --- a/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts +++ b/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts @@ -83,7 +83,7 @@ export function useLatencyCorrelations() { params: { body: { ...fetchParams, - percentileThreshold: DEFAULT_PERCENTILE_THRESHOLD + '', + percentileThreshold: DEFAULT_PERCENTILE_THRESHOLD, }, }, })) as Response; From a6e75d7c8970450c5085def50d215b0a9a231fba Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Fri, 29 Oct 2021 15:11:41 +0200 Subject: [PATCH 27/39] [ML] Remove usage of deprecated useUrlParams. --- .../app/correlations/use_fetch_params.ts | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/apm/public/components/app/correlations/use_fetch_params.ts b/x-pack/plugins/apm/public/components/app/correlations/use_fetch_params.ts index 8c7dea16af456..827604f776c5a 100644 --- a/x-pack/plugins/apm/public/components/app/correlations/use_fetch_params.ts +++ b/x-pack/plugins/apm/public/components/app/correlations/use_fetch_params.ts @@ -8,19 +8,22 @@ import { useMemo } from 'react'; import { useApmServiceContext } from '../../../context/apm_service/use_apm_service_context'; -import { useUrlParams } from '../../../context/url_params_context/use_url_params'; import { useApmParams } from '../../../hooks/use_apm_params'; import { useTimeRange } from '../../../hooks/use_time_range'; export const useFetchParams = () => { - const { serviceName, transactionType } = useApmServiceContext(); - - const { urlParams } = useUrlParams(); - const { transactionName } = urlParams; + const { serviceName } = useApmServiceContext(); const { - query: { kuery, environment, rangeFrom, rangeTo }, + query: { + kuery, + environment, + rangeFrom, + rangeTo, + transactionName, + transactionType, + }, } = useApmParams('/services/{serviceName}/transactions/view'); const { start, end } = useTimeRange({ rangeFrom, rangeTo }); From deddd7931f08b3dbf51af40ecf5356006ca4504f Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Fri, 29 Oct 2021 15:24:07 +0200 Subject: [PATCH 28/39] [ML] Remove casting as Response. --- .../use_failed_transactions_correlations.ts | 23 +++++++++++-------- .../correlations/use_latency_correlations.ts | 17 ++++++++------ 2 files changed, 23 insertions(+), 17 deletions(-) diff --git a/x-pack/plugins/apm/public/components/app/correlations/use_failed_transactions_correlations.ts b/x-pack/plugins/apm/public/components/app/correlations/use_failed_transactions_correlations.ts index 50983ae0d1bfb..989df8c597366 100644 --- a/x-pack/plugins/apm/public/components/app/correlations/use_failed_transactions_correlations.ts +++ b/x-pack/plugins/apm/public/components/app/correlations/use_failed_transactions_correlations.ts @@ -29,8 +29,6 @@ import { } from './utils/analysis_hook_utils'; import { useFetchParams } from './use_fetch_params'; -type Response = FailedTransactionsCorrelationsResponse; - // Overall progress is a float from 0 to 1. const LOADED_OVERALL_HISTOGRAM = 0.05; const LOADED_FIELD_CANDIDATES = LOADED_OVERALL_HISTOGRAM + 0.05; @@ -43,7 +41,7 @@ export function useFailedTransactionsCorrelations() { // This use of useReducer (the dispatch function won't get reinstantiated // on every update) and debounce avoids flooding consuming components with updates. const [response, setResponseUnDebounced] = useReducer( - getReducer(), + getReducer(), getInitialResponse() ); const setResponse = useMemo( @@ -72,11 +70,15 @@ export function useFailedTransactionsCorrelations() { setResponse.flush(); try { - // Initial call to fetch the overall distribution for the log-log plot. // `responseUpdate` will be enriched with additional data with subsequent - // calls to fetch error histograms, field candidates, field value pairs, correlation results + // calls to the overall histogram, field candidates, field value pairs, correlation results // and histogram data for statistically significant results. - const responseUpdate = (await callApmApi({ + const responseUpdate: FailedTransactionsCorrelationsResponse = { + ccsWarning: false, + }; + + // Initial call to fetch the overall distribution for the log-log plot. + const { overallHistogram } = await callApmApi({ endpoint: 'POST /internal/apm/latency/overall_distribution', signal: null, params: { @@ -85,9 +87,10 @@ export function useFailedTransactionsCorrelations() { percentileThreshold: DEFAULT_PERCENTILE_THRESHOLD, }, }, - })) as Response; + }); + responseUpdate.overallHistogram = overallHistogram; - const { overallHistogram: errorHistogram } = (await callApmApi({ + const { overallHistogram: errorHistogram } = await callApmApi({ endpoint: 'POST /internal/apm/latency/overall_distribution', signal: null, params: { @@ -99,7 +102,8 @@ export function useFailedTransactionsCorrelations() { ], }, }, - })) as Response; + }); + responseUpdate.errorHistogram = errorHistogram; if (isCancelledRef.current) { return; @@ -107,7 +111,6 @@ export function useFailedTransactionsCorrelations() { setResponse({ ...responseUpdate, - errorHistogram, loaded: LOADED_OVERALL_HISTOGRAM, }); setResponse.flush(); diff --git a/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts b/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts index dc37021b9b949..2d63a057c656f 100644 --- a/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts +++ b/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts @@ -29,8 +29,6 @@ import { } from './utils/analysis_hook_utils'; import { useFetchParams } from './use_fetch_params'; -type Response = LatencyCorrelationsResponse; - // Overall progress is a float from 0 to 1. const LOADED_OVERALL_HISTOGRAM = 0.05; const LOADED_FIELD_CANDIDATES = LOADED_OVERALL_HISTOGRAM + 0.05; @@ -45,7 +43,7 @@ export function useLatencyCorrelations() { // This use of useReducer (the dispatch function won't get reinstantiated // on every update) and debounce avoids flooding consuming components with updates. const [response, setResponseUnDebounced] = useReducer( - getReducer(), + getReducer(), getInitialResponse() ); const setResponse = useMemo( @@ -73,11 +71,15 @@ export function useLatencyCorrelations() { setResponse.flush(); try { - // Initial call to fetch the overall distribution for the log-log plot. // `responseUpdate` will be enriched with additional data with subsequent - // calls to fetch field candidates, field value pairs, correlation results + // calls to the overall histogram, field candidates, field value pairs, correlation results // and histogram data for statistically significant results. - const responseUpdate = (await callApmApi({ + const responseUpdate: LatencyCorrelationsResponse = { + ccsWarning: false, + }; + + // Initial call to fetch the overall distribution for the log-log plot. + const { overallHistogram } = await callApmApi({ endpoint: 'POST /internal/apm/latency/overall_distribution', signal: null, params: { @@ -86,7 +88,8 @@ export function useLatencyCorrelations() { percentileThreshold: DEFAULT_PERCENTILE_THRESHOLD, }, }, - })) as Response; + }); + responseUpdate.overallHistogram = overallHistogram; if (isCancelledRef.current) { return; From 45da6c542a223b4a76ec38dd39a1f552f923e0c0 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Fri, 29 Oct 2021 16:11:12 +0200 Subject: [PATCH 29/39] [ML] Add abort signal. --- .../use_failed_transactions_correlations.ts | 15 ++++++++++----- .../correlations/use_latency_correlations.ts | 19 ++++++++++++------- 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/x-pack/plugins/apm/public/components/app/correlations/use_failed_transactions_correlations.ts b/x-pack/plugins/apm/public/components/app/correlations/use_failed_transactions_correlations.ts index 989df8c597366..4b8ce30308123 100644 --- a/x-pack/plugins/apm/public/components/app/correlations/use_failed_transactions_correlations.ts +++ b/x-pack/plugins/apm/public/components/app/correlations/use_failed_transactions_correlations.ts @@ -36,6 +36,8 @@ const LOADED_DONE = 1; const PROGRESS_STEP_P_VALUES = 0.8; export function useFailedTransactionsCorrelations() { + const abortCtrl = useRef(new AbortController()); + const fetchParams = useFetchParams(); // This use of useReducer (the dispatch function won't get reinstantiated @@ -54,6 +56,8 @@ export function useFailedTransactionsCorrelations() { const isCancelledRef = useRef(false); const startFetch = useCallback(async () => { + abortCtrl.current.abort(); + abortCtrl.current = new AbortController(); isCancelledRef.current = false; setResponse({ @@ -80,7 +84,7 @@ export function useFailedTransactionsCorrelations() { // Initial call to fetch the overall distribution for the log-log plot. const { overallHistogram } = await callApmApi({ endpoint: 'POST /internal/apm/latency/overall_distribution', - signal: null, + signal: abortCtrl.current.signal, params: { body: { ...fetchParams, @@ -92,7 +96,7 @@ export function useFailedTransactionsCorrelations() { const { overallHistogram: errorHistogram } = await callApmApi({ endpoint: 'POST /internal/apm/latency/overall_distribution', - signal: null, + signal: abortCtrl.current.signal, params: { body: { ...fetchParams, @@ -117,7 +121,7 @@ export function useFailedTransactionsCorrelations() { const { fieldCandidates: candidates } = await callApmApi({ endpoint: 'GET /internal/apm/correlations/field_candidates', - signal: null, + signal: abortCtrl.current.signal, params: { query: fetchParams, }, @@ -144,7 +148,7 @@ export function useFailedTransactionsCorrelations() { for (const fieldCandidatesChunk of fieldCandidatesChunks) { const pValues = await callApmApi({ endpoint: 'POST /internal/apm/correlations/p_values', - signal: null, + signal: abortCtrl.current.signal, params: { body: { ...fetchParams, fieldCandidates: fieldCandidatesChunk }, }, @@ -180,7 +184,7 @@ export function useFailedTransactionsCorrelations() { const fieldStats = await callApmApi({ endpoint: 'POST /internal/apm/correlations/field_stats', - signal: null, + signal: abortCtrl.current.signal, params: { body: { ...fetchParams, @@ -206,6 +210,7 @@ export function useFailedTransactionsCorrelations() { const cancelFetch = useCallback(() => { isCancelledRef.current = true; + abortCtrl.current.abort(); setResponse({ isRunning: false, }); diff --git a/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts b/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts index 2d63a057c656f..4e0e179734e7b 100644 --- a/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts +++ b/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts @@ -38,6 +38,8 @@ const PROGRESS_STEP_FIELD_VALUE_PAIRS = 0.3; const PROGRESS_STEP_CORRELATIONS = 0.6; export function useLatencyCorrelations() { + const abortCtrl = useRef(new AbortController()); + const fetchParams = useFetchParams(); // This use of useReducer (the dispatch function won't get reinstantiated @@ -56,6 +58,8 @@ export function useLatencyCorrelations() { const isCancelledRef = useRef(false); const startFetch = useCallback(async () => { + abortCtrl.current.abort(); + abortCtrl.current = new AbortController(); isCancelledRef.current = false; setResponse({ @@ -81,7 +85,7 @@ export function useLatencyCorrelations() { // Initial call to fetch the overall distribution for the log-log plot. const { overallHistogram } = await callApmApi({ endpoint: 'POST /internal/apm/latency/overall_distribution', - signal: null, + signal: abortCtrl.current.signal, params: { body: { ...fetchParams, @@ -103,7 +107,7 @@ export function useLatencyCorrelations() { const { fieldCandidates } = await callApmApi({ endpoint: 'GET /internal/apm/correlations/field_candidates', - signal: null, + signal: abortCtrl.current.signal, params: { query: fetchParams, }, @@ -125,10 +129,10 @@ export function useLatencyCorrelations() { for (const fieldCandidateChunk of fieldCandidateChunks) { const fieldValuePairChunkResponse = await callApmApi({ - endpoint: 'GET /internal/apm/correlations/field_value_pairs', - signal: null, + endpoint: 'POST /internal/apm/correlations/field_value_pairs', + signal: abortCtrl.current.signal, params: { - query: { + body: { ...fetchParams, fieldCandidates: fieldCandidateChunk, }, @@ -168,7 +172,7 @@ export function useLatencyCorrelations() { for (const fieldValuePairChunk of fieldValuePairChunks) { const significantCorrelations = await callApmApi({ endpoint: 'POST /internal/apm/correlations/significant_correlations', - signal: null, + signal: abortCtrl.current.signal, params: { body: { ...fetchParams, fieldValuePairs: fieldValuePairChunk }, }, @@ -202,7 +206,7 @@ export function useLatencyCorrelations() { const fieldStats = await callApmApi({ endpoint: 'POST /internal/apm/correlations/field_stats', - signal: null, + signal: abortCtrl.current.signal, params: { body: { ...fetchParams, @@ -232,6 +236,7 @@ export function useLatencyCorrelations() { const cancelFetch = useCallback(() => { isCancelledRef.current = true; + abortCtrl.current.abort(); setResponse({ isRunning: false, }); From 71270c161baf2f263d9be79d88e77c905115c85f Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Fri, 29 Oct 2021 16:12:16 +0200 Subject: [PATCH 30/39] [ML] Switch endpoint to POST. --- x-pack/plugins/apm/server/routes/correlations.ts | 6 +++--- .../tests/correlations/field_value_pairs.ts | 4 ++-- .../apm_api_integration/tests/correlations/latency.ts | 8 ++++---- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/x-pack/plugins/apm/server/routes/correlations.ts b/x-pack/plugins/apm/server/routes/correlations.ts index 5c0c285f34229..479c3d8ca73bf 100644 --- a/x-pack/plugins/apm/server/routes/correlations.ts +++ b/x-pack/plugins/apm/server/routes/correlations.ts @@ -113,9 +113,9 @@ const fieldStatsRoute = createApmServerRoute({ }); const fieldValuePairsRoute = createApmServerRoute({ - endpoint: 'GET /internal/apm/correlations/field_value_pairs', + endpoint: 'POST /internal/apm/correlations/field_value_pairs', params: t.type({ - query: t.intersection([ + body: t.intersection([ t.partial({ serviceName: t.string, transactionName: t.string, @@ -139,7 +139,7 @@ const fieldValuePairsRoute = createApmServerRoute({ const { indices } = await setupRequest(resources); const esClient = resources.context.core.elasticsearch.client.asCurrentUser; - const { fieldCandidates, ...params } = resources.params.query; + const { fieldCandidates, ...params } = resources.params.body; return withApmSpan('get_correlations_field_value_pairs', async () => ({ fieldValuePairs: await fetchTransactionDurationFieldValuePairs( diff --git a/x-pack/test/apm_api_integration/tests/correlations/field_value_pairs.ts b/x-pack/test/apm_api_integration/tests/correlations/field_value_pairs.ts index 800938a36d127..f0a0e17561ac7 100644 --- a/x-pack/test/apm_api_integration/tests/correlations/field_value_pairs.ts +++ b/x-pack/test/apm_api_integration/tests/correlations/field_value_pairs.ts @@ -12,11 +12,11 @@ import { registry } from '../../common/registry'; export default function ApiTest({ getService }: FtrProviderContext) { const apmApiClient = getService('apmApiClient'); - const endpoint = 'GET /internal/apm/correlations/field_value_pairs'; + const endpoint = 'POST /internal/apm/correlations/field_value_pairs'; const getOptions = () => ({ params: { - query: { + body: { environment: 'ENVIRONMENT_ALL', start: '2020', end: '2021', diff --git a/x-pack/test/apm_api_integration/tests/correlations/latency.ts b/x-pack/test/apm_api_integration/tests/correlations/latency.ts index bb5cfa0fb7cee..b9824b15c1908 100644 --- a/x-pack/test/apm_api_integration/tests/correlations/latency.ts +++ b/x-pack/test/apm_api_integration/tests/correlations/latency.ts @@ -57,9 +57,9 @@ export default function ApiTest({ getService }: FtrProviderContext) { ); const fieldValuePairsResponse = await apmApiClient.readUser({ - endpoint: 'GET /internal/apm/correlations/field_value_pairs', + endpoint: 'POST /internal/apm/correlations/field_value_pairs', params: { - query: { + body: { ...getOptions(), fieldCandidates: fieldCandidatesResponse.body?.fieldCandidates, }, @@ -140,9 +140,9 @@ export default function ApiTest({ getService }: FtrProviderContext) { ); const fieldValuePairsResponse = await apmApiClient.readUser({ - endpoint: 'GET /internal/apm/correlations/field_value_pairs', + endpoint: 'POST /internal/apm/correlations/field_value_pairs', params: { - query: { + body: { ...getOptions(), fieldCandidates: fieldCandidatesResponse.body?.fieldCandidates, }, From 312e9b2686da17f930968128b1feaad03a70ac9d Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Fri, 29 Oct 2021 18:41:20 +0200 Subject: [PATCH 31/39] [ML] Improved error handling. --- .../correlations/failed_transactions_correlations.tsx | 5 ++--- .../app/correlations/latency_correlations.tsx | 5 ++--- .../use_failed_transactions_correlations.ts | 11 +++++++---- .../app/correlations/use_latency_correlations.ts | 11 +++++++---- .../app/correlations/utils/analysis_hook_utils.ts | 4 +--- 5 files changed, 19 insertions(+), 17 deletions(-) diff --git a/x-pack/plugins/apm/public/components/app/correlations/failed_transactions_correlations.tsx b/x-pack/plugins/apm/public/components/app/correlations/failed_transactions_correlations.tsx index 56ca110b5e7b8..f13d360444923 100644 --- a/x-pack/plugins/apm/public/components/app/correlations/failed_transactions_correlations.tsx +++ b/x-pack/plugins/apm/public/components/app/correlations/failed_transactions_correlations.tsx @@ -46,7 +46,6 @@ import { push } from '../../shared/Links/url_helpers'; import { CorrelationsTable } from './correlations_table'; import { FailedTransactionsCorrelationsHelpPopover } from './failed_transactions_correlations_help_popover'; -import { isErrorMessage } from './utils/is_error_message'; import { getFailedTransactionsCorrelationImpactLabel } from './utils/get_failed_transactions_correlation_impact_label'; import { getOverallHistogram } from './utils/get_overall_histogram'; import { @@ -355,7 +354,7 @@ export function FailedTransactionsCorrelations({ }, [fieldStats, onAddFilter, showStats]); useEffect(() => { - if (isErrorMessage(progress.error)) { + if (progress.error) { notifications.toasts.addDanger({ title: i18n.translate( 'xpack.apm.correlations.failedTransactions.errorTitle', @@ -364,7 +363,7 @@ export function FailedTransactionsCorrelations({ 'An error occurred performing correlations on failed transactions', } ), - text: progress.error.toString(), + text: progress.error, }); } }, [progress.error, notifications.toasts]); diff --git a/x-pack/plugins/apm/public/components/app/correlations/latency_correlations.tsx b/x-pack/plugins/apm/public/components/app/correlations/latency_correlations.tsx index 964896bd04729..b67adc03d40e9 100644 --- a/x-pack/plugins/apm/public/components/app/correlations/latency_correlations.tsx +++ b/x-pack/plugins/apm/public/components/app/correlations/latency_correlations.tsx @@ -43,7 +43,6 @@ import { push } from '../../shared/Links/url_helpers'; import { CorrelationsTable } from './correlations_table'; import { LatencyCorrelationsHelpPopover } from './latency_correlations_help_popover'; -import { isErrorMessage } from './utils/is_error_message'; import { getOverallHistogram } from './utils/get_overall_histogram'; import { CorrelationsEmptyStatePrompt } from './empty_state_prompt'; import { CrossClusterSearchCompatibilityWarning } from './cross_cluster_search_warning'; @@ -75,7 +74,7 @@ export function LatencyCorrelations({ onFilter }: { onFilter: () => void }) { }, [response?.fieldStats]); useEffect(() => { - if (isErrorMessage(progress.error)) { + if (progress.error) { notifications.toasts.addDanger({ title: i18n.translate( 'xpack.apm.correlations.latencyCorrelations.errorTitle', @@ -83,7 +82,7 @@ export function LatencyCorrelations({ onFilter }: { onFilter: () => void }) { defaultMessage: 'An error occurred fetching correlations', } ), - text: progress.error.toString(), + text: progress.error, }); } }, [progress.error, notifications.toasts]); diff --git a/x-pack/plugins/apm/public/components/app/correlations/use_failed_transactions_correlations.ts b/x-pack/plugins/apm/public/components/app/correlations/use_failed_transactions_correlations.ts index 4b8ce30308123..d110fa6924d19 100644 --- a/x-pack/plugins/apm/public/components/app/correlations/use_failed_transactions_correlations.ts +++ b/x-pack/plugins/apm/public/components/app/correlations/use_failed_transactions_correlations.ts @@ -8,6 +8,8 @@ import { useCallback, useEffect, useMemo, useReducer, useRef } from 'react'; import { chunk, debounce } from 'lodash'; +import { IHttpFetchError } from 'src/core/public'; + import { EVENT_OUTCOME } from '../../../../common/elasticsearch_fieldnames'; import { EventOutcome } from '../../../../common/event_outcome'; import { @@ -197,11 +199,12 @@ export function useFailedTransactionsCorrelations() { setResponse({ ...responseUpdate, loaded: LOADED_DONE, isRunning: false }); setResponse.flush(); } catch (e) { - // TODO Improve error handling - // const err = e as Error | IHttpFetchError; - // const message = error.body?.message ?? error.response?.statusText; + const err = e as Error | IHttpFetchError; setResponse({ - error: e as Error, + error: + 'response' in err + ? err.body?.message ?? err.response?.statusText + : err.message, isRunning: false, }); setResponse.flush(); diff --git a/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts b/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts index 4e0e179734e7b..d661e09a04d11 100644 --- a/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts +++ b/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts @@ -8,6 +8,8 @@ import { useCallback, useEffect, useMemo, useReducer, useRef } from 'react'; import { chunk, debounce } from 'lodash'; +import { IHttpFetchError } from 'src/core/public'; + import { DEBOUNCE_INTERVAL, DEFAULT_PERCENTILE_THRESHOLD, @@ -223,11 +225,12 @@ export function useLatencyCorrelations() { }); setResponse.flush(); } catch (e) { - // TODO Improve error handling - // const err = e as Error | IHttpFetchError; - // const message = error.body?.message ?? error.response?.statusText; + const err = e as Error | IHttpFetchError; setResponse({ - error: e as Error, + error: + 'response' in err + ? err.body?.message ?? err.response?.statusText + : err.message, isRunning: false, }); setResponse.flush(); diff --git a/x-pack/plugins/apm/public/components/app/correlations/utils/analysis_hook_utils.ts b/x-pack/plugins/apm/public/components/app/correlations/utils/analysis_hook_utils.ts index 51f4a0a48cbe8..24cd76846fa9f 100644 --- a/x-pack/plugins/apm/public/components/app/correlations/utils/analysis_hook_utils.ts +++ b/x-pack/plugins/apm/public/components/app/correlations/utils/analysis_hook_utils.ts @@ -5,13 +5,11 @@ * 2.0. */ -import { IHttpFetchError } from 'src/core/public'; - import type { FailedTransactionsCorrelation } from '../../../../../common/correlations/failed_transactions_correlations/types'; import type { LatencyCorrelation } from '../../../../../common/correlations/latency_correlations/types'; export interface CorrelationsProgress { - error?: Error | IHttpFetchError; + error?: string; isRunning: boolean; loaded: number; } From f47e65426132f399c1b347fc26063309b5186d9d Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Wed, 3 Nov 2021 10:31:57 +0100 Subject: [PATCH 32/39] [ML] Fix unmounting. Fix percentileThresholdValue. Add unit tests to hooks. --- ...e_failed_transaction_correlations.test.tsx | 376 ++++++++++++++++++ .../use_failed_transactions_correlations.ts | 42 +- .../use_latency_correlations.test.tsx | 334 ++++++++++++++++ .../correlations/use_latency_correlations.ts | 44 +- 4 files changed, 762 insertions(+), 34 deletions(-) create mode 100644 x-pack/plugins/apm/public/components/app/correlations/use_failed_transaction_correlations.test.tsx create mode 100644 x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.test.tsx diff --git a/x-pack/plugins/apm/public/components/app/correlations/use_failed_transaction_correlations.test.tsx b/x-pack/plugins/apm/public/components/app/correlations/use_failed_transaction_correlations.test.tsx new file mode 100644 index 0000000000000..81e3428648fd3 --- /dev/null +++ b/x-pack/plugins/apm/public/components/app/correlations/use_failed_transaction_correlations.test.tsx @@ -0,0 +1,376 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { ReactNode } from 'react'; +import { merge } from 'lodash'; +import { createMemoryHistory } from 'history'; +import { renderHook } from '@testing-library/react-hooks'; + +import { ApmPluginContextValue } from '../../../context/apm_plugin/apm_plugin_context'; +import { + mockApmPluginContextValue, + MockApmPluginContextWrapper, +} from '../../../context/apm_plugin/mock_apm_plugin_context'; +import { delay } from '../../../utils/testHelpers'; + +import { fromQuery } from '../../shared/Links/url_helpers'; + +import { useFailedTransactionsCorrelations } from './use_failed_transactions_correlations'; + +function wrapper({ + children, + error = false, +}: { + children?: ReactNode; + error: boolean; +}) { + const httpMethodMock = jest.fn().mockImplementation(async (endpoint) => { + await delay(100); + if (error) { + throw new Error('Something went wrong'); + } + switch (endpoint) { + case '/internal/apm/latency/overall_distribution': + return { + overallHistogram: [{ key: 'the-key', doc_count: 1234 }], + percentileThresholdValue: 1.234, + }; + case '/internal/apm/correlations/field_candidates': + return { fieldCandidates: ['field-1', 'field2'] }; + case '/internal/apm/correlations/field_value_pairs': + return { + fieldValuePairs: [ + { fieldName: 'field-name-1', fieldValue: 'field-value-1' }, + ], + }; + case '/internal/apm/correlations/p_values': + return { + failedTransactionsCorrelations: [ + { + fieldName: 'field-name-1', + fieldValue: 'field-value-1', + doc_count: 123, + bg_count: 1234, + score: 0.66, + pValue: 0.01, + normalizedScore: 0.85, + failurePercentage: 30, + successPercentage: 70, + histogram: [{ key: 'the-key', doc_count: 123 }], + }, + ], + }; + case '/internal/apm/correlations/field_stats': + return { + stats: [ + { fieldName: 'field-name-1', count: 123 }, + { fieldName: 'field-name-2', count: 1111 }, + ], + }; + default: + return {}; + } + }); + + const history = createMemoryHistory(); + jest.spyOn(history, 'push'); + jest.spyOn(history, 'replace'); + + history.replace({ + pathname: '/services/the-service-name/transactions/view', + search: fromQuery({ + transactionName: 'the-transaction-name', + rangeFrom: 'now-15m', + rangeTo: 'now', + }), + }); + + const mockPluginContext = merge({}, mockApmPluginContextValue, { + core: { http: { get: httpMethodMock, post: httpMethodMock } }, + }) as unknown as ApmPluginContextValue; + + return ( + + {children} + + ); +} + +describe('useFailedTransactionsCorrelations', () => { + beforeEach(async () => { + jest.useFakeTimers(); + }); + // Running all pending timers and switching to real timers using Jest + afterEach(() => { + jest.runOnlyPendingTimers(); + jest.useRealTimers(); + }); + + describe('when successfully loading results', () => { + it('should automatically start fetching results', async () => { + const { result, unmount } = renderHook( + () => useFailedTransactionsCorrelations(), + { + wrapper, + } + ); + + try { + expect(result.current.progress).toEqual({ + isRunning: true, + loaded: 0, + }); + expect(result.current.response).toEqual({ ccsWarning: false }); + expect(typeof result.current.startFetch).toEqual('function'); + expect(typeof result.current.cancelFetch).toEqual('function'); + } finally { + unmount(); + } + }); + + it('should not have received any results after 50ms', async () => { + const { result, unmount } = renderHook( + () => useFailedTransactionsCorrelations(), + { + wrapper, + } + ); + + try { + jest.advanceTimersByTime(50); + + expect(result.current.progress).toEqual({ + isRunning: true, + loaded: 0, + }); + expect(result.current.response).toEqual({ ccsWarning: false }); + } finally { + unmount(); + } + }); + + it('should receive partial updates and finish running', async () => { + const { result, unmount, waitFor } = renderHook( + () => useFailedTransactionsCorrelations(), + { + wrapper, + } + ); + + try { + jest.advanceTimersByTime(50); + await waitFor(() => expect(result.current.progress.loaded).toBe(0)); + jest.advanceTimersByTime(100); + await waitFor(() => expect(result.current.progress.loaded).toBe(0)); + jest.advanceTimersByTime(100); + await waitFor(() => expect(result.current.progress.loaded).toBe(0)); + jest.advanceTimersByTime(100); + await waitFor(() => expect(result.current.progress.loaded).toBe(0.05)); + + expect(result.current.progress).toEqual({ + error: undefined, + isRunning: true, + loaded: 0.05, + }); + expect(result.current.response).toEqual({ + ccsWarning: false, + fieldStats: undefined, + errorHistogram: [ + { + doc_count: 1234, + key: 'the-key', + }, + ], + failedTransactionsCorrelations: undefined, + overallHistogram: [ + { + doc_count: 1234, + key: 'the-key', + }, + ], + percentileThresholdValue: 1.234, + }); + + jest.advanceTimersByTime(100); + await waitFor(() => result.current.progress.loaded === 0.1); + + // field candidates are an implementation detail and + // will not be exposed, it will just set loaded to 0.1. + expect(result.current.progress).toEqual({ + error: undefined, + isRunning: true, + loaded: 0.1, + }); + + jest.advanceTimersByTime(100); + await waitFor(() => result.current.progress.loaded === 1); + + expect(result.current.progress).toEqual({ + error: undefined, + isRunning: true, + loaded: 1, + }); + + expect(result.current.response).toEqual({ + ccsWarning: false, + fieldStats: undefined, + errorHistogram: [ + { + doc_count: 1234, + key: 'the-key', + }, + ], + failedTransactionsCorrelations: [ + { + fieldName: 'field-name-1', + fieldValue: 'field-value-1', + doc_count: 123, + bg_count: 1234, + score: 0.66, + pValue: 0.01, + normalizedScore: 0.85, + failurePercentage: 30, + successPercentage: 70, + histogram: [{ key: 'the-key', doc_count: 123 }], + }, + ], + overallHistogram: [ + { + doc_count: 1234, + key: 'the-key', + }, + ], + percentileThresholdValue: 1.234, + }); + + jest.advanceTimersByTime(100); + await waitFor(() => result.current.response.fieldStats !== undefined); + + expect(result.current.progress).toEqual({ + error: undefined, + isRunning: false, + loaded: 1, + }); + + expect(result.current.response).toEqual({ + ccsWarning: false, + fieldStats: [ + { fieldName: 'field-name-1', count: 123 }, + { fieldName: 'field-name-2', count: 1111 }, + ], + errorHistogram: [ + { + doc_count: 1234, + key: 'the-key', + }, + ], + failedTransactionsCorrelations: [ + { + fieldName: 'field-name-1', + fieldValue: 'field-value-1', + doc_count: 123, + bg_count: 1234, + score: 0.66, + pValue: 0.01, + normalizedScore: 0.85, + failurePercentage: 30, + successPercentage: 70, + histogram: [{ key: 'the-key', doc_count: 123 }], + }, + ], + overallHistogram: [ + { + doc_count: 1234, + key: 'the-key', + }, + ], + percentileThresholdValue: 1.234, + }); + } finally { + unmount(); + } + }); + }); + describe('when throwing an error', () => { + it('should automatically start fetching results', async () => { + const { result, unmount } = renderHook( + () => useFailedTransactionsCorrelations(), + { + wrapper, + initialProps: { + error: true, + }, + } + ); + + try { + expect(result.current.progress).toEqual({ + isRunning: true, + loaded: 0, + }); + } finally { + unmount(); + } + }); + + it('should still be running after 50ms', async () => { + const { result, unmount } = renderHook( + () => useFailedTransactionsCorrelations(), + { + wrapper, + initialProps: { + error: true, + }, + } + ); + + try { + jest.advanceTimersByTime(50); + + expect(result.current.progress).toEqual({ + isRunning: true, + loaded: 0, + }); + expect(result.current.response).toEqual({ ccsWarning: false }); + } finally { + unmount(); + } + }); + + it('should stop and return an error after more than 100ms', async () => { + const { result, unmount, waitFor } = renderHook( + () => useFailedTransactionsCorrelations(), + { + wrapper, + initialProps: { + error: true, + }, + } + ); + + try { + jest.advanceTimersByTime(150); + await waitFor(() => result.current.progress.error !== undefined); + + expect(result.current.progress).toEqual({ + error: 'Something went wrong', + isRunning: false, + loaded: 0, + }); + expect(result.current.response).toEqual({ + ccsWarning: false, + fieldStats: undefined, + latencyCorrelations: undefined, + overallHistogram: undefined, + percentileThresholdValue: undefined, + }); + } finally { + unmount(); + } + }); + }); +}); diff --git a/x-pack/plugins/apm/public/components/app/correlations/use_failed_transactions_correlations.ts b/x-pack/plugins/apm/public/components/app/correlations/use_failed_transactions_correlations.ts index d110fa6924d19..f03fa4998e4be 100644 --- a/x-pack/plugins/apm/public/components/app/correlations/use_failed_transactions_correlations.ts +++ b/x-pack/plugins/apm/public/components/app/correlations/use_failed_transactions_correlations.ts @@ -8,7 +8,7 @@ import { useCallback, useEffect, useMemo, useReducer, useRef } from 'react'; import { chunk, debounce } from 'lodash'; -import { IHttpFetchError } from 'src/core/public'; +import { IHttpFetchError, ResponseErrorBody } from 'src/core/public'; import { EVENT_OUTCOME } from '../../../../common/elasticsearch_fieldnames'; import { EventOutcome } from '../../../../common/event_outcome'; @@ -35,7 +35,7 @@ import { useFetchParams } from './use_fetch_params'; const LOADED_OVERALL_HISTOGRAM = 0.05; const LOADED_FIELD_CANDIDATES = LOADED_OVERALL_HISTOGRAM + 0.05; const LOADED_DONE = 1; -const PROGRESS_STEP_P_VALUES = 0.8; +const PROGRESS_STEP_P_VALUES = 0.9; export function useFailedTransactionsCorrelations() { const abortCtrl = useRef(new AbortController()); @@ -84,7 +84,7 @@ export function useFailedTransactionsCorrelations() { }; // Initial call to fetch the overall distribution for the log-log plot. - const { overallHistogram } = await callApmApi({ + const { overallHistogram, percentileThresholdValue } = await callApmApi({ endpoint: 'POST /internal/apm/latency/overall_distribution', signal: abortCtrl.current.signal, params: { @@ -95,6 +95,7 @@ export function useFailedTransactionsCorrelations() { }, }); responseUpdate.overallHistogram = overallHistogram; + responseUpdate.percentileThresholdValue = percentileThresholdValue; const { overallHistogram: errorHistogram } = await callApmApi({ endpoint: 'POST /internal/apm/latency/overall_distribution', @@ -138,6 +139,7 @@ export function useFailedTransactionsCorrelations() { setResponse({ loaded: LOADED_FIELD_CANDIDATES, }); + setResponse.flush(); const failedTransactionsCorrelations: FailedTransactionsCorrelation[] = []; @@ -169,6 +171,7 @@ export function useFailedTransactionsCorrelations() { ]); } + chunkLoadCounter++; setResponse({ ...responseUpdate, loaded: @@ -180,11 +183,11 @@ export function useFailedTransactionsCorrelations() { if (isCancelledRef.current) { return; } - - chunkLoadCounter++; } - const fieldStats = await callApmApi({ + setResponse.flush(); + + const { stats } = await callApmApi({ endpoint: 'POST /internal/apm/correlations/field_stats', signal: abortCtrl.current.signal, params: { @@ -195,19 +198,21 @@ export function useFailedTransactionsCorrelations() { }, }); - responseUpdate.fieldStats = fieldStats.stats; + responseUpdate.fieldStats = stats; setResponse({ ...responseUpdate, loaded: LOADED_DONE, isRunning: false }); setResponse.flush(); } catch (e) { - const err = e as Error | IHttpFetchError; - setResponse({ - error: - 'response' in err - ? err.body?.message ?? err.response?.statusText - : err.message, - isRunning: false, - }); - setResponse.flush(); + if (!isCancelledRef.current) { + const err = e as Error | IHttpFetchError; + setResponse({ + error: + 'response' in err + ? err.body?.message ?? err.response?.statusText + : err.message, + isRunning: false, + }); + setResponse.flush(); + } } }, [fetchParams, setResponse]); @@ -222,7 +227,10 @@ export function useFailedTransactionsCorrelations() { // auto-update useEffect(() => { startFetch(); - return cancelFetch; + return () => { + isCancelledRef.current = true; + abortCtrl.current.abort(); + }; }, [startFetch, cancelFetch]); const { error, loaded, isRunning, ...returnedResponse } = response; diff --git a/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.test.tsx b/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.test.tsx new file mode 100644 index 0000000000000..0f18284cb7351 --- /dev/null +++ b/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.test.tsx @@ -0,0 +1,334 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { ReactNode } from 'react'; +import { merge } from 'lodash'; +import { createMemoryHistory } from 'history'; +import { renderHook } from '@testing-library/react-hooks'; + +import { ApmPluginContextValue } from '../../../context/apm_plugin/apm_plugin_context'; +import { + mockApmPluginContextValue, + MockApmPluginContextWrapper, +} from '../../../context/apm_plugin/mock_apm_plugin_context'; +import { delay } from '../../../utils/testHelpers'; + +import { fromQuery } from '../../shared/Links/url_helpers'; + +import { useLatencyCorrelations } from './use_latency_correlations'; + +function wrapper({ + children, + error = false, +}: { + children?: ReactNode; + error: boolean; +}) { + const httpMethodMock = jest.fn().mockImplementation(async (endpoint) => { + await delay(100); + if (error) { + throw new Error('Something went wrong'); + } + switch (endpoint) { + case '/internal/apm/latency/overall_distribution': + return { + overallHistogram: [{ key: 'the-key', doc_count: 1234 }], + percentileThresholdValue: 1.234, + }; + case '/internal/apm/correlations/field_candidates': + return { fieldCandidates: ['field-1', 'field2'] }; + case '/internal/apm/correlations/field_value_pairs': + return { + fieldValuePairs: [ + { fieldName: 'field-name-1', fieldValue: 'field-value-1' }, + ], + }; + case '/internal/apm/correlations/significant_correlations': + return { + latencyCorrelations: [ + { + fieldName: 'field-name-1', + fieldValue: 'field-value-1', + correlation: 0.5, + histogram: [{ key: 'the-key', doc_count: 123 }], + ksTest: 0.001, + }, + ], + }; + case '/internal/apm/correlations/field_stats': + return { + stats: [ + { fieldName: 'field-name-1', count: 123 }, + { fieldName: 'field-name-2', count: 1111 }, + ], + }; + default: + return {}; + } + }); + + const history = createMemoryHistory(); + jest.spyOn(history, 'push'); + jest.spyOn(history, 'replace'); + + history.replace({ + pathname: '/services/the-service-name/transactions/view', + search: fromQuery({ + transactionName: 'the-transaction-name', + rangeFrom: 'now-15m', + rangeTo: 'now', + }), + }); + + const mockPluginContext = merge({}, mockApmPluginContextValue, { + core: { http: { get: httpMethodMock, post: httpMethodMock } }, + }) as unknown as ApmPluginContextValue; + + return ( + + {children} + + ); +} + +describe('useLatencyCorrelations', () => { + beforeEach(async () => { + jest.useFakeTimers(); + }); + afterEach(() => { + jest.useRealTimers(); + }); + + describe('when successfully loading results', () => { + it('should automatically start fetching results', async () => { + const { result, unmount } = renderHook(() => useLatencyCorrelations(), { + wrapper, + }); + + try { + expect(result.current.progress).toEqual({ + isRunning: true, + loaded: 0, + }); + expect(result.current.response).toEqual({ ccsWarning: false }); + expect(typeof result.current.startFetch).toEqual('function'); + expect(typeof result.current.cancelFetch).toEqual('function'); + } finally { + unmount(); + } + }); + + it('should not have received any results after 50ms', async () => { + const { result, unmount } = renderHook(() => useLatencyCorrelations(), { + wrapper, + }); + + try { + jest.advanceTimersByTime(50); + + expect(result.current.progress).toEqual({ + isRunning: true, + loaded: 0, + }); + expect(result.current.response).toEqual({ ccsWarning: false }); + } finally { + unmount(); + } + }); + + it('should receive partial updates and finish running', async () => { + const { result, unmount, waitFor } = renderHook( + () => useLatencyCorrelations(), + { + wrapper, + } + ); + + try { + jest.advanceTimersByTime(150); + await waitFor(() => result.current.progress.loaded === 0.05); + + expect(result.current.progress).toEqual({ + error: undefined, + isRunning: true, + loaded: 0.05, + }); + expect(result.current.response).toEqual({ + ccsWarning: false, + fieldStats: undefined, + latencyCorrelations: undefined, + overallHistogram: [ + { + doc_count: 1234, + key: 'the-key', + }, + ], + percentileThresholdValue: 1.234, + }); + + jest.advanceTimersByTime(100); + await waitFor(() => result.current.progress.loaded === 0.1); + + // field candidates are an implementation detail and + // will not be exposed, it will just set loaded to 0.1. + expect(result.current.progress).toEqual({ + error: undefined, + isRunning: true, + loaded: 0.1, + }); + + jest.advanceTimersByTime(100); + await waitFor(() => result.current.progress.loaded === 0.4); + + // field value pairs are an implementation detail and + // will not be exposed, it will just set loaded to 0.4. + expect(result.current.progress).toEqual({ + error: undefined, + isRunning: true, + loaded: 0.4, + }); + + jest.advanceTimersByTime(100); + await waitFor(() => result.current.progress.loaded === 1); + + expect(result.current.progress).toEqual({ + error: undefined, + isRunning: true, + loaded: 1, + }); + + expect(result.current.response).toEqual({ + ccsWarning: false, + fieldStats: undefined, + latencyCorrelations: [ + { + fieldName: 'field-name-1', + fieldValue: 'field-value-1', + correlation: 0.5, + histogram: [{ key: 'the-key', doc_count: 123 }], + ksTest: 0.001, + }, + ], + overallHistogram: [ + { + doc_count: 1234, + key: 'the-key', + }, + ], + percentileThresholdValue: 1.234, + }); + + jest.advanceTimersByTime(100); + await waitFor(() => result.current.response.fieldStats !== undefined); + + expect(result.current.progress).toEqual({ + error: undefined, + isRunning: false, + loaded: 1, + }); + + expect(result.current.response).toEqual({ + ccsWarning: false, + fieldStats: [ + { fieldName: 'field-name-1', count: 123 }, + { fieldName: 'field-name-2', count: 1111 }, + ], + latencyCorrelations: [ + { + fieldName: 'field-name-1', + fieldValue: 'field-value-1', + correlation: 0.5, + histogram: [{ key: 'the-key', doc_count: 123 }], + ksTest: 0.001, + }, + ], + overallHistogram: [ + { + doc_count: 1234, + key: 'the-key', + }, + ], + percentileThresholdValue: 1.234, + }); + } finally { + unmount(); + } + }); + }); + describe('when throwing an error', () => { + it('should automatically start fetching results', async () => { + const { result, unmount } = renderHook(() => useLatencyCorrelations(), { + wrapper, + initialProps: { + error: true, + }, + }); + + try { + expect(result.current.progress).toEqual({ + isRunning: true, + loaded: 0, + }); + } finally { + unmount(); + } + }); + + it('should still be running after 50ms', async () => { + const { result, unmount } = renderHook(() => useLatencyCorrelations(), { + wrapper, + initialProps: { + error: true, + }, + }); + + try { + jest.advanceTimersByTime(50); + + expect(result.current.progress).toEqual({ + isRunning: true, + loaded: 0, + }); + expect(result.current.response).toEqual({ ccsWarning: false }); + } finally { + unmount(); + } + }); + + it('should stop and return an error after more than 100ms', async () => { + const { result, unmount, waitFor } = renderHook( + () => useLatencyCorrelations(), + { + wrapper, + initialProps: { + error: true, + }, + } + ); + + try { + jest.advanceTimersByTime(150); + await waitFor(() => result.current.progress.error !== undefined); + + expect(result.current.progress).toEqual({ + error: 'Something went wrong', + isRunning: false, + loaded: 0, + }); + expect(result.current.response).toEqual({ + ccsWarning: false, + fieldStats: undefined, + latencyCorrelations: undefined, + overallHistogram: undefined, + percentileThresholdValue: undefined, + }); + } finally { + unmount(); + } + }); + }); +}); diff --git a/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts b/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts index d661e09a04d11..b0a222ca45460 100644 --- a/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts +++ b/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts @@ -8,7 +8,7 @@ import { useCallback, useEffect, useMemo, useReducer, useRef } from 'react'; import { chunk, debounce } from 'lodash'; -import { IHttpFetchError } from 'src/core/public'; +import { IHttpFetchError, ResponseErrorBody } from 'src/core/public'; import { DEBOUNCE_INTERVAL, @@ -85,7 +85,7 @@ export function useLatencyCorrelations() { }; // Initial call to fetch the overall distribution for the log-log plot. - const { overallHistogram } = await callApmApi({ + const { overallHistogram, percentileThresholdValue } = await callApmApi({ endpoint: 'POST /internal/apm/latency/overall_distribution', signal: abortCtrl.current.signal, params: { @@ -96,6 +96,7 @@ export function useLatencyCorrelations() { }, }); responseUpdate.overallHistogram = overallHistogram; + responseUpdate.percentileThresholdValue = percentileThresholdValue; if (isCancelledRef.current) { return; @@ -122,6 +123,7 @@ export function useLatencyCorrelations() { setResponse({ loaded: LOADED_FIELD_CANDIDATES, }); + setResponse.flush(); const chunkSize = 10; let chunkLoadCounter = 0; @@ -149,19 +151,21 @@ export function useLatencyCorrelations() { return; } + chunkLoadCounter++; setResponse({ loaded: LOADED_FIELD_CANDIDATES + (chunkLoadCounter / fieldCandidateChunks.length) * PROGRESS_STEP_FIELD_VALUE_PAIRS, }); - chunkLoadCounter++; } if (isCancelledRef.current) { return; } + setResponse.flush(); + chunkLoadCounter = 0; const fieldsToSample = new Set(); @@ -191,6 +195,7 @@ export function useLatencyCorrelations() { getLatencyCorrelationsSortedByCorrelation([...latencyCorrelations]); } + chunkLoadCounter++; setResponse({ ...responseUpdate, loaded: @@ -202,11 +207,11 @@ export function useLatencyCorrelations() { if (isCancelledRef.current) { return; } - - chunkLoadCounter++; } - const fieldStats = await callApmApi({ + setResponse.flush(); + + const { stats } = await callApmApi({ endpoint: 'POST /internal/apm/correlations/field_stats', signal: abortCtrl.current.signal, params: { @@ -217,7 +222,7 @@ export function useLatencyCorrelations() { }, }); - responseUpdate.fieldStats = fieldStats.stats; + responseUpdate.fieldStats = stats; setResponse({ ...responseUpdate, loaded: LOADED_DONE, @@ -225,15 +230,17 @@ export function useLatencyCorrelations() { }); setResponse.flush(); } catch (e) { - const err = e as Error | IHttpFetchError; - setResponse({ - error: - 'response' in err - ? err.body?.message ?? err.response?.statusText - : err.message, - isRunning: false, - }); - setResponse.flush(); + if (!isCancelledRef.current) { + const err = e as Error | IHttpFetchError; + setResponse({ + error: + 'response' in err + ? err.body?.message ?? err.response?.statusText + : err.message, + isRunning: false, + }); + setResponse.flush(); + } } }, [fetchParams, setResponse]); @@ -248,7 +255,10 @@ export function useLatencyCorrelations() { // auto-update useEffect(() => { startFetch(); - return cancelFetch; + return () => { + isCancelledRef.current = true; + abortCtrl.current.abort(); + }; }, [startFetch, cancelFetch]); const { error, loaded, isRunning, ...returnedResponse } = response; From b65bab6f985f6ceca45612f0aed99f392f2a68e1 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Wed, 3 Nov 2021 10:45:38 +0100 Subject: [PATCH 33/39] [ML] Focus assertion on error only. --- .../use_failed_transaction_correlations.test.tsx | 7 ------- .../app/correlations/use_latency_correlations.test.tsx | 7 ------- 2 files changed, 14 deletions(-) diff --git a/x-pack/plugins/apm/public/components/app/correlations/use_failed_transaction_correlations.test.tsx b/x-pack/plugins/apm/public/components/app/correlations/use_failed_transaction_correlations.test.tsx index 81e3428648fd3..e85f67eba035e 100644 --- a/x-pack/plugins/apm/public/components/app/correlations/use_failed_transaction_correlations.test.tsx +++ b/x-pack/plugins/apm/public/components/app/correlations/use_failed_transaction_correlations.test.tsx @@ -361,13 +361,6 @@ describe('useFailedTransactionsCorrelations', () => { isRunning: false, loaded: 0, }); - expect(result.current.response).toEqual({ - ccsWarning: false, - fieldStats: undefined, - latencyCorrelations: undefined, - overallHistogram: undefined, - percentileThresholdValue: undefined, - }); } finally { unmount(); } diff --git a/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.test.tsx b/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.test.tsx index 0f18284cb7351..ca3b5d7678e15 100644 --- a/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.test.tsx +++ b/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.test.tsx @@ -319,13 +319,6 @@ describe('useLatencyCorrelations', () => { isRunning: false, loaded: 0, }); - expect(result.current.response).toEqual({ - ccsWarning: false, - fieldStats: undefined, - latencyCorrelations: undefined, - overallHistogram: undefined, - percentileThresholdValue: undefined, - }); } finally { unmount(); } From 55472efd26dc6e816888648d88275d3e98fb994e Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Fri, 5 Nov 2021 10:56:03 +0100 Subject: [PATCH 34/39] [ML] Tweak test assertions and adds tests for cancellation. --- ...failed_transactions_correlations.test.tsx} | 42 +++++++++++++++-- .../use_failed_transactions_correlations.ts | 2 + .../use_latency_correlations.test.tsx | 47 ++++++++++++++++--- .../correlations/use_latency_correlations.ts | 2 + 4 files changed, 81 insertions(+), 12 deletions(-) rename x-pack/plugins/apm/public/components/app/correlations/{use_failed_transaction_correlations.test.tsx => use_failed_transactions_correlations.test.tsx} (90%) diff --git a/x-pack/plugins/apm/public/components/app/correlations/use_failed_transaction_correlations.test.tsx b/x-pack/plugins/apm/public/components/app/correlations/use_failed_transactions_correlations.test.tsx similarity index 90% rename from x-pack/plugins/apm/public/components/app/correlations/use_failed_transaction_correlations.test.tsx rename to x-pack/plugins/apm/public/components/app/correlations/use_failed_transactions_correlations.test.tsx index e85f67eba035e..f4252e2d95554 100644 --- a/x-pack/plugins/apm/public/components/app/correlations/use_failed_transaction_correlations.test.tsx +++ b/x-pack/plugins/apm/public/components/app/correlations/use_failed_transactions_correlations.test.tsx @@ -8,7 +8,7 @@ import React, { ReactNode } from 'react'; import { merge } from 'lodash'; import { createMemoryHistory } from 'history'; -import { renderHook } from '@testing-library/react-hooks'; +import { renderHook, act } from '@testing-library/react-hooks'; import { ApmPluginContextValue } from '../../../context/apm_plugin/apm_plugin_context'; import { @@ -196,7 +196,7 @@ describe('useFailedTransactionsCorrelations', () => { }); jest.advanceTimersByTime(100); - await waitFor(() => result.current.progress.loaded === 0.1); + await waitFor(() => expect(result.current.progress.loaded).toBe(0.1)); // field candidates are an implementation detail and // will not be exposed, it will just set loaded to 0.1. @@ -207,7 +207,7 @@ describe('useFailedTransactionsCorrelations', () => { }); jest.advanceTimersByTime(100); - await waitFor(() => result.current.progress.loaded === 1); + await waitFor(() => expect(result.current.progress.loaded).toBe(1)); expect(result.current.progress).toEqual({ error: undefined, @@ -248,7 +248,9 @@ describe('useFailedTransactionsCorrelations', () => { }); jest.advanceTimersByTime(100); - await waitFor(() => result.current.response.fieldStats !== undefined); + await waitFor(() => + expect(result.current.response.fieldStats).toBeDefined() + ); expect(result.current.progress).toEqual({ error: undefined, @@ -354,7 +356,9 @@ describe('useFailedTransactionsCorrelations', () => { try { jest.advanceTimersByTime(150); - await waitFor(() => result.current.progress.error !== undefined); + await waitFor(() => + expect(result.current.progress.error).toBeDefined() + ); expect(result.current.progress).toEqual({ error: 'Something went wrong', @@ -366,4 +370,32 @@ describe('useFailedTransactionsCorrelations', () => { } }); }); + + describe('when canceled', () => { + it('should stop running', async () => { + const { result, unmount, waitFor } = renderHook( + () => useFailedTransactionsCorrelations(), + { + wrapper, + } + ); + + try { + jest.advanceTimersByTime(50); + await waitFor(() => expect(result.current.progress.loaded).toBe(0)); + + expect(result.current.progress.isRunning).toBe(true); + + act(() => { + result.current.cancelFetch(); + }); + + await waitFor(() => + expect(result.current.progress.isRunning).toEqual(false) + ); + } finally { + unmount(); + } + }); + }); }); diff --git a/x-pack/plugins/apm/public/components/app/correlations/use_failed_transactions_correlations.ts b/x-pack/plugins/apm/public/components/app/correlations/use_failed_transactions_correlations.ts index f03fa4998e4be..a1283ec8d7c28 100644 --- a/x-pack/plugins/apm/public/components/app/correlations/use_failed_transactions_correlations.ts +++ b/x-pack/plugins/apm/public/components/app/correlations/use_failed_transactions_correlations.ts @@ -44,6 +44,7 @@ export function useFailedTransactionsCorrelations() { // This use of useReducer (the dispatch function won't get reinstantiated // on every update) and debounce avoids flooding consuming components with updates. + // `setResponse.flush()` can be used to enforce an update. const [response, setResponseUnDebounced] = useReducer( getReducer(), getInitialResponse() @@ -222,6 +223,7 @@ export function useFailedTransactionsCorrelations() { setResponse({ isRunning: false, }); + setResponse.flush(); }, [setResponse]); // auto-update diff --git a/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.test.tsx b/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.test.tsx index ca3b5d7678e15..90d976c389c58 100644 --- a/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.test.tsx +++ b/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.test.tsx @@ -8,7 +8,7 @@ import React, { ReactNode } from 'react'; import { merge } from 'lodash'; import { createMemoryHistory } from 'history'; -import { renderHook } from '@testing-library/react-hooks'; +import { renderHook, act } from '@testing-library/react-hooks'; import { ApmPluginContextValue } from '../../../context/apm_plugin/apm_plugin_context'; import { @@ -150,7 +150,7 @@ describe('useLatencyCorrelations', () => { try { jest.advanceTimersByTime(150); - await waitFor(() => result.current.progress.loaded === 0.05); + await waitFor(() => expect(result.current.progress.loaded).toBe(0.05)); expect(result.current.progress).toEqual({ error: undefined, @@ -171,7 +171,7 @@ describe('useLatencyCorrelations', () => { }); jest.advanceTimersByTime(100); - await waitFor(() => result.current.progress.loaded === 0.1); + await waitFor(() => expect(result.current.progress.loaded).toBe(0.1)); // field candidates are an implementation detail and // will not be exposed, it will just set loaded to 0.1. @@ -182,7 +182,7 @@ describe('useLatencyCorrelations', () => { }); jest.advanceTimersByTime(100); - await waitFor(() => result.current.progress.loaded === 0.4); + await waitFor(() => expect(result.current.progress.loaded).toBe(0.4)); // field value pairs are an implementation detail and // will not be exposed, it will just set loaded to 0.4. @@ -193,7 +193,7 @@ describe('useLatencyCorrelations', () => { }); jest.advanceTimersByTime(100); - await waitFor(() => result.current.progress.loaded === 1); + await waitFor(() => expect(result.current.progress.loaded).toBe(1)); expect(result.current.progress).toEqual({ error: undefined, @@ -223,7 +223,9 @@ describe('useLatencyCorrelations', () => { }); jest.advanceTimersByTime(100); - await waitFor(() => result.current.response.fieldStats !== undefined); + await waitFor(() => + expect(result.current.response.fieldStats).toBeDefined() + ); expect(result.current.progress).toEqual({ error: undefined, @@ -259,6 +261,7 @@ describe('useLatencyCorrelations', () => { } }); }); + describe('when throwing an error', () => { it('should automatically start fetching results', async () => { const { result, unmount } = renderHook(() => useLatencyCorrelations(), { @@ -312,7 +315,9 @@ describe('useLatencyCorrelations', () => { try { jest.advanceTimersByTime(150); - await waitFor(() => result.current.progress.error !== undefined); + await waitFor(() => + expect(result.current.progress.error).toBeDefined() + ); expect(result.current.progress).toEqual({ error: 'Something went wrong', @@ -324,4 +329,32 @@ describe('useLatencyCorrelations', () => { } }); }); + + describe('when canceled', () => { + it('should stop running', async () => { + const { result, unmount, waitFor } = renderHook( + () => useLatencyCorrelations(), + { + wrapper, + } + ); + + try { + jest.advanceTimersByTime(150); + await waitFor(() => expect(result.current.progress.loaded).toBe(0.05)); + + expect(result.current.progress.isRunning).toBe(true); + + act(() => { + result.current.cancelFetch(); + }); + + await waitFor(() => + expect(result.current.progress.isRunning).toEqual(false) + ); + } finally { + unmount(); + } + }); + }); }); diff --git a/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts b/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts index b0a222ca45460..a5fd9adc96cd1 100644 --- a/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts +++ b/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts @@ -46,6 +46,7 @@ export function useLatencyCorrelations() { // This use of useReducer (the dispatch function won't get reinstantiated // on every update) and debounce avoids flooding consuming components with updates. + // `setResponse.flush()` can be used to enforce an update. const [response, setResponseUnDebounced] = useReducer( getReducer(), getInitialResponse() @@ -250,6 +251,7 @@ export function useLatencyCorrelations() { setResponse({ isRunning: false, }); + setResponse.flush(); }, [setResponse]); // auto-update From 72bcc11aaa2b6ee8b74b3df447861a0c25ac6b7c Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Fri, 5 Nov 2021 12:24:22 +0100 Subject: [PATCH 35/39] [ML] Adds comment to clarify the use of abortControler and isCancelledRef --- .../app/correlations/use_failed_transactions_correlations.ts | 5 +++-- .../components/app/correlations/use_latency_correlations.ts | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/apm/public/components/app/correlations/use_failed_transactions_correlations.ts b/x-pack/plugins/apm/public/components/app/correlations/use_failed_transactions_correlations.ts index a1283ec8d7c28..2aa9134b96126 100644 --- a/x-pack/plugins/apm/public/components/app/correlations/use_failed_transactions_correlations.ts +++ b/x-pack/plugins/apm/public/components/app/correlations/use_failed_transactions_correlations.ts @@ -38,8 +38,6 @@ const LOADED_DONE = 1; const PROGRESS_STEP_P_VALUES = 0.9; export function useFailedTransactionsCorrelations() { - const abortCtrl = useRef(new AbortController()); - const fetchParams = useFetchParams(); // This use of useReducer (the dispatch function won't get reinstantiated @@ -54,6 +52,9 @@ export function useFailedTransactionsCorrelations() { [] ); + // `abortCtrl` is used to cancel individual requests that already started. + // `isCancelledRef` is used to cancel the overall task in between requests in the `startFetch` callback. + const abortCtrl = useRef(new AbortController()); // We're using a ref here because otherwise the startFetch function might have // a stale value for checking if the task has been cancelled. const isCancelledRef = useRef(false); diff --git a/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts b/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts index a5fd9adc96cd1..80c465c795b60 100644 --- a/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts +++ b/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts @@ -40,8 +40,6 @@ const PROGRESS_STEP_FIELD_VALUE_PAIRS = 0.3; const PROGRESS_STEP_CORRELATIONS = 0.6; export function useLatencyCorrelations() { - const abortCtrl = useRef(new AbortController()); - const fetchParams = useFetchParams(); // This use of useReducer (the dispatch function won't get reinstantiated @@ -56,6 +54,9 @@ export function useLatencyCorrelations() { [] ); + // `abortCtrl` is used to cancel individual requests that already started. + // `isCancelledRef` is used to cancel the overall task in between requests in the `startFetch` callback. + const abortCtrl = useRef(new AbortController()); // We're using a ref here because otherwise the startFetch function might have // a stale value for checking if the task has been cancelled. const isCancelledRef = useRef(false); From 97890cff0ef9e7537b9f3859fdbff04661d80ad1 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Fri, 5 Nov 2021 13:51:21 +0100 Subject: [PATCH 36/39] [ML] Use abort signal instead of isCancelledRef. --- ..._failed_transactions_correlations.test.tsx | 2 - .../use_failed_transactions_correlations.ts | 78 ++++++++++--------- .../correlations/use_latency_correlations.ts | 20 ++--- 3 files changed, 46 insertions(+), 54 deletions(-) diff --git a/x-pack/plugins/apm/public/components/app/correlations/use_failed_transactions_correlations.test.tsx b/x-pack/plugins/apm/public/components/app/correlations/use_failed_transactions_correlations.test.tsx index f4252e2d95554..929cc4f7f4cd3 100644 --- a/x-pack/plugins/apm/public/components/app/correlations/use_failed_transactions_correlations.test.tsx +++ b/x-pack/plugins/apm/public/components/app/correlations/use_failed_transactions_correlations.test.tsx @@ -167,8 +167,6 @@ describe('useFailedTransactionsCorrelations', () => { jest.advanceTimersByTime(100); await waitFor(() => expect(result.current.progress.loaded).toBe(0)); jest.advanceTimersByTime(100); - await waitFor(() => expect(result.current.progress.loaded).toBe(0)); - jest.advanceTimersByTime(100); await waitFor(() => expect(result.current.progress.loaded).toBe(0.05)); expect(result.current.progress).toEqual({ diff --git a/x-pack/plugins/apm/public/components/app/correlations/use_failed_transactions_correlations.ts b/x-pack/plugins/apm/public/components/app/correlations/use_failed_transactions_correlations.ts index 2aa9134b96126..163223e744a22 100644 --- a/x-pack/plugins/apm/public/components/app/correlations/use_failed_transactions_correlations.ts +++ b/x-pack/plugins/apm/public/components/app/correlations/use_failed_transactions_correlations.ts @@ -52,17 +52,11 @@ export function useFailedTransactionsCorrelations() { [] ); - // `abortCtrl` is used to cancel individual requests that already started. - // `isCancelledRef` is used to cancel the overall task in between requests in the `startFetch` callback. const abortCtrl = useRef(new AbortController()); - // We're using a ref here because otherwise the startFetch function might have - // a stale value for checking if the task has been cancelled. - const isCancelledRef = useRef(false); const startFetch = useCallback(async () => { abortCtrl.current.abort(); abortCtrl.current = new AbortController(); - isCancelledRef.current = false; setResponse({ ...getInitialResponse(), @@ -85,36 +79,46 @@ export function useFailedTransactionsCorrelations() { ccsWarning: false, }; - // Initial call to fetch the overall distribution for the log-log plot. - const { overallHistogram, percentileThresholdValue } = await callApmApi({ - endpoint: 'POST /internal/apm/latency/overall_distribution', - signal: abortCtrl.current.signal, - params: { - body: { - ...fetchParams, - percentileThreshold: DEFAULT_PERCENTILE_THRESHOLD, - }, - }, - }); - responseUpdate.overallHistogram = overallHistogram; - responseUpdate.percentileThresholdValue = percentileThresholdValue; + const [overallHistogramResponse, errorHistogramRespone] = + await Promise.all([ + // Initial call to fetch the overall distribution for the log-log plot. + callApmApi({ + endpoint: 'POST /internal/apm/latency/overall_distribution', + signal: abortCtrl.current.signal, + params: { + body: { + ...fetchParams, + percentileThreshold: DEFAULT_PERCENTILE_THRESHOLD, + }, + }, + }), + callApmApi({ + endpoint: 'POST /internal/apm/latency/overall_distribution', + signal: abortCtrl.current.signal, + params: { + body: { + ...fetchParams, + percentileThreshold: DEFAULT_PERCENTILE_THRESHOLD, + termFilters: [ + { + fieldName: EVENT_OUTCOME, + fieldValue: EventOutcome.failure, + }, + ], + }, + }, + }), + ]); + + const { overallHistogram, percentileThresholdValue } = + overallHistogramResponse; + const { overallHistogram: errorHistogram } = errorHistogramRespone; - const { overallHistogram: errorHistogram } = await callApmApi({ - endpoint: 'POST /internal/apm/latency/overall_distribution', - signal: abortCtrl.current.signal, - params: { - body: { - ...fetchParams, - percentileThreshold: DEFAULT_PERCENTILE_THRESHOLD, - termFilters: [ - { fieldName: EVENT_OUTCOME, fieldValue: EventOutcome.failure }, - ], - }, - }, - }); responseUpdate.errorHistogram = errorHistogram; + responseUpdate.overallHistogram = overallHistogram; + responseUpdate.percentileThresholdValue = percentileThresholdValue; - if (isCancelledRef.current) { + if (abortCtrl.current.signal.aborted) { return; } @@ -132,7 +136,7 @@ export function useFailedTransactionsCorrelations() { }, }); - if (isCancelledRef.current) { + if (abortCtrl.current.signal.aborted) { return; } @@ -182,7 +186,7 @@ export function useFailedTransactionsCorrelations() { PROGRESS_STEP_P_VALUES, }); - if (isCancelledRef.current) { + if (abortCtrl.current.signal.aborted) { return; } } @@ -204,7 +208,7 @@ export function useFailedTransactionsCorrelations() { setResponse({ ...responseUpdate, loaded: LOADED_DONE, isRunning: false }); setResponse.flush(); } catch (e) { - if (!isCancelledRef.current) { + if (!abortCtrl.current.signal.aborted) { const err = e as Error | IHttpFetchError; setResponse({ error: @@ -219,7 +223,6 @@ export function useFailedTransactionsCorrelations() { }, [fetchParams, setResponse]); const cancelFetch = useCallback(() => { - isCancelledRef.current = true; abortCtrl.current.abort(); setResponse({ isRunning: false, @@ -231,7 +234,6 @@ export function useFailedTransactionsCorrelations() { useEffect(() => { startFetch(); return () => { - isCancelledRef.current = true; abortCtrl.current.abort(); }; }, [startFetch, cancelFetch]); diff --git a/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts b/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts index 80c465c795b60..358d436f8f0a5 100644 --- a/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts +++ b/x-pack/plugins/apm/public/components/app/correlations/use_latency_correlations.ts @@ -54,17 +54,11 @@ export function useLatencyCorrelations() { [] ); - // `abortCtrl` is used to cancel individual requests that already started. - // `isCancelledRef` is used to cancel the overall task in between requests in the `startFetch` callback. const abortCtrl = useRef(new AbortController()); - // We're using a ref here because otherwise the startFetch function might have - // a stale value for checking if the task has been cancelled. - const isCancelledRef = useRef(false); const startFetch = useCallback(async () => { abortCtrl.current.abort(); abortCtrl.current = new AbortController(); - isCancelledRef.current = false; setResponse({ ...getInitialResponse(), @@ -100,7 +94,7 @@ export function useLatencyCorrelations() { responseUpdate.overallHistogram = overallHistogram; responseUpdate.percentileThresholdValue = percentileThresholdValue; - if (isCancelledRef.current) { + if (abortCtrl.current.signal.aborted) { return; } @@ -118,7 +112,7 @@ export function useLatencyCorrelations() { }, }); - if (isCancelledRef.current) { + if (abortCtrl.current.signal.aborted) { return; } @@ -149,7 +143,7 @@ export function useLatencyCorrelations() { fieldValuePairs.push(...fieldValuePairChunkResponse.fieldValuePairs); } - if (isCancelledRef.current) { + if (abortCtrl.current.signal.aborted) { return; } @@ -162,7 +156,7 @@ export function useLatencyCorrelations() { }); } - if (isCancelledRef.current) { + if (abortCtrl.current.signal.aborted) { return; } @@ -206,7 +200,7 @@ export function useLatencyCorrelations() { PROGRESS_STEP_CORRELATIONS, }); - if (isCancelledRef.current) { + if (abortCtrl.current.signal.aborted) { return; } } @@ -232,7 +226,7 @@ export function useLatencyCorrelations() { }); setResponse.flush(); } catch (e) { - if (!isCancelledRef.current) { + if (!abortCtrl.current.signal.aborted) { const err = e as Error | IHttpFetchError; setResponse({ error: @@ -247,7 +241,6 @@ export function useLatencyCorrelations() { }, [fetchParams, setResponse]); const cancelFetch = useCallback(() => { - isCancelledRef.current = true; abortCtrl.current.abort(); setResponse({ isRunning: false, @@ -259,7 +252,6 @@ export function useLatencyCorrelations() { useEffect(() => { startFetch(); return () => { - isCancelledRef.current = true; abortCtrl.current.abort(); }; }, [startFetch, cancelFetch]); From 86b78c14416acfd21e7b082308c501998dfc59e5 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Fri, 5 Nov 2021 20:13:25 +0100 Subject: [PATCH 37/39] [ML] Fix test imports. --- .../apm_api_integration/tests/correlations/field_candidates.ts | 2 +- .../apm_api_integration/tests/correlations/field_value_pairs.ts | 2 +- x-pack/test/apm_api_integration/tests/correlations/p_values.ts | 2 +- .../tests/correlations/significant_correlations.ts | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/x-pack/test/apm_api_integration/tests/correlations/field_candidates.ts b/x-pack/test/apm_api_integration/tests/correlations/field_candidates.ts index 16c157d026d33..a62145da25326 100644 --- a/x-pack/test/apm_api_integration/tests/correlations/field_candidates.ts +++ b/x-pack/test/apm_api_integration/tests/correlations/field_candidates.ts @@ -7,10 +7,10 @@ import expect from '@kbn/expect'; import { FtrProviderContext } from '../../common/ftr_provider_context'; -import { registry } from '../../common/registry'; export default function ApiTest({ getService }: FtrProviderContext) { const apmApiClient = getService('apmApiClient'); + const registry = getService('registry'); const endpoint = 'GET /internal/apm/correlations/field_candidates'; diff --git a/x-pack/test/apm_api_integration/tests/correlations/field_value_pairs.ts b/x-pack/test/apm_api_integration/tests/correlations/field_value_pairs.ts index f0a0e17561ac7..df9314546d6de 100644 --- a/x-pack/test/apm_api_integration/tests/correlations/field_value_pairs.ts +++ b/x-pack/test/apm_api_integration/tests/correlations/field_value_pairs.ts @@ -7,10 +7,10 @@ import expect from '@kbn/expect'; import { FtrProviderContext } from '../../common/ftr_provider_context'; -import { registry } from '../../common/registry'; export default function ApiTest({ getService }: FtrProviderContext) { const apmApiClient = getService('apmApiClient'); + const registry = getService('registry'); const endpoint = 'POST /internal/apm/correlations/field_value_pairs'; diff --git a/x-pack/test/apm_api_integration/tests/correlations/p_values.ts b/x-pack/test/apm_api_integration/tests/correlations/p_values.ts index 444cf57d35325..1f3dd58063087 100644 --- a/x-pack/test/apm_api_integration/tests/correlations/p_values.ts +++ b/x-pack/test/apm_api_integration/tests/correlations/p_values.ts @@ -7,10 +7,10 @@ import expect from '@kbn/expect'; import { FtrProviderContext } from '../../common/ftr_provider_context'; -import { registry } from '../../common/registry'; export default function ApiTest({ getService }: FtrProviderContext) { const apmApiClient = getService('apmApiClient'); + const registry = getService('registry'); const endpoint = 'POST /internal/apm/correlations/p_values'; diff --git a/x-pack/test/apm_api_integration/tests/correlations/significant_correlations.ts b/x-pack/test/apm_api_integration/tests/correlations/significant_correlations.ts index 404584f959380..994f23bbf2a4e 100644 --- a/x-pack/test/apm_api_integration/tests/correlations/significant_correlations.ts +++ b/x-pack/test/apm_api_integration/tests/correlations/significant_correlations.ts @@ -7,10 +7,10 @@ import expect from '@kbn/expect'; import { FtrProviderContext } from '../../common/ftr_provider_context'; -import { registry } from '../../common/registry'; export default function ApiTest({ getService }: FtrProviderContext) { const apmApiClient = getService('apmApiClient'); + const registry = getService('registry'); const endpoint = 'POST /internal/apm/correlations/significant_correlations'; From 22f785cd9919cbd753c7b988fd8ebf570d513c05 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Mon, 8 Nov 2021 12:16:02 +0100 Subject: [PATCH 38/39] fix field value pair error handling --- .../queries/query_field_value_pairs.ts | 14 +++++++----- .../plugins/apm/server/routes/correlations.ts | 22 ++++++++++--------- 2 files changed, 21 insertions(+), 15 deletions(-) diff --git a/x-pack/plugins/apm/server/lib/correlations/queries/query_field_value_pairs.ts b/x-pack/plugins/apm/server/lib/correlations/queries/query_field_value_pairs.ts index e4c61e31e6fb1..16c4dacb5ef95 100644 --- a/x-pack/plugins/apm/server/lib/correlations/queries/query_field_value_pairs.ts +++ b/x-pack/plugins/apm/server/lib/correlations/queries/query_field_value_pairs.ts @@ -15,6 +15,8 @@ import type { } from '../../../../common/correlations/types'; import { TERMS_SIZE } from '../../../../common/correlations/constants'; +import { splitAllSettledPromises } from '../utils'; + import { getQueryWithParams } from './get_query_with_params'; import { getRequestBase } from './get_request_base'; @@ -73,12 +75,14 @@ export const fetchTransactionDurationFieldValuePairs = async ( esClient: ElasticsearchClient, params: CorrelationsParams, fieldCandidates: string[] -): Promise => { - const responses = await Promise.all( - fieldCandidates.map((fieldCandidate) => - fetchTransactionDurationFieldTerms(esClient, params, fieldCandidate) +): Promise<{ fieldValuePairs: FieldValuePair[]; errors: any[] }> => { + const { fulfilled: responses, rejected: errors } = splitAllSettledPromises( + await Promise.allSettled( + fieldCandidates.map((fieldCandidate) => + fetchTransactionDurationFieldTerms(esClient, params, fieldCandidate) + ) ) ); - return responses.flat(); + return { fieldValuePairs: responses.flat(), errors }; }; diff --git a/x-pack/plugins/apm/server/routes/correlations.ts b/x-pack/plugins/apm/server/routes/correlations.ts index 479c3d8ca73bf..8b20d57d25d67 100644 --- a/x-pack/plugins/apm/server/routes/correlations.ts +++ b/x-pack/plugins/apm/server/routes/correlations.ts @@ -141,16 +141,18 @@ const fieldValuePairsRoute = createApmServerRoute({ const { fieldCandidates, ...params } = resources.params.body; - return withApmSpan('get_correlations_field_value_pairs', async () => ({ - fieldValuePairs: await fetchTransactionDurationFieldValuePairs( - esClient, - { - ...params, - index: indices.transaction, - }, - fieldCandidates - ), - })); + return withApmSpan( + 'get_correlations_field_value_pairs', + async () => + await fetchTransactionDurationFieldValuePairs( + esClient, + { + ...params, + index: indices.transaction, + }, + fieldCandidates + ) + ); }, }); From 9872afc413f96b60fab7e665135cbc4dcb732a1e Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Mon, 8 Nov 2021 14:07:46 +0100 Subject: [PATCH 39/39] [ML] Fix jest test. --- .../lib/correlations/queries/query_field_value_pairs.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/apm/server/lib/correlations/queries/query_field_value_pairs.test.ts b/x-pack/plugins/apm/server/lib/correlations/queries/query_field_value_pairs.test.ts index 91a32cc0f5297..80016930184b3 100644 --- a/x-pack/plugins/apm/server/lib/correlations/queries/query_field_value_pairs.test.ts +++ b/x-pack/plugins/apm/server/lib/correlations/queries/query_field_value_pairs.test.ts @@ -69,7 +69,8 @@ describe('query_field_value_pairs', () => { fieldCandidates ); - expect(resp).toEqual([ + expect(resp.errors).toEqual([]); + expect(resp.fieldValuePairs).toEqual([ { fieldName: 'myFieldCandidate1', fieldValue: 'myValue1' }, { fieldName: 'myFieldCandidate1', fieldValue: 'myValue2' }, { fieldName: 'myFieldCandidate2', fieldValue: 'myValue1' },