From f94c42ee7317a0826e20b7557b9341dcb54c2037 Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Tue, 29 Jun 2021 19:44:48 -0400 Subject: [PATCH] [ML] APM Latency Correlations (#99905) (#103745) * [ML] Async Search Service. * [ML] Log Log Area Charts Grid. Scatterplot with streaming data. * [ML] Remove scatterplot streaming demo. * [ML] Improved histogram bins for log scale. * [ML] Move client side code from Ml to APM plugin. * [ML] Adds correlation table. * [ML] Layout tweaks. * [ML] Move server side code from ML to APM. * [ML] Remove console.logs. * [ML] Fix types. * [ML] Chart area line type fix. Slowness %. * [ML] Comment unused vars. * Fix missing pluginsStart data in context * Add KS test value and rename table columns * Update percentiles to be correct * Make columns optional * Update fractionals/expectations to match with backend logic * Update so progress is 100% when all is completed * Make pre-processing steps smaller part of overall progress(to show charts earlier) * Add no correlations found message * Fix progress logic * Fix incorrect threshold, types * Add back APM filtering functionality to match with other table * Improve histogram equality check with random sampling * Show overall latency distribution right away * Rename demo tab to latencyCorrelationsTab * Update percentiles query to use hdr * [ML] Fix issue where apm-* might have multiple indices with different mappings where keyword is not the only type - Fix to check for keyword mapping more thoroughly - Add try catch if one of the es search fail, don't quit the entire fetch * Remove commented code * Remove kstest column, round correlation to 2 sigfig dec * Remove old latency tab, replace with ml latency correlations tab as first/default one * Set axis to start at 0 because agg may results in weird * Remove commented code for grouping duplicates * Update msg to mean significant correlations * Add i18n * Change correlations flyout back to medium size * Add name of service or transaction for clarity * Share i18n * Consolidate roundToDecimalPlace usage * Remove redundant isDuplicate * Create MAX_CORRELATIONS_TO_SHOW * Update mlCorrelationcolumns * Fix i18n quotations * Update query to include filter * Revert "Update query to include filter" This reverts commit 9a37eec0 * Rename MlCorrelations to MlLatencyCorrelations for clarity * Add pagination * Update include/exclude logic for field candidates and add ip field support * Add transactionName filter suport * Reorganize math utils * Group duplicates together * Fix typescript, better hasPrefixToInclude support check, * Remove Finished toast * Add title to y axis * Reduce number of tick labels to show * Highlight table row that is being used for graph * Add from to follow MDN guideline * Add APM style prefix * Fix i18n * [ML] Fix logic for tick format to only show power of 10 * Replace roundToDecimalPlace with asPreciseDecimal * Switch to lodash range * Clean up get_query_with_params * Prioritize candidates using field_terms * Update percentiles result type to be array instead of objects * Use observability's rangeQuery instead * Change arg format of query * Revert candidate_terms logic * Consolidate fractions, expectations, and ranges calc * Add tooltip for Correlation * Change terms size to 20 * Move env/service/transaction sticky header to top level, remove link * Add support for http.response.status_code * Replace histogram circular markers with bars * Delete unused roundToDecimalPlace * Add fractions calculation * Make notes of fractions and fix bucket correlation * Remove any, commented code, consolidate cancelFetch * Use es6 max * Align tooltip at the top * Get rid of getCoreServices, param docs, rename type, remove rangeQuery * Adjust range * Show all values without grouping duplicates * Fix pagination * Make flyout larger Co-authored-by: Quynh Nguyen Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Co-authored-by: Walter Rafelsberger Co-authored-by: Quynh Nguyen --- .../search_strategies/correlations/types.ts | 51 +++ .../apm/common/utils/formatters/formatters.ts | 8 + .../plugins/apm/public/application/index.tsx | 1 + .../app/correlations/correlations_chart.tsx | 221 ++++++++++ .../app/correlations/correlations_table.tsx | 62 ++- .../components/app/correlations/index.tsx | 155 ++++--- .../correlations/ml_latency_correlations.tsx | 382 ++++++++++++++++++ .../app/correlations/use_correlations.ts | 120 ++++++ .../correlations/async_search_service.ts | 225 +++++++++++ .../correlations/constants.ts | 78 ++++ .../correlations/get_query_with_params.ts | 89 ++++ .../search_strategies/correlations/index.ts | 8 + .../correlations/query_correlation.ts | 139 +++++++ .../correlations/query_field_candidates.ts | 105 +++++ .../correlations/query_field_value_pairs.ts | 89 ++++ .../correlations/query_fractions.ts | 64 +++ .../correlations/query_histogram.ts | 70 ++++ .../correlations/query_histogram_interval.ts | 52 +++ .../query_histogram_rangesteps.ts | 65 +++ .../correlations/query_percentiles.ts | 84 ++++ .../correlations/query_ranges.ts | 96 +++++ .../correlations/search_strategy.ts | 94 +++++ .../correlations/utils/aggregation_utils.ts | 53 +++ .../correlations/utils/index.ts | 9 + .../correlations/utils/math_utils.ts | 70 ++++ x-pack/plugins/apm/server/plugin.ts | 8 + x-pack/plugins/ml/server/types.ts | 7 + 27 files changed, 2353 insertions(+), 52 deletions(-) create mode 100644 x-pack/plugins/apm/common/search_strategies/correlations/types.ts create mode 100644 x-pack/plugins/apm/public/components/app/correlations/correlations_chart.tsx create mode 100644 x-pack/plugins/apm/public/components/app/correlations/ml_latency_correlations.tsx create mode 100644 x-pack/plugins/apm/public/components/app/correlations/use_correlations.ts create mode 100644 x-pack/plugins/apm/server/lib/search_strategies/correlations/async_search_service.ts create mode 100644 x-pack/plugins/apm/server/lib/search_strategies/correlations/constants.ts create mode 100644 x-pack/plugins/apm/server/lib/search_strategies/correlations/get_query_with_params.ts create mode 100644 x-pack/plugins/apm/server/lib/search_strategies/correlations/index.ts create mode 100644 x-pack/plugins/apm/server/lib/search_strategies/correlations/query_correlation.ts create mode 100644 x-pack/plugins/apm/server/lib/search_strategies/correlations/query_field_candidates.ts create mode 100644 x-pack/plugins/apm/server/lib/search_strategies/correlations/query_field_value_pairs.ts create mode 100644 x-pack/plugins/apm/server/lib/search_strategies/correlations/query_fractions.ts create mode 100644 x-pack/plugins/apm/server/lib/search_strategies/correlations/query_histogram.ts create mode 100644 x-pack/plugins/apm/server/lib/search_strategies/correlations/query_histogram_interval.ts create mode 100644 x-pack/plugins/apm/server/lib/search_strategies/correlations/query_histogram_rangesteps.ts create mode 100644 x-pack/plugins/apm/server/lib/search_strategies/correlations/query_percentiles.ts create mode 100644 x-pack/plugins/apm/server/lib/search_strategies/correlations/query_ranges.ts create mode 100644 x-pack/plugins/apm/server/lib/search_strategies/correlations/search_strategy.ts create mode 100644 x-pack/plugins/apm/server/lib/search_strategies/correlations/utils/aggregation_utils.ts create mode 100644 x-pack/plugins/apm/server/lib/search_strategies/correlations/utils/index.ts create mode 100644 x-pack/plugins/apm/server/lib/search_strategies/correlations/utils/math_utils.ts diff --git a/x-pack/plugins/apm/common/search_strategies/correlations/types.ts b/x-pack/plugins/apm/common/search_strategies/correlations/types.ts new file mode 100644 index 0000000000000..1698708aeb77e --- /dev/null +++ b/x-pack/plugins/apm/common/search_strategies/correlations/types.ts @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export interface HistogramItem { + key: number; + doc_count: number; +} + +export interface ResponseHitSource { + [s: string]: unknown; +} + +export interface ResponseHit { + _source: ResponseHitSource; +} + +export interface SearchServiceParams { + index: string; + environment?: string; + kuery?: string; + serviceName?: string; + transactionName?: string; + transactionType?: string; + start?: string; + end?: string; + percentileThreshold?: number; + percentileThresholdValue?: number; +} + +export interface SearchServiceValue { + histogram: HistogramItem[]; + value: string; + field: string; + correlation: number; + ksTest: number; + duplicatedFields?: string[]; +} + +export interface AsyncSearchProviderProgress { + started: number; + loadedHistogramStepsize: number; + loadedOverallHistogram: number; + loadedFieldCanditates: number; + loadedFieldValuePairs: number; + loadedHistograms: number; + getOverallProgress: () => number; +} diff --git a/x-pack/plugins/apm/common/utils/formatters/formatters.ts b/x-pack/plugins/apm/common/utils/formatters/formatters.ts index 8fb593f8bab26..4da73a6d2c29a 100644 --- a/x-pack/plugins/apm/common/utils/formatters/formatters.ts +++ b/x-pack/plugins/apm/common/utils/formatters/formatters.ts @@ -18,6 +18,14 @@ export function asDecimal(value?: number | null) { return numeral(value).format('0,0.0'); } +export function asPreciseDecimal(value?: number | null, dp: number = 3) { + if (!isFiniteNumber(value)) { + return NOT_AVAILABLE_LABEL; + } + + return numeral(value).format(`0,0.${'0'.repeat(dp)}`); +} + export function asInteger(value?: number | null) { if (!isFiniteNumber(value)) { return NOT_AVAILABLE_LABEL; diff --git a/x-pack/plugins/apm/public/application/index.tsx b/x-pack/plugins/apm/public/application/index.tsx index 6924a8b9161b2..a6b0dc61a3260 100644 --- a/x-pack/plugins/apm/public/application/index.tsx +++ b/x-pack/plugins/apm/public/application/index.tsx @@ -47,6 +47,7 @@ export const renderApp = ({ config, core: coreStart, plugins: pluginsSetup, + data: pluginsStart.data, observability: pluginsStart.observability, observabilityRuleTypeRegistry, }; diff --git a/x-pack/plugins/apm/public/components/app/correlations/correlations_chart.tsx b/x-pack/plugins/apm/public/components/app/correlations/correlations_chart.tsx new file mode 100644 index 0000000000000..f4e39c37e289e --- /dev/null +++ b/x-pack/plugins/apm/public/components/app/correlations/correlations_chart.tsx @@ -0,0 +1,221 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import { + AnnotationDomainType, + Chart, + CurveType, + Settings, + Axis, + ScaleType, + Position, + AreaSeries, + RecursivePartial, + AxisStyle, + PartialTheme, + LineAnnotation, + LineAnnotationDatum, +} from '@elastic/charts'; + +import euiVars from '@elastic/eui/dist/eui_theme_light.json'; + +import { EuiSpacer } from '@elastic/eui'; + +import { i18n } from '@kbn/i18n'; + +import { getDurationFormatter } from '../../../../common/utils/formatters'; + +import { useTheme } from '../../../hooks/use_theme'; +import { HistogramItem } from '../../../../common/search_strategies/correlations/types'; + +const { euiColorMediumShade } = euiVars; +const axisColor = euiColorMediumShade; + +const axes: RecursivePartial = { + axisLine: { + stroke: axisColor, + }, + tickLabel: { + fontSize: 10, + fill: axisColor, + padding: 0, + }, + tickLine: { + stroke: axisColor, + size: 5, + }, + gridLine: { + horizontal: { + dash: [1, 2], + }, + vertical: { + strokeWidth: 1, + }, + }, +}; +const chartTheme: PartialTheme = { + axes, + legend: { + spacingBuffer: 100, + }, + areaSeriesStyle: { + line: { + visible: false, + }, + }, +}; + +interface CorrelationsChartProps { + field?: string; + value?: string; + histogram?: HistogramItem[]; + markerValue: number; + markerPercentile: number; + overallHistogram: HistogramItem[]; +} + +const annotationsStyle = { + line: { + strokeWidth: 1, + stroke: 'gray', + opacity: 0.8, + }, + details: { + fontSize: 8, + fontFamily: 'Arial', + fontStyle: 'normal', + fill: 'gray', + padding: 0, + }, +}; + +const CHART_PLACEHOLDER_VALUE = 0.0001; + +// Elastic charts will show any lone bin (i.e. a populated bin followed by empty bin) +// as a circular marker instead of a bar +// This provides a workaround by making the next bin not empty +export const replaceHistogramDotsWithBars = ( + originalHistogram: HistogramItem[] | undefined +) => { + if (originalHistogram === undefined) return; + const histogram = [...originalHistogram]; + { + for (let i = 0; i < histogram.length - 1; i++) { + if ( + histogram[i].doc_count > 0 && + histogram[i].doc_count !== CHART_PLACEHOLDER_VALUE && + histogram[i + 1].doc_count === 0 + ) { + histogram[i + 1].doc_count = CHART_PLACEHOLDER_VALUE; + } + } + return histogram; + } +}; + +export function CorrelationsChart({ + field, + value, + histogram: originalHistogram, + markerValue, + markerPercentile, + overallHistogram, +}: CorrelationsChartProps) { + const euiTheme = useTheme(); + + if (!Array.isArray(overallHistogram)) return
; + const annotationsDataValues: LineAnnotationDatum[] = [ + { dataValue: markerValue, details: `${markerPercentile}th percentile` }, + ]; + + const xMax = Math.max(...overallHistogram.map((d) => d.key)) ?? 0; + + const durationFormatter = getDurationFormatter(xMax); + + const histogram = replaceHistogramDotsWithBars(originalHistogram); + + return ( +
+ + + + durationFormatter(d).formatted} + /> + + d === 0 || Number.isInteger(Math.log10(d)) ? d : '' + } + /> + + {Array.isArray(histogram) && + field !== undefined && + value !== undefined && ( + + )} + + +
+ ); +} diff --git a/x-pack/plugins/apm/public/components/app/correlations/correlations_table.tsx b/x-pack/plugins/apm/public/components/app/correlations/correlations_table.tsx index 75f3cca05c5c5..62d566963699d 100644 --- a/x-pack/plugins/apm/public/components/app/correlations/correlations_table.tsx +++ b/x-pack/plugins/apm/public/components/app/correlations/correlations_table.tsx @@ -5,7 +5,7 @@ * 2.0. */ -import React, { useCallback } from 'react'; +import React, { useCallback, useMemo, useState } from 'react'; import { debounce } from 'lodash'; import { EuiIcon, @@ -22,12 +22,14 @@ import { FETCH_STATUS } from '../../../hooks/use_fetcher'; import { createHref, push } from '../../shared/Links/url_helpers'; import { ImpactBar } from '../../shared/ImpactBar'; import { useUiTracker } from '../../../../../observability/public'; +import { useTheme } from '../../../hooks/use_theme'; +const PAGINATION_SIZE_OPTIONS = [5, 10, 20, 50]; type CorrelationsApiResponse = | APIReturnType<'GET /api/apm/correlations/errors/failed_transactions'> | APIReturnType<'GET /api/apm/correlations/latency/slow_transactions'>; -type SignificantTerm = CorrelationsApiResponse['significantTerms'][0]; +export type SignificantTerm = CorrelationsApiResponse['significantTerms'][0]; export type SelectedSignificantTerm = Pick< SignificantTerm, @@ -37,9 +39,11 @@ export type SelectedSignificantTerm = Pick< interface Props { significantTerms?: T[]; status: FETCH_STATUS; - percentageColumnName: string; + percentageColumnName?: string; setSelectedSignificantTerm: (term: SelectedSignificantTerm | null) => void; + selectedTerm?: { fieldName: string; fieldValue: string }; onFilter: () => void; + columns?: Array>; } export function CorrelationsTable({ @@ -48,7 +52,10 @@ export function CorrelationsTable({ percentageColumnName, setSelectedSignificantTerm, onFilter, + columns, + selectedTerm, }: Props) { + const euiTheme = useTheme(); const trackApmEvent = useUiTracker({ app: 'apm' }); const trackSelectSignificantTerm = useCallback( () => @@ -59,7 +66,33 @@ export function CorrelationsTable({ [trackApmEvent] ); const history = useHistory(); - const columns: Array> = [ + + const [pageIndex, setPageIndex] = useState(0); + const [pageSize, setPageSize] = useState(10); + + const { pagination, pageOfItems } = useMemo(() => { + const pageStart = pageIndex * pageSize; + + const itemCount = significantTerms?.length ?? 0; + return { + pageOfItems: significantTerms?.slice(pageStart, pageStart + pageSize), + pagination: { + pageIndex, + pageSize, + totalItemCount: itemCount, + pageSizeOptions: PAGINATION_SIZE_OPTIONS, + }, + }; + }, [pageIndex, pageSize, significantTerms]); + + const onTableChange = useCallback(({ page }) => { + const { index, size } = page; + + setPageIndex(index); + setPageSize(size); + }, []); + + const tableColumns: Array> = columns ?? [ { width: '116px', field: 'impact', @@ -73,7 +106,12 @@ export function CorrelationsTable({ }, { field: 'percentage', - name: percentageColumnName, + name: + percentageColumnName ?? + i18n.translate( + 'xpack.apm.correlations.correlationsTable.percentageLabel', + { defaultMessage: 'Percentage' } + ), render: (_: any, term: T) => { return ( ({ return ( { return { onMouseEnter: () => { @@ -203,8 +241,18 @@ export function CorrelationsTable({ trackSelectSignificantTerm(); }, onMouseLeave: () => setSelectedSignificantTerm(null), + style: + selectedTerm && + selectedTerm.fieldValue === term.fieldValue && + selectedTerm.fieldName === term.fieldName + ? { + backgroundColor: euiTheme.eui.euiColorLightestShade, + } + : null, }; }} + pagination={pagination} + onChange={onTableChange} /> ); } diff --git a/x-pack/plugins/apm/public/components/app/correlations/index.tsx b/x-pack/plugins/apm/public/components/app/correlations/index.tsx index 62c547aa69e0d..7b6328916d445 100644 --- a/x-pack/plugins/apm/public/components/app/correlations/index.tsx +++ b/x-pack/plugins/apm/public/components/app/correlations/index.tsx @@ -5,7 +5,7 @@ * 2.0. */ -import React, { useState } from 'react'; +import React, { useMemo, useState } from 'react'; import { EuiButtonEmpty, EuiFlyout, @@ -23,8 +23,8 @@ import { EuiBetaBadge, } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; -import { useHistory } from 'react-router-dom'; -import { LatencyCorrelations } from './latency_correlations'; +import { useHistory, useParams } from 'react-router-dom'; +import { MlLatencyCorrelations } from './ml_latency_correlations'; import { ErrorCorrelations } from './error_correlations'; import { useUrlParams } from '../../../context/url_params_context/use_url_params'; import { createHref } from '../../shared/Links/url_helpers'; @@ -36,14 +36,20 @@ import { isActivePlatinumLicense } from '../../../../common/license_check'; import { useLicenseContext } from '../../../context/license/use_license_context'; import { LicensePrompt } from '../../shared/license_prompt'; import { IUrlParams } from '../../../context/url_params_context/types'; +import { + IStickyProperty, + StickyProperties, +} from '../../shared/StickyProperties'; +import { + getEnvironmentLabel, + getNextEnvironmentUrlParam, +} from '../../../../common/environment_filter_values'; +import { + SERVICE_ENVIRONMENT, + SERVICE_NAME, + TRANSACTION_NAME, +} from '../../../../common/elasticsearch_fieldnames'; -const latencyTab = { - key: 'latency', - label: i18n.translate('xpack.apm.correlations.tabs.latencyLabel', { - defaultMessage: 'Latency', - }), - component: LatencyCorrelations, -}; const errorRateTab = { key: 'errorRate', label: i18n.translate('xpack.apm.correlations.tabs.errorRateLabel', { @@ -51,17 +57,26 @@ const errorRateTab = { }), component: ErrorCorrelations, }; -const tabs = [latencyTab, errorRateTab]; +const latencyCorrelationsTab = { + key: 'latencyCorrelations', + label: i18n.translate('xpack.apm.correlations.tabs.latencyLabel', { + defaultMessage: 'Latency', + }), + component: MlLatencyCorrelations, +}; +const tabs = [latencyCorrelationsTab, errorRateTab]; export function Correlations() { const license = useLicenseContext(); const hasActivePlatinumLicense = isActivePlatinumLicense(license); const { urlParams } = useUrlParams(); + const { serviceName } = useParams<{ serviceName: string }>(); + const history = useHistory(); const [isFlyoutVisible, setIsFlyoutVisible] = useState(false); - const [currentTab, setCurrentTab] = useState(latencyTab.key); + const [currentTab, setCurrentTab] = useState(latencyCorrelationsTab.key); const { component: TabContent } = - tabs.find((tab) => tab.key === currentTab) ?? latencyTab; + tabs.find((tab) => tab.key === currentTab) ?? latencyCorrelationsTab; const metric = { app: 'apm' as const, metric: hasActivePlatinumLicense @@ -72,6 +87,47 @@ export function Correlations() { useTrackMetric(metric); useTrackMetric({ ...metric, delay: 15000 }); + const stickyProperties: IStickyProperty[] = useMemo(() => { + const nextEnvironment = getNextEnvironmentUrlParam({ + requestedEnvironment: serviceName, + currentEnvironmentUrlParam: urlParams.environment, + }); + + const properties: IStickyProperty[] = []; + if (serviceName !== undefined && nextEnvironment !== undefined) { + properties.push({ + label: i18n.translate('xpack.apm.correlations.serviceLabel', { + defaultMessage: 'Service', + }), + fieldName: SERVICE_NAME, + val: serviceName, + width: '20%', + }); + } + if (urlParams.transactionName) { + properties.push({ + label: i18n.translate('xpack.apm.correlations.transactionLabel', { + defaultMessage: 'Transaction', + }), + fieldName: TRANSACTION_NAME, + val: urlParams.transactionName, + width: '20%', + }); + } + if (urlParams.environment) { + properties.push({ + label: i18n.translate('xpack.apm.correlations.environmentLabel', { + defaultMessage: 'Environment', + }), + fieldName: SERVICE_ENVIRONMENT, + val: getEnvironmentLabel(urlParams.environment), + width: '20%', + }); + } + + return properties; + }, [serviceName, urlParams.environment, urlParams.transactionName]); + return ( <> setIsFlyoutVisible(false)} > @@ -112,25 +168,37 @@ export function Correlations() { {hasActivePlatinumLicense && ( - - {tabs.map(({ key, label }) => ( - { - setCurrentTab(key); - }} - > - {label} - - ))} - + <> + + + + {urlParams.kuery ? ( + <> + + + + ) : ( + + )} + + {tabs.map(({ key, label }) => ( + { + setCurrentTab(key); + }} + > + {label} + + ))} + + )} {hasActivePlatinumLicense ? ( <> - setIsFlyoutVisible(false)} /> ) : ( @@ -163,24 +231,21 @@ function Filters({ } return ( - <> - - - {i18n.translate('xpack.apm.correlations.filteringByLabel', { - defaultMessage: 'Filtering by', + + + {i18n.translate('xpack.apm.correlations.filteringByLabel', { + defaultMessage: 'Filtering by', + })} + + {urlParams.kuery} + + + {i18n.translate('xpack.apm.correlations.clearFiltersLabel', { + defaultMessage: 'Clear', })} - - {urlParams.kuery} - - - {i18n.translate('xpack.apm.correlations.clearFiltersLabel', { - defaultMessage: 'Clear', - })} - - - - - + + + ); } diff --git a/x-pack/plugins/apm/public/components/app/correlations/ml_latency_correlations.tsx b/x-pack/plugins/apm/public/components/app/correlations/ml_latency_correlations.tsx new file mode 100644 index 0000000000000..f9536353747ee --- /dev/null +++ b/x-pack/plugins/apm/public/components/app/correlations/ml_latency_correlations.tsx @@ -0,0 +1,382 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { useEffect, useMemo, useState } from 'react'; +import { useHistory, useParams } from 'react-router-dom'; +import { + EuiIcon, + EuiBasicTableColumn, + EuiButton, + EuiFlexGroup, + EuiFlexItem, + EuiProgress, + EuiSpacer, + EuiText, + EuiTitle, + EuiToolTip, +} from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import { FormattedMessage } from '@kbn/i18n/react'; +import { useUrlParams } from '../../../context/url_params_context/use_url_params'; +import { useApmPluginContext } from '../../../context/apm_plugin/use_apm_plugin_context'; +import { FETCH_STATUS } from '../../../hooks/use_fetcher'; +import { + CorrelationsChart, + replaceHistogramDotsWithBars, +} from './correlations_chart'; +import { + CorrelationsTable, + SelectedSignificantTerm, +} from './correlations_table'; +import { useCorrelations } from './use_correlations'; +import { push } from '../../shared/Links/url_helpers'; +import { useUiTracker } from '../../../../../observability/public'; +import { asPreciseDecimal } from '../../../../common/utils/formatters'; + +const DEFAULT_PERCENTILE_THRESHOLD = 95; +const isErrorMessage = (arg: unknown): arg is Error => { + return arg instanceof Error; +}; + +interface Props { + onClose: () => void; +} + +interface MlCorrelationsTerms { + correlation: number; + ksTest: number; + fieldName: string; + fieldValue: string; + duplicatedFields?: string[]; +} + +export function MlLatencyCorrelations({ onClose }: Props) { + const { + core: { notifications }, + } = useApmPluginContext(); + + const { serviceName } = useParams<{ serviceName: string }>(); + const { urlParams } = useUrlParams(); + + const fetchOptions = useMemo( + () => ({ + ...{ + serviceName, + ...urlParams, + }, + }), + [serviceName, urlParams] + ); + + const { + error, + histograms, + percentileThresholdValue, + isRunning, + progress, + startFetch, + cancelFetch, + overallHistogram: originalOverallHistogram, + } = useCorrelations({ + index: 'apm-*', + ...{ + ...fetchOptions, + percentileThreshold: DEFAULT_PERCENTILE_THRESHOLD, + }, + }); + + const overallHistogram = useMemo( + () => replaceHistogramDotsWithBars(originalOverallHistogram), + [originalOverallHistogram] + ); + + // start fetching on load + // we want this effect to execute exactly once after the component mounts + useEffect(() => { + startFetch(); + + return () => { + // cancel any running async partial request when unmounting the component + // we want this effect to execute exactly once after the component mounts + cancelFetch(); + }; + // eslint-disable-next-line react-hooks/exhaustive-deps + }, []); + + useEffect(() => { + if (isErrorMessage(error)) { + notifications.toasts.addDanger({ + title: i18n.translate( + 'xpack.apm.correlations.latencyCorrelations.errorTitle', + { + defaultMessage: 'An error occurred fetching correlations', + } + ), + text: error.toString(), + }); + } + }, [error, notifications.toasts]); + + const [ + selectedSignificantTerm, + setSelectedSignificantTerm, + ] = useState(null); + + let selectedHistogram = histograms.length > 0 ? histograms[0] : undefined; + + if (histograms.length > 0 && selectedSignificantTerm !== null) { + selectedHistogram = histograms.find( + (h) => + h.field === selectedSignificantTerm.fieldName && + h.value === selectedSignificantTerm.fieldValue + ); + } + const history = useHistory(); + const trackApmEvent = useUiTracker({ app: 'apm' }); + + const mlCorrelationColumns: Array< + EuiBasicTableColumn + > = useMemo( + () => [ + { + width: '116px', + field: 'correlation', + name: ( + + <> + {i18n.translate( + 'xpack.apm.correlations.latencyCorrelations.correlationsTable.correlationLabel', + { + defaultMessage: 'Correlation', + } + )} + + + + ), + render: (correlation: number) => { + return
{asPreciseDecimal(correlation, 2)}
; + }, + }, + { + field: 'fieldName', + name: i18n.translate( + 'xpack.apm.correlations.latencyCorrelations.correlationsTable.fieldNameLabel', + { defaultMessage: 'Field name' } + ), + }, + { + field: 'fieldValue', + name: i18n.translate( + 'xpack.apm.correlations.latencyCorrelations.correlationsTable.fieldValueLabel', + { defaultMessage: 'Field value' } + ), + render: (fieldValue: string) => String(fieldValue).slice(0, 50), + }, + { + width: '100px', + actions: [ + { + name: i18n.translate( + 'xpack.apm.correlations.latencyCorrelations.correlationsTable.filterLabel', + { defaultMessage: 'Filter' } + ), + description: i18n.translate( + 'xpack.apm.correlations.latencyCorrelations.correlationsTable.filterDescription', + { defaultMessage: 'Filter by value' } + ), + icon: 'plusInCircle', + type: 'icon', + onClick: (term: MlCorrelationsTerms) => { + push(history, { + query: { + kuery: `${term.fieldName}:"${encodeURIComponent( + term.fieldValue + )}"`, + }, + }); + onClose(); + trackApmEvent({ metric: 'correlations_term_include_filter' }); + }, + }, + { + name: i18n.translate( + 'xpack.apm.correlations.latencyCorrelations.correlationsTable.excludeLabel', + { defaultMessage: 'Exclude' } + ), + description: i18n.translate( + 'xpack.apm.correlations.latencyCorrelations.correlationsTable.excludeDescription', + { defaultMessage: 'Filter out value' } + ), + icon: 'minusInCircle', + type: 'icon', + onClick: (term: MlCorrelationsTerms) => { + push(history, { + query: { + kuery: `not ${term.fieldName}:"${encodeURIComponent( + term.fieldValue + )}"`, + }, + }); + onClose(); + trackApmEvent({ metric: 'correlations_term_exclude_filter' }); + }, + }, + ], + name: i18n.translate( + 'xpack.apm.correlations.latencyCorrelations.correlationsTable.actionsLabel', + { defaultMessage: 'Filter' } + ), + }, + ], + [history, onClose, trackApmEvent] + ); + + const histogramTerms: MlCorrelationsTerms[] = useMemo(() => { + return histograms.map((d) => { + return { + fieldName: d.field, + fieldValue: d.value, + ksTest: d.ksTest, + correlation: d.correlation, + duplicatedFields: d.duplicatedFields, + }; + }); + }, [histograms]); + + return ( + <> + +

+ {i18n.translate( + 'xpack.apm.correlations.latencyCorrelations.description', + { + defaultMessage: + 'What is slowing down my service? Correlations will help discover a slower performance in a particular cohort of your data.', + } + )} +

+ + + + + + + {!isRunning && ( + + + + )} + {isRunning && ( + + + + )} + + + + + + + + + + + + + + + + + {overallHistogram !== undefined ? ( + <> + +

+ {i18n.translate( + 'xpack.apm.correlations.latencyCorrelations.chartTitle', + { + defaultMessage: 'Latency distribution for {name}', + values: { + name: + fetchOptions.transactionName ?? fetchOptions.serviceName, + }, + } + )} +

+ + + + + + + ) : null} + + {histograms.length > 0 && selectedHistogram !== undefined && ( + + )} + {histograms.length < 1 && progress > 0.99 ? ( + <> + + + + + + ) : null} + + ); +} diff --git a/x-pack/plugins/apm/public/components/app/correlations/use_correlations.ts b/x-pack/plugins/apm/public/components/app/correlations/use_correlations.ts new file mode 100644 index 0000000000000..8c874571d23db --- /dev/null +++ b/x-pack/plugins/apm/public/components/app/correlations/use_correlations.ts @@ -0,0 +1,120 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { useRef, useState } from 'react'; +import type { Subscription } from 'rxjs'; +import { + IKibanaSearchRequest, + IKibanaSearchResponse, + isCompleteResponse, + isErrorResponse, +} from '../../../../../../../src/plugins/data/public'; +import type { + HistogramItem, + SearchServiceValue, +} from '../../../../common/search_strategies/correlations/types'; +import { useKibana } from '../../../../../../../src/plugins/kibana_react/public'; +import { ApmPluginStartDeps } from '../../../plugin'; + +interface CorrelationsOptions { + index: string; + environment?: string; + kuery?: string; + serviceName?: string; + transactionName?: string; + transactionType?: string; + start?: string; + end?: string; +} + +interface RawResponse { + percentileThresholdValue?: number; + took: number; + values: SearchServiceValue[]; + overallHistogram: HistogramItem[]; +} + +export const useCorrelations = (params: CorrelationsOptions) => { + const { + services: { data }, + } = useKibana(); + + const [error, setError] = useState(); + const [isComplete, setIsComplete] = useState(false); + const [isRunning, setIsRunning] = useState(false); + const [loaded, setLoaded] = useState(0); + const [rawResponse, setRawResponse] = useState(); + const [timeTook, setTimeTook] = useState(); + const [total, setTotal] = useState(100); + const abortCtrl = useRef(new AbortController()); + const searchSubscription$ = useRef(); + + function setResponse(response: IKibanaSearchResponse) { + // @TODO: optimize rawResponse.overallHistogram if histogram is the same + setIsRunning(response.isRunning || false); + setRawResponse(response.rawResponse); + setLoaded(response.loaded!); + setTotal(response.total!); + setTimeTook(response.rawResponse.took); + } + + const startFetch = () => { + setError(undefined); + setIsComplete(false); + searchSubscription$.current?.unsubscribe(); + abortCtrl.current.abort(); + abortCtrl.current = new AbortController(); + + const req = { params }; + + // Submit the search request using the `data.search` service. + searchSubscription$.current = data.search + .search>(req, { + strategy: 'apmCorrelationsSearchStrategy', + abortSignal: abortCtrl.current.signal, + }) + .subscribe({ + next: (res: IKibanaSearchResponse) => { + setResponse(res); + if (isCompleteResponse(res)) { + searchSubscription$.current?.unsubscribe(); + setIsRunning(false); + setIsComplete(true); + } else if (isErrorResponse(res)) { + searchSubscription$.current?.unsubscribe(); + setError((res as unknown) as Error); + setIsRunning(false); + } + }, + error: (e: Error) => { + setError(e); + setIsRunning(false); + }, + }); + }; + + const cancelFetch = () => { + searchSubscription$.current?.unsubscribe(); + searchSubscription$.current = undefined; + abortCtrl.current.abort(); + setIsRunning(false); + }; + + return { + error, + histograms: rawResponse?.values ?? [], + percentileThresholdValue: + rawResponse?.percentileThresholdValue ?? undefined, + overallHistogram: rawResponse?.overallHistogram, + isComplete, + isRunning, + progress: loaded / total, + timeTook, + startFetch, + cancelFetch, + }; +}; diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/async_search_service.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/async_search_service.ts new file mode 100644 index 0000000000000..5820fd952c449 --- /dev/null +++ b/x-pack/plugins/apm/server/lib/search_strategies/correlations/async_search_service.ts @@ -0,0 +1,225 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { shuffle, range } from 'lodash'; +import type { ElasticsearchClient } from 'src/core/server'; +import { fetchTransactionDurationFieldCandidates } from './query_field_candidates'; +import { fetchTransactionDurationFieldValuePairs } from './query_field_value_pairs'; +import { fetchTransactionDurationPecentiles } from './query_percentiles'; +import { fetchTransactionDurationCorrelation } from './query_correlation'; +import { fetchTransactionDurationHistogramRangesteps } from './query_histogram_rangesteps'; +import { fetchTransactionDurationRanges, HistogramItem } from './query_ranges'; +import type { + AsyncSearchProviderProgress, + SearchServiceParams, + SearchServiceValue, +} from '../../../../common/search_strategies/correlations/types'; +import { computeExpectationsAndRanges } from './utils/aggregation_utils'; +import { fetchTransactionDurationFractions } from './query_fractions'; + +const CORRELATION_THRESHOLD = 0.3; +const KS_TEST_THRESHOLD = 0.1; + +export const asyncSearchServiceProvider = ( + esClient: ElasticsearchClient, + params: SearchServiceParams +) => { + let isCancelled = false; + let isRunning = true; + let error: Error; + + const progress: AsyncSearchProviderProgress = { + started: Date.now(), + loadedHistogramStepsize: 0, + loadedOverallHistogram: 0, + loadedFieldCanditates: 0, + loadedFieldValuePairs: 0, + loadedHistograms: 0, + getOverallProgress: () => + progress.loadedHistogramStepsize * 0.025 + + progress.loadedOverallHistogram * 0.025 + + progress.loadedFieldCanditates * 0.025 + + progress.loadedFieldValuePairs * 0.025 + + progress.loadedHistograms * 0.9, + }; + + const values: SearchServiceValue[] = []; + let overallHistogram: HistogramItem[] | undefined; + + let percentileThresholdValue: number; + + const cancel = () => { + isCancelled = true; + }; + + const fetchCorrelations = async () => { + try { + // 95th percentile to be displayed as a marker in the log log chart + const percentileThreshold = await fetchTransactionDurationPecentiles( + esClient, + params, + params.percentileThreshold ? [params.percentileThreshold] : undefined + ); + percentileThresholdValue = + percentileThreshold[`${params.percentileThreshold}.0`]; + + const histogramRangeSteps = await fetchTransactionDurationHistogramRangesteps( + esClient, + params + ); + progress.loadedHistogramStepsize = 1; + + if (isCancelled) { + isRunning = false; + return; + } + + const overallLogHistogramChartData = await fetchTransactionDurationRanges( + esClient, + params, + histogramRangeSteps + ); + progress.loadedOverallHistogram = 1; + overallHistogram = overallLogHistogramChartData; + + if (isCancelled) { + isRunning = false; + return; + } + + // Create an array of ranges [2, 4, 6, ..., 98] + const percents = Array.from(range(2, 100, 2)); + const percentilesRecords = await fetchTransactionDurationPecentiles( + esClient, + params, + percents + ); + const percentiles = Object.values(percentilesRecords); + + if (isCancelled) { + isRunning = false; + return; + } + + const { fieldCandidates } = await fetchTransactionDurationFieldCandidates( + esClient, + params + ); + + progress.loadedFieldCanditates = 1; + + const fieldValuePairs = await fetchTransactionDurationFieldValuePairs( + esClient, + params, + fieldCandidates, + progress + ); + + if (isCancelled) { + isRunning = false; + return; + } + + const { expectations, ranges } = computeExpectationsAndRanges( + percentiles + ); + + const { + fractions, + totalDocCount, + } = await fetchTransactionDurationFractions(esClient, params, ranges); + + async function* fetchTransactionDurationHistograms() { + for (const item of shuffle(fieldValuePairs)) { + if (item === undefined || isCancelled) { + isRunning = false; + return; + } + + // If one of the fields have an error + // We don't want to stop the whole process + try { + const { + correlation, + ksTest, + } = await fetchTransactionDurationCorrelation( + esClient, + params, + expectations, + ranges, + fractions, + totalDocCount, + item.field, + item.value + ); + + if (isCancelled) { + isRunning = false; + return; + } + + if ( + correlation !== null && + correlation > CORRELATION_THRESHOLD && + ksTest !== null && + ksTest < KS_TEST_THRESHOLD + ) { + const logHistogram = await fetchTransactionDurationRanges( + esClient, + params, + histogramRangeSteps, + item.field, + item.value + ); + yield { + ...item, + correlation, + ksTest, + histogram: logHistogram, + }; + } else { + yield undefined; + } + } catch (e) { + error = e; + } + } + } + + let loadedHistograms = 0; + for await (const item of fetchTransactionDurationHistograms()) { + if (item !== undefined) { + values.push(item); + } + loadedHistograms++; + progress.loadedHistograms = loadedHistograms / fieldValuePairs.length; + } + + isRunning = false; + } catch (e) { + error = e; + } + }; + + fetchCorrelations(); + + return () => { + const sortedValues = values.sort((a, b) => b.correlation - a.correlation); + + return { + error, + isRunning, + loaded: Math.round(progress.getOverallProgress() * 100), + overallHistogram, + started: progress.started, + total: 100, + values: sortedValues, + percentileThresholdValue, + cancel, + }; + }; +}; diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/constants.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/constants.ts new file mode 100644 index 0000000000000..5420479bfffb7 --- /dev/null +++ b/x-pack/plugins/apm/server/lib/search_strategies/correlations/constants.ts @@ -0,0 +1,78 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/** + * Fields to exclude as potential field candidates + */ +export const FIELDS_TO_EXCLUDE_AS_CANDIDATE = new Set([ + // Exclude for all usage Contexts + 'parent.id', + 'trace.id', + 'transaction.id', + '@timestamp', + 'timestamp.us', + 'agent.ephemeral_id', + 'ecs.version', + 'event.ingested', + 'http.response.finished', + 'parent.id', + 'trace.id', + 'transaction.duration.us', + 'transaction.id', + 'process.pid', + 'process.ppid', + 'processor.event', + 'processor.name', + 'transaction.sampled', + 'transaction.span_count.dropped', + // Exclude for correlation on a Single Service + 'agent.name', + 'http.request.method', + 'service.framework.name', + 'service.language.name', + 'service.name', + 'service.runtime.name', + 'transaction.name', + 'transaction.type', +]); + +export const FIELD_PREFIX_TO_EXCLUDE_AS_CANDIDATE = ['observer.']; + +/** + * Fields to include/prioritize as potential field candidates + */ +export const FIELDS_TO_ADD_AS_CANDIDATE = new Set([ + 'service.version', + 'service.node.name', + 'service.framework.version', + 'service.language.version', + 'service.runtime.version', + 'kubernetes.pod.name', + 'kubernetes.pod.uid', + 'container.id', + 'source.ip', + 'client.ip', + 'host.ip', + 'service.environment', + 'process.args', + 'http.response.status_code', +]); +export const FIELD_PREFIX_TO_ADD_AS_CANDIDATE = [ + 'cloud.', + 'labels.', + 'user_agent.', +]; + +/** + * Other constants + */ +export const POPULATED_DOC_COUNT_SAMPLE_SIZE = 1000; + +export const PERCENTILES_STEP = 2; +export const TERMS_SIZE = 20; +export const SIGNIFICANT_FRACTION = 3; +export const SIGNIFICANT_VALUE_DIGITS = 3; diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/get_query_with_params.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/get_query_with_params.ts new file mode 100644 index 0000000000000..e7cf8173b5bac --- /dev/null +++ b/x-pack/plugins/apm/server/lib/search_strategies/correlations/get_query_with_params.ts @@ -0,0 +1,89 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { estypes } from '@elastic/elasticsearch'; +import { + PROCESSOR_EVENT, + SERVICE_NAME, + TRANSACTION_DURATION, + TRANSACTION_NAME, +} from '../../../../common/elasticsearch_fieldnames'; +import type { SearchServiceParams } from '../../../../common/search_strategies/correlations/types'; +import { environmentQuery as getEnvironmentQuery } from '../../../utils/queries'; +import { ProcessorEvent } from '../../../../common/processor_event'; + +const getPercentileThresholdValueQuery = ( + percentileThresholdValue: number | undefined +): estypes.QueryDslQueryContainer[] => { + return percentileThresholdValue + ? [ + { + range: { + [TRANSACTION_DURATION]: { + gte: percentileThresholdValue, + }, + }, + }, + ] + : []; +}; + +export const getTermsQuery = ( + fieldName: string | undefined, + fieldValue: string | undefined +) => { + return fieldName && fieldValue ? [{ term: { [fieldName]: fieldValue } }] : []; +}; + +const getRangeQuery = ( + start?: string, + end?: string +): estypes.QueryDslQueryContainer[] => { + return [ + { + range: { + '@timestamp': { + ...(start !== undefined ? { gte: start } : {}), + ...(end !== undefined ? { lte: end } : {}), + }, + }, + }, + ]; +}; + +interface QueryParams { + params: SearchServiceParams; + fieldName?: string; + fieldValue?: string; +} +export const getQueryWithParams = ({ + params, + fieldName, + fieldValue, +}: QueryParams) => { + const { + environment, + serviceName, + start, + end, + percentileThresholdValue, + transactionName, + } = params; + return { + bool: { + filter: [ + ...getTermsQuery(PROCESSOR_EVENT, ProcessorEvent.transaction), + ...getTermsQuery(SERVICE_NAME, serviceName), + ...getTermsQuery(TRANSACTION_NAME, transactionName), + ...getTermsQuery(fieldName, fieldValue), + ...getRangeQuery(start, end), + ...getEnvironmentQuery(environment), + ...getPercentileThresholdValueQuery(percentileThresholdValue), + ] as estypes.QueryDslQueryContainer[], + }, + }; +}; diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/index.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/index.ts new file mode 100644 index 0000000000000..5ba7b4d7c957a --- /dev/null +++ b/x-pack/plugins/apm/server/lib/search_strategies/correlations/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { apmCorrelationsSearchStrategyProvider } from './search_strategy'; diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_correlation.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_correlation.ts new file mode 100644 index 0000000000000..9894ac54eccb6 --- /dev/null +++ b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_correlation.ts @@ -0,0 +1,139 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { estypes } from '@elastic/elasticsearch'; + +import type { ElasticsearchClient } from 'src/core/server'; + +import { TRANSACTION_DURATION } from '../../../../common/elasticsearch_fieldnames'; +import type { SearchServiceParams } from '../../../../common/search_strategies/correlations/types'; + +import { getQueryWithParams } from './get_query_with_params'; + +export interface HistogramItem { + key: number; + doc_count: number; +} + +interface ResponseHitSource { + [s: string]: unknown; +} +interface ResponseHit { + _source: ResponseHitSource; +} + +interface BucketCorrelation { + buckets_path: string; + function: { + count_correlation: { + indicator: { + fractions: number[]; + expectations: number[]; + doc_count: number; + }; + }; + }; +} + +export const getTransactionDurationCorrelationRequest = ( + params: SearchServiceParams, + expectations: number[], + ranges: estypes.AggregationsAggregationRange[], + fractions: number[], + totalDocCount: number, + fieldName?: string, + fieldValue?: string +): estypes.SearchRequest => { + const query = getQueryWithParams({ params, fieldName, fieldValue }); + + const bucketCorrelation: BucketCorrelation = { + buckets_path: 'latency_ranges>_count', + function: { + count_correlation: { + indicator: { + fractions, + expectations, + doc_count: totalDocCount, + }, + }, + }, + }; + + const body = { + query, + size: 0, + aggs: { + latency_ranges: { + range: { + field: TRANSACTION_DURATION, + ranges, + }, + }, + // Pearson correlation value + transaction_duration_correlation: { + bucket_correlation: bucketCorrelation, + } as estypes.AggregationsAggregationContainer, + // KS test p value = ks_test.less + ks_test: { + bucket_count_ks_test: { + // Remove 0 after https://github.com/elastic/elasticsearch/pull/74624 is merged + fractions: [0, ...fractions], + buckets_path: 'latency_ranges>_count', + alternative: ['less', 'greater', 'two_sided'], + }, + } as estypes.AggregationsAggregationContainer, + }, + }; + return { + index: params.index, + body, + }; +}; + +export const fetchTransactionDurationCorrelation = async ( + esClient: ElasticsearchClient, + params: SearchServiceParams, + expectations: number[], + ranges: estypes.AggregationsAggregationRange[], + fractions: number[], + totalDocCount: number, + fieldName?: string, + fieldValue?: string +): Promise<{ + ranges: unknown[]; + correlation: number | null; + ksTest: number | null; +}> => { + const resp = await esClient.search( + getTransactionDurationCorrelationRequest( + params, + expectations, + ranges, + fractions, + totalDocCount, + fieldName, + fieldValue + ) + ); + + if (resp.body.aggregations === undefined) { + throw new Error( + 'fetchTransactionDurationCorrelation failed, did not return aggregations.' + ); + } + + const result = { + ranges: (resp.body.aggregations + .latency_ranges as estypes.AggregationsMultiBucketAggregate).buckets, + correlation: (resp.body.aggregations + .transaction_duration_correlation as estypes.AggregationsValueAggregate) + .value, + // @ts-ignore + ksTest: resp.body.aggregations.ks_test.less, + }; + return result; +}; diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_field_candidates.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_field_candidates.ts new file mode 100644 index 0000000000000..4f1840971da7d --- /dev/null +++ b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_field_candidates.ts @@ -0,0 +1,105 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { estypes } from '@elastic/elasticsearch'; + +import type { ElasticsearchClient } from 'src/core/server'; + +import type { SearchServiceParams } from '../../../../common/search_strategies/correlations/types'; + +import { getQueryWithParams } from './get_query_with_params'; +import { Field } from './query_field_value_pairs'; +import { + FIELD_PREFIX_TO_ADD_AS_CANDIDATE, + FIELD_PREFIX_TO_EXCLUDE_AS_CANDIDATE, + FIELDS_TO_ADD_AS_CANDIDATE, + FIELDS_TO_EXCLUDE_AS_CANDIDATE, + POPULATED_DOC_COUNT_SAMPLE_SIZE, +} from './constants'; + +const shouldBeExcluded = (fieldName: string) => { + return ( + FIELDS_TO_EXCLUDE_AS_CANDIDATE.has(fieldName) || + FIELD_PREFIX_TO_EXCLUDE_AS_CANDIDATE.some((prefix) => + fieldName.startsWith(prefix) + ) + ); +}; + +const hasPrefixToInclude = (fieldName: string) => { + return FIELD_PREFIX_TO_ADD_AS_CANDIDATE.some((prefix) => + fieldName.startsWith(prefix) + ); +}; + +export const getRandomDocsRequest = ( + params: SearchServiceParams +): estypes.SearchRequest => ({ + index: params.index, + body: { + fields: ['*'], + _source: false, + query: { + function_score: { + query: getQueryWithParams({ params }), + // @ts-ignore + random_score: {}, + }, + }, + // Required value for later correlation queries + track_total_hits: true, + size: POPULATED_DOC_COUNT_SAMPLE_SIZE, + }, +}); + +export const fetchTransactionDurationFieldCandidates = async ( + esClient: ElasticsearchClient, + params: SearchServiceParams +): Promise<{ fieldCandidates: Field[]; totalHits: number }> => { + const { index } = params; + // Get all fields with keyword mapping + const respMapping = await esClient.fieldCaps({ + index, + fields: '*', + }); + + const finalFieldCandidates = new Set(FIELDS_TO_ADD_AS_CANDIDATE); + const acceptableFields: Set = new Set(); + + Object.entries(respMapping.body.fields).forEach(([key, value]) => { + const fieldTypes = Object.keys(value); + const isSupportedType = fieldTypes.some( + (type) => type === 'keyword' || type === 'ip' + ); + // Definitely include if field name matches any of the wild card + if (hasPrefixToInclude(key) && isSupportedType) { + finalFieldCandidates.add(key); + } + + // Check if fieldName is something we can aggregate on + if (isSupportedType) { + acceptableFields.add(key); + } + }); + + const resp = await esClient.search(getRandomDocsRequest(params)); + const sampledDocs = resp.body.hits.hits.map((d) => d.fields ?? {}); + + // Get all field names for each returned doc and flatten it + // to a list of unique field names used across all docs + // and filter by list of acceptable fields and some APM specific unique fields. + [...new Set(sampledDocs.map(Object.keys).flat(1))].forEach((field) => { + if (acceptableFields.has(field) && !shouldBeExcluded(field)) { + finalFieldCandidates.add(field); + } + }); + + return { + fieldCandidates: [...finalFieldCandidates], + totalHits: (resp.body.hits.total as estypes.SearchTotalHits).value, + }; +}; diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_field_value_pairs.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_field_value_pairs.ts new file mode 100644 index 0000000000000..703a203c89207 --- /dev/null +++ b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_field_value_pairs.ts @@ -0,0 +1,89 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ElasticsearchClient } from 'src/core/server'; + +import type { estypes } from '@elastic/elasticsearch'; + +import type { + AsyncSearchProviderProgress, + SearchServiceParams, +} from '../../../../common/search_strategies/correlations/types'; + +import { getQueryWithParams } from './get_query_with_params'; +import { TERMS_SIZE } from './constants'; + +interface FieldValuePair { + field: string; + value: string; +} +type FieldValuePairs = FieldValuePair[]; + +export type Field = string; + +export const getTermsAggRequest = ( + params: SearchServiceParams, + fieldName: string +): estypes.SearchRequest => ({ + index: params.index, + body: { + query: getQueryWithParams({ params }), + size: 0, + aggs: { + attribute_terms: { + terms: { + field: fieldName, + size: TERMS_SIZE, + }, + }, + }, + }, +}); + +export const fetchTransactionDurationFieldValuePairs = async ( + esClient: ElasticsearchClient, + params: SearchServiceParams, + fieldCandidates: Field[], + progress: AsyncSearchProviderProgress +): Promise => { + const fieldValuePairs: FieldValuePairs = []; + + let fieldValuePairsProgress = 0; + + for (let i = 0; i < fieldCandidates.length; i++) { + const fieldName = fieldCandidates[i]; + // mutate progress + progress.loadedFieldValuePairs = + fieldValuePairsProgress / fieldCandidates.length; + + try { + const resp = await esClient.search(getTermsAggRequest(params, fieldName)); + + if (resp.body.aggregations === undefined) { + fieldValuePairsProgress++; + continue; + } + const buckets = (resp.body.aggregations + .attribute_terms as estypes.AggregationsMultiBucketAggregate<{ + key: string; + }>)?.buckets; + if (buckets.length >= 1) { + fieldValuePairs.push( + ...buckets.map((d) => ({ + field: fieldName, + value: d.key, + })) + ); + } + + fieldValuePairsProgress++; + } catch (e) { + fieldValuePairsProgress++; + } + } + return fieldValuePairs; +}; diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_fractions.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_fractions.ts new file mode 100644 index 0000000000000..3d623a4df8c34 --- /dev/null +++ b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_fractions.ts @@ -0,0 +1,64 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient } from 'kibana/server'; +import { estypes } from '@elastic/elasticsearch'; +import { SearchServiceParams } from '../../../../common/search_strategies/correlations/types'; +import { getQueryWithParams } from './get_query_with_params'; +import { TRANSACTION_DURATION } from '../../../../common/elasticsearch_fieldnames'; + +export const getTransactionDurationRangesRequest = ( + params: SearchServiceParams, + ranges: estypes.AggregationsAggregationRange[] +): estypes.SearchRequest => ({ + index: params.index, + body: { + query: getQueryWithParams({ params }), + size: 0, + aggs: { + latency_ranges: { + range: { + field: TRANSACTION_DURATION, + ranges, + }, + }, + }, + }, +}); + +/** + * Compute the actual percentile bucket counts and actual fractions + */ +export const fetchTransactionDurationFractions = async ( + esClient: ElasticsearchClient, + params: SearchServiceParams, + ranges: estypes.AggregationsAggregationRange[] +): Promise<{ fractions: number[]; totalDocCount: number }> => { + const resp = await esClient.search( + getTransactionDurationRangesRequest(params, ranges) + ); + if (resp.body.aggregations === undefined) { + throw new Error( + 'fetchTransactionDurationFractions failed, did not return aggregations.' + ); + } + + const buckets = (resp.body.aggregations + .latency_ranges as estypes.AggregationsMultiBucketAggregate<{ + doc_count: number; + }>)?.buckets; + + const totalDocCount = buckets.reduce((acc, bucket) => { + return acc + bucket.doc_count; + }, 0); + + // Compute (doc count per bucket/total doc count) + return { + fractions: buckets.map((bucket) => bucket.doc_count / totalDocCount), + totalDocCount, + }; +}; diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_histogram.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_histogram.ts new file mode 100644 index 0000000000000..6f61ecbfdcf08 --- /dev/null +++ b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_histogram.ts @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { estypes } from '@elastic/elasticsearch'; + +import type { ElasticsearchClient } from 'src/core/server'; + +import { TRANSACTION_DURATION } from '../../../../common/elasticsearch_fieldnames'; +import type { + HistogramItem, + ResponseHit, + SearchServiceParams, +} from '../../../../common/search_strategies/correlations/types'; + +import { getQueryWithParams } from './get_query_with_params'; + +export const getTransactionDurationHistogramRequest = ( + params: SearchServiceParams, + interval: number, + fieldName?: string, + fieldValue?: string +): estypes.SearchRequest => { + const query = getQueryWithParams({ params, fieldName, fieldValue }); + + return { + index: params.index, + body: { + query, + size: 0, + aggs: { + transaction_duration_histogram: { + histogram: { field: TRANSACTION_DURATION, interval }, + }, + }, + }, + }; +}; + +export const fetchTransactionDurationHistogram = async ( + esClient: ElasticsearchClient, + params: SearchServiceParams, + interval: number, + fieldName?: string, + fieldValue?: string +): Promise => { + const resp = await esClient.search( + getTransactionDurationHistogramRequest( + params, + interval, + fieldName, + fieldValue + ) + ); + + if (resp.body.aggregations === undefined) { + throw new Error( + 'fetchTransactionDurationHistogram failed, did not return aggregations.' + ); + } + + return ( + (resp.body.aggregations + .transaction_duration_histogram as estypes.AggregationsMultiBucketAggregate) + .buckets ?? [] + ); +}; diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_histogram_interval.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_histogram_interval.ts new file mode 100644 index 0000000000000..c4d1abf24b4d6 --- /dev/null +++ b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_histogram_interval.ts @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { estypes } from '@elastic/elasticsearch'; + +import type { ElasticsearchClient } from 'src/core/server'; + +import { TRANSACTION_DURATION } from '../../../../common/elasticsearch_fieldnames'; +import type { SearchServiceParams } from '../../../../common/search_strategies/correlations/types'; + +import { getQueryWithParams } from './get_query_with_params'; + +const HISTOGRAM_INTERVALS = 1000; + +export const getHistogramIntervalRequest = ( + params: SearchServiceParams +): estypes.SearchRequest => ({ + index: params.index, + body: { + query: getQueryWithParams({ params }), + size: 0, + aggs: { + transaction_duration_min: { min: { field: TRANSACTION_DURATION } }, + transaction_duration_max: { max: { field: TRANSACTION_DURATION } }, + }, + }, +}); + +export const fetchTransactionDurationHistogramInterval = async ( + esClient: ElasticsearchClient, + params: SearchServiceParams +): Promise => { + const resp = await esClient.search(getHistogramIntervalRequest(params)); + + if (resp.body.aggregations === undefined) { + throw new Error( + 'fetchTransactionDurationHistogramInterval failed, did not return aggregations.' + ); + } + + const transactionDurationDelta = + (resp.body.aggregations + .transaction_duration_max as estypes.AggregationsValueAggregate).value - + (resp.body.aggregations + .transaction_duration_min as estypes.AggregationsValueAggregate).value; + + return transactionDurationDelta / (HISTOGRAM_INTERVALS - 1); +}; diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_histogram_rangesteps.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_histogram_rangesteps.ts new file mode 100644 index 0000000000000..e537165ca53f3 --- /dev/null +++ b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_histogram_rangesteps.ts @@ -0,0 +1,65 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { scaleLog } from 'd3-scale'; + +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { estypes } from '@elastic/elasticsearch'; + +import type { ElasticsearchClient } from 'src/core/server'; + +import { TRANSACTION_DURATION } from '../../../../common/elasticsearch_fieldnames'; +import type { SearchServiceParams } from '../../../../common/search_strategies/correlations/types'; + +import { getQueryWithParams } from './get_query_with_params'; + +export const getHistogramIntervalRequest = ( + params: SearchServiceParams +): estypes.SearchRequest => ({ + index: params.index, + body: { + query: getQueryWithParams({ params }), + size: 0, + aggs: { + transaction_duration_min: { min: { field: TRANSACTION_DURATION } }, + transaction_duration_max: { max: { field: TRANSACTION_DURATION } }, + }, + }, +}); + +export const fetchTransactionDurationHistogramRangesteps = async ( + esClient: ElasticsearchClient, + params: SearchServiceParams +): Promise => { + const resp = await esClient.search(getHistogramIntervalRequest(params)); + + if (resp.body.aggregations === undefined) { + throw new Error( + 'fetchTransactionDurationHistogramInterval failed, did not return aggregations.' + ); + } + + const steps = 100; + const min = (resp.body.aggregations + .transaction_duration_min as estypes.AggregationsValueAggregate).value; + const max = + (resp.body.aggregations + .transaction_duration_max as estypes.AggregationsValueAggregate).value * + 2; + + // A d3 based scale function as a helper to get equally distributed bins on a log scale. + const logFn = scaleLog().domain([min, max]).range([1, steps]); + return [...Array(steps).keys()] + .map(logFn.invert) + .map((d) => (isNaN(d) ? 0 : d)); +}; diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_percentiles.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_percentiles.ts new file mode 100644 index 0000000000000..013c1ba3cbc23 --- /dev/null +++ b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_percentiles.ts @@ -0,0 +1,84 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { estypes } from '@elastic/elasticsearch'; + +import type { ElasticsearchClient } from 'src/core/server'; + +import { TRANSACTION_DURATION } from '../../../../common/elasticsearch_fieldnames'; +import type { SearchServiceParams } from '../../../../common/search_strategies/correlations/types'; + +import { getQueryWithParams } from './get_query_with_params'; +import { SIGNIFICANT_VALUE_DIGITS } from './constants'; + +export interface HistogramItem { + key: number; + doc_count: number; +} + +interface ResponseHitSource { + [s: string]: unknown; +} +interface ResponseHit { + _source: ResponseHitSource; +} + +export const getTransactionDurationPercentilesRequest = ( + params: SearchServiceParams, + percents?: number[], + fieldName?: string, + fieldValue?: string +): estypes.SearchRequest => { + const query = getQueryWithParams({ params, fieldName, fieldValue }); + + return { + index: params.index, + body: { + query, + size: 0, + aggs: { + transaction_duration_percentiles: { + percentiles: { + hdr: { + number_of_significant_value_digits: SIGNIFICANT_VALUE_DIGITS, + }, + field: TRANSACTION_DURATION, + ...(Array.isArray(percents) ? { percents } : {}), + }, + }, + }, + }, + }; +}; + +export const fetchTransactionDurationPecentiles = async ( + esClient: ElasticsearchClient, + params: SearchServiceParams, + percents?: number[], + fieldName?: string, + fieldValue?: string +): Promise> => { + const resp = await esClient.search( + getTransactionDurationPercentilesRequest( + params, + percents, + fieldName, + fieldValue + ) + ); + + if (resp.body.aggregations === undefined) { + throw new Error( + 'fetchTransactionDurationPecentiles failed, did not return aggregations.' + ); + } + return ( + (resp.body.aggregations + .transaction_duration_percentiles as estypes.AggregationsTDigestPercentilesAggregate) + .values ?? {} + ); +}; diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_ranges.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_ranges.ts new file mode 100644 index 0000000000000..88256f79150fc --- /dev/null +++ b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_ranges.ts @@ -0,0 +1,96 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { estypes } from '@elastic/elasticsearch'; + +import type { ElasticsearchClient } from 'src/core/server'; + +import { TRANSACTION_DURATION } from '../../../../common/elasticsearch_fieldnames'; +import type { SearchServiceParams } from '../../../../common/search_strategies/correlations/types'; + +import { getQueryWithParams } from './get_query_with_params'; + +export interface HistogramItem { + key: number; + doc_count: number; +} + +interface ResponseHitSource { + [s: string]: unknown; +} +interface ResponseHit { + _source: ResponseHitSource; +} + +export const getTransactionDurationRangesRequest = ( + params: SearchServiceParams, + rangesSteps: number[], + fieldName?: string, + fieldValue?: string +): estypes.SearchRequest => { + const query = getQueryWithParams({ params, fieldName, fieldValue }); + + const ranges = rangesSteps.reduce( + (p, to) => { + const from = p[p.length - 1].to; + p.push({ from, to }); + return p; + }, + [{ to: 0 }] as Array<{ from?: number; to?: number }> + ); + ranges.push({ from: ranges[ranges.length - 1].to }); + + return { + index: params.index, + body: { + query, + size: 0, + aggs: { + logspace_ranges: { + range: { + field: TRANSACTION_DURATION, + ranges, + }, + }, + }, + }, + }; +}; + +export const fetchTransactionDurationRanges = async ( + esClient: ElasticsearchClient, + params: SearchServiceParams, + rangesSteps: number[], + fieldName?: string, + fieldValue?: string +): Promise> => { + const resp = await esClient.search( + getTransactionDurationRangesRequest( + params, + rangesSteps, + fieldName, + fieldValue + ) + ); + + if (resp.body.aggregations === undefined) { + throw new Error( + 'fetchTransactionDurationCorrelation failed, did not return aggregations.' + ); + } + + return (resp.body.aggregations + .logspace_ranges as estypes.AggregationsMultiBucketAggregate<{ + from: number; + doc_count: number; + }>).buckets + .map((d) => ({ + key: d.from, + doc_count: d.doc_count, + })) + .filter((d) => d.key !== undefined); +}; diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/search_strategy.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/search_strategy.ts new file mode 100644 index 0000000000000..d6b4e0e7094b3 --- /dev/null +++ b/x-pack/plugins/apm/server/lib/search_strategies/correlations/search_strategy.ts @@ -0,0 +1,94 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import uuid from 'uuid'; +import { of } from 'rxjs'; + +import type { ISearchStrategy } from '../../../../../../../src/plugins/data/server'; +import { + IKibanaSearchRequest, + IKibanaSearchResponse, +} from '../../../../../../../src/plugins/data/common'; + +import type { + SearchServiceParams, + SearchServiceValue, +} from '../../../../common/search_strategies/correlations/types'; + +import { asyncSearchServiceProvider } from './async_search_service'; + +export type PartialSearchRequest = IKibanaSearchRequest; +export type PartialSearchResponse = IKibanaSearchResponse<{ + values: SearchServiceValue[]; +}>; + +export const apmCorrelationsSearchStrategyProvider = (): ISearchStrategy< + PartialSearchRequest, + PartialSearchResponse +> => { + const asyncSearchServiceMap = new Map< + string, + ReturnType + >(); + + return { + search: (request, options, deps) => { + if (request.params === undefined) { + throw new Error('Invalid request parameters.'); + } + + const id = request.id ?? uuid(); + + const getAsyncSearchServiceState = + asyncSearchServiceMap.get(id) ?? + asyncSearchServiceProvider(deps.esClient.asCurrentUser, request.params); + + const { + error, + isRunning, + loaded, + started, + total, + values, + percentileThresholdValue, + overallHistogram, + } = getAsyncSearchServiceState(); + + if (error instanceof Error) { + asyncSearchServiceMap.delete(id); + throw error; + } else if (isRunning) { + asyncSearchServiceMap.set(id, getAsyncSearchServiceState); + } else { + asyncSearchServiceMap.delete(id); + } + + const took = Date.now() - started; + + return of({ + id, + loaded, + total, + isRunning, + isPartial: isRunning, + rawResponse: { + took, + values, + percentileThresholdValue, + overallHistogram, + }, + }); + }, + cancel: async (id, options, deps) => { + const getAsyncSearchServiceState = asyncSearchServiceMap.get(id); + if (getAsyncSearchServiceState !== undefined) { + getAsyncSearchServiceState().cancel(); + asyncSearchServiceMap.delete(id); + } + }, + }; +}; diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/utils/aggregation_utils.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/utils/aggregation_utils.ts new file mode 100644 index 0000000000000..34e5ae2795d58 --- /dev/null +++ b/x-pack/plugins/apm/server/lib/search_strategies/correlations/utils/aggregation_utils.ts @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { estypes } from '@elastic/elasticsearch'; +import { PERCENTILES_STEP } from '../constants'; + +export const computeExpectationsAndRanges = ( + percentiles: number[], + step = PERCENTILES_STEP +): { + expectations: number[]; + ranges: estypes.AggregationsAggregationRange[]; +} => { + const tempPercentiles = [percentiles[0]]; + const tempFractions = [step / 100]; + // Collapse duplicates + for (let i = 1; i < percentiles.length; i++) { + if (percentiles[i] !== percentiles[i - 1]) { + tempPercentiles.push(percentiles[i]); + tempFractions.push(2 / 100); + } else { + tempFractions[tempFractions.length - 1] = + tempFractions[tempFractions.length - 1] + step / 100; + } + } + tempFractions.push(2 / 100); + + const ranges = percentiles.reduce((p, to) => { + const from = p[p.length - 1]?.to; + if (from) { + p.push({ from, to }); + } else { + p.push({ to }); + } + return p; + }, [] as Array<{ from?: number; to?: number }>); + ranges.push({ from: ranges[ranges.length - 1].to }); + + const expectations = [tempPercentiles[0]]; + for (let i = 1; i < tempPercentiles.length; i++) { + expectations.push( + (tempFractions[i - 1] * tempPercentiles[i - 1] + + tempFractions[i] * tempPercentiles[i]) / + (tempFractions[i - 1] + tempFractions[i]) + ); + } + expectations.push(tempPercentiles[tempPercentiles.length - 1]); + return { expectations, ranges }; +}; diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/utils/index.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/utils/index.ts new file mode 100644 index 0000000000000..ab6190fb288ad --- /dev/null +++ b/x-pack/plugins/apm/server/lib/search_strategies/correlations/utils/index.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './math_utils'; +export * from './aggregation_utils'; diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/utils/math_utils.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/utils/math_utils.ts new file mode 100644 index 0000000000000..01e856e511fc2 --- /dev/null +++ b/x-pack/plugins/apm/server/lib/search_strategies/correlations/utils/math_utils.ts @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { range } from 'lodash'; +import { HistogramItem } from '../query_ranges'; +import { asPreciseDecimal } from '../../../../../common/utils/formatters'; + +// From https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Math/random +export function getRandomInt(min: number, max: number) { + min = Math.ceil(min); + max = Math.floor(max); + return Math.floor(Math.random() * (max - min + 1) + min); // The maximum is inclusive and the minimum is inclusive +} + +// Roughly compare histograms by sampling random bins +// And rounding up histogram count to account for different floating points +export const isHistogramRoughlyEqual = ( + a: HistogramItem[], + b: HistogramItem[], + { numBinsToSample = 10, significantFraction = 3 } +) => { + if (a.length !== b.length) return false; + + const sampledIndices = Array.from(Array(numBinsToSample).keys()).map(() => + getRandomInt(0, a.length - 1) + ); + return !sampledIndices.some((idx) => { + return ( + asPreciseDecimal(a[idx].key, significantFraction) !== + asPreciseDecimal(b[idx].key, significantFraction) && + roundToNearest(a[idx].doc_count) !== roundToNearest(b[idx].doc_count) + ); + }); +}; + +/** Round numeric to the nearest 5 + * E.g. if roundBy = 5, results will be 11 -> 10, 14 -> 10, 16 -> 20 + */ +export const roundToNearest = (n: number, roundBy = 5) => { + return Math.ceil((n + 1) / roundBy) * roundBy; +}; + +/** + * Create a rough stringified version of the histogram + */ +export const hashHistogram = ( + histogram: HistogramItem[], + { significantFraction = 3, numBinsToSample = 10 } +) => { + // Generate bins to sample evenly + const sampledIndices = Array.from( + range( + 0, + histogram.length - 1, + Math.ceil(histogram.length / numBinsToSample) + ) + ); + return JSON.stringify( + sampledIndices.map((idx) => { + return `${asPreciseDecimal( + histogram[idx].key, + significantFraction + )}-${roundToNearest(histogram[idx].doc_count)}`; + }) + ); +}; diff --git a/x-pack/plugins/apm/server/plugin.ts b/x-pack/plugins/apm/server/plugin.ts index dd422e51550a2..2d3638272508e 100644 --- a/x-pack/plugins/apm/server/plugin.ts +++ b/x-pack/plugins/apm/server/plugin.ts @@ -28,6 +28,7 @@ import { registerFleetPolicyCallbacks } from './lib/fleet/register_fleet_policy_ import { createApmTelemetry } from './lib/apm_telemetry'; import { createApmEventClient } from './lib/helpers/create_es_client/create_apm_event_client'; import { getInternalSavedObjectsClient } from './lib/helpers/get_internal_saved_objects_client'; +import { apmCorrelationsSearchStrategyProvider } from './lib/search_strategies/correlations'; import { createApmAgentConfigurationIndex } from './lib/settings/agent_configuration/create_agent_config_index'; import { getApmIndices } from './lib/settings/apm_indices/get_apm_indices'; import { createApmCustomLinkIndex } from './lib/settings/custom_link/create_custom_link_index'; @@ -236,6 +237,13 @@ export class APMPlugin logger: this.logger, }); + // search strategies for async partial search results + if (plugins.data?.search?.registerSearchStrategy !== undefined) { + plugins.data.search.registerSearchStrategy( + 'apmCorrelationsSearchStrategy', + apmCorrelationsSearchStrategyProvider() + ); + } return { config$: mergedConfig$, getApmIndices: boundGetApmIndices, diff --git a/x-pack/plugins/ml/server/types.ts b/x-pack/plugins/ml/server/types.ts index 013feb568ca53..b04b8d8601772 100644 --- a/x-pack/plugins/ml/server/types.ts +++ b/x-pack/plugins/ml/server/types.ts @@ -12,12 +12,17 @@ import type { SecurityPluginSetup } from '../../security/server'; import type { PluginSetupContract as FeaturesPluginSetup } from '../../features/server'; import type { LicensingPluginSetup } from '../../licensing/server'; import type { SpacesPluginSetup, SpacesPluginStart } from '../../spaces/server'; + import type { MlLicense } from '../common/license'; import type { ResolveMlCapabilities } from '../common/types/capabilities'; import type { RouteGuard } from './lib/route_guard'; import type { AlertingPlugin } from '../../alerting/server'; import type { ActionsPlugin } from '../../actions/server'; import type { UsageCollectionSetup } from '../../../../src/plugins/usage_collection/server'; +import type { + PluginSetup as DataPluginSetup, + PluginStart as DataPluginStart, +} from '../../../../src/plugins/data/server'; export interface LicenseCheckResult { isAvailable: boolean; @@ -41,6 +46,7 @@ export interface SavedObjectsRouteDeps { export interface PluginsSetup { cloud: CloudSetup; + data: DataPluginSetup; features: FeaturesPluginSetup; home: HomeServerPluginSetup; licensing: LicensingPluginSetup; @@ -52,6 +58,7 @@ export interface PluginsSetup { } export interface PluginsStart { + data: DataPluginStart; spaces?: SpacesPluginStart; }