[ML] APM Latency Correlations (#99905) (#103745)

* [ML] Async Search Service.

* [ML] Log Log Area Charts Grid. Scatterplot with streaming data.

* [ML] Remove scatterplot streaming demo.

* [ML] Improved histogram bins for log scale.

* [ML] Move client side code from Ml to APM plugin.

* [ML] Adds correlation table.

* [ML] Layout tweaks.

* [ML] Move server side code from ML to APM.

* [ML] Remove console.logs.

* [ML] Fix types.

* [ML] Chart area line type fix. Slowness %.

* [ML] Comment unused vars.

* Fix missing pluginsStart data in context

* Add KS test value and rename table columns

* Update percentiles to be correct

* Make columns optional

* Update fractionals/expectations to match with backend logic

* Update so progress is 100% when all is completed

* Make pre-processing steps smaller part of overall progress(to show charts earlier)

* Add no correlations found message

* Fix progress logic

* Fix incorrect threshold, types

* Add back APM filtering functionality to match with other table

* Improve histogram equality check with random sampling

* Show overall latency distribution right away

* Rename demo tab to latencyCorrelationsTab

* Update percentiles query to use hdr

* [ML] Fix issue where apm-* might have multiple indices with different mappings where keyword is not the only type

- Fix to check for keyword mapping more thoroughly
- Add try catch if one of the es search fail, don't quit the entire fetch

* Remove commented code

* Remove kstest column, round correlation to 2 sigfig dec

* Remove old latency tab, replace with ml latency correlations tab as first/default one

* Set axis to start at 0 because agg may results in weird

* Remove commented code for grouping duplicates

* Update msg to mean significant correlations

* Add i18n

* Change correlations flyout back to medium size

* Add name of service or transaction for clarity

* Share i18n

* Consolidate roundToDecimalPlace usage

* Remove redundant isDuplicate

* Create MAX_CORRELATIONS_TO_SHOW

* Update mlCorrelationcolumns

* Fix i18n quotations

* Update query to include filter

* Revert "Update query to include filter"

This reverts commit 9a37eec

* Rename MlCorrelations to MlLatencyCorrelations for clarity

* Add pagination

* Update include/exclude logic for field candidates and add ip field support

* Add transactionName filter suport

* Reorganize math utils

* Group duplicates together

* Fix typescript, better hasPrefixToInclude support check,

* Remove Finished toast

* Add title to y axis

* Reduce number of tick labels to show

* Highlight table row that is being used for graph

* Add from to follow MDN guideline

* Add APM style prefix

* Fix i18n

* [ML] Fix logic for tick format to only show power of 10

* Replace roundToDecimalPlace with asPreciseDecimal

* Switch to lodash range

* Clean up get_query_with_params

* Prioritize candidates using field_terms

* Update percentiles result type to be array instead of objects

* Use observability's rangeQuery instead

* Change arg format of query

* Revert candidate_terms logic

* Consolidate fractions, expectations, and ranges calc

* Add tooltip for Correlation

* Change terms size to 20

* Move env/service/transaction sticky header to top level, remove link

* Add support for http.response.status_code

* Replace histogram circular markers with bars

* Delete unused roundToDecimalPlace

* Add fractions calculation

* Make notes of fractions and fix bucket correlation

* Remove any, commented code, consolidate cancelFetch

* Use es6 max

* Align tooltip at the top

* Get rid of getCoreServices, param docs, rename type, remove rangeQuery

* Adjust range

* Show all values without grouping duplicates

* Fix pagination

* Make flyout larger

Co-authored-by: Quynh Nguyen <[email protected]>
Co-authored-by: Kibana Machine <[email protected]>

Co-authored-by: Walter Rafelsberger <[email protected]>
Co-authored-by: Quynh Nguyen <[email protected]>

x-pack/plugins/apm/common/search_strategies/correlations/types.ts

@@ -0,0 +1,51 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export interface HistogramItem {
+  key: number;
+  doc_count: number;
+}
+
+export interface ResponseHitSource {
+  [s: string]: unknown;
+}
+
+export interface ResponseHit {
+  _source: ResponseHitSource;
+}
+
+export interface SearchServiceParams {
+  index: string;
+  environment?: string;
+  kuery?: string;
+  serviceName?: string;
+  transactionName?: string;
+  transactionType?: string;
+  start?: string;
+  end?: string;
+  percentileThreshold?: number;
+  percentileThresholdValue?: number;
+}
+
+export interface SearchServiceValue {
+  histogram: HistogramItem[];
+  value: string;
+  field: string;
+  correlation: number;
+  ksTest: number;
+  duplicatedFields?: string[];
+}
+
+export interface AsyncSearchProviderProgress {
+  started: number;
+  loadedHistogramStepsize: number;
+  loadedOverallHistogram: number;
+  loadedFieldCanditates: number;
+  loadedFieldValuePairs: number;
+  loadedHistograms: number;
+  getOverallProgress: () => number;
+}

x-pack/plugins/apm/common/utils/formatters/formatters.ts

@@ -18,6 +18,14 @@ export function asDecimal(value?: number | null) {
   return numeral(value).format('0,0.0');
 }
 
+export function asPreciseDecimal(value?: number | null, dp: number = 3) {
+  if (!isFiniteNumber(value)) {
+    return NOT_AVAILABLE_LABEL;
+  }
+
+  return numeral(value).format(`0,0.${'0'.repeat(dp)}`);
+}
+
 export function asInteger(value?: number | null) {
   if (!isFiniteNumber(value)) {
     return NOT_AVAILABLE_LABEL;

x-pack/plugins/apm/public/application/index.tsx

@@ -47,6 +47,7 @@ export const renderApp = ({
     config,
     core: coreStart,
     plugins: pluginsSetup,
+    data: pluginsStart.data,
     observability: pluginsStart.observability,
     observabilityRuleTypeRegistry,
   };

x-pack/plugins/apm/public/components/app/correlations/correlations_chart.tsx

@@ -0,0 +1,221 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import React from 'react';
+import {
+  AnnotationDomainType,
+  Chart,
+  CurveType,
+  Settings,
+  Axis,
+  ScaleType,
+  Position,
+  AreaSeries,
+  RecursivePartial,
+  AxisStyle,
+  PartialTheme,
+  LineAnnotation,
+  LineAnnotationDatum,
+} from '@elastic/charts';
+
+import euiVars from '@elastic/eui/dist/eui_theme_light.json';
+
+import { EuiSpacer } from '@elastic/eui';
+
+import { i18n } from '@kbn/i18n';
+
+import { getDurationFormatter } from '../../../../common/utils/formatters';
+
+import { useTheme } from '../../../hooks/use_theme';
+import { HistogramItem } from '../../../../common/search_strategies/correlations/types';
+
+const { euiColorMediumShade } = euiVars;
+const axisColor = euiColorMediumShade;
+
+const axes: RecursivePartial<AxisStyle> = {
+  axisLine: {
+    stroke: axisColor,
+  },
+  tickLabel: {
+    fontSize: 10,
+    fill: axisColor,
+    padding: 0,
+  },
+  tickLine: {
+    stroke: axisColor,
+    size: 5,
+  },
+  gridLine: {
+    horizontal: {
+      dash: [1, 2],
+    },
+    vertical: {
+      strokeWidth: 1,
+    },
+  },
+};
+const chartTheme: PartialTheme = {
+  axes,
+  legend: {
+    spacingBuffer: 100,
+  },
+  areaSeriesStyle: {
+    line: {
+      visible: false,
+    },
+  },
+};
+
+interface CorrelationsChartProps {
+  field?: string;
+  value?: string;
+  histogram?: HistogramItem[];
+  markerValue: number;
+  markerPercentile: number;
+  overallHistogram: HistogramItem[];
+}
+
+const annotationsStyle = {
+  line: {
+    strokeWidth: 1,
+    stroke: 'gray',
+    opacity: 0.8,
+  },
+  details: {
+    fontSize: 8,
+    fontFamily: 'Arial',
+    fontStyle: 'normal',
+    fill: 'gray',
+    padding: 0,
+  },
+};
+
+const CHART_PLACEHOLDER_VALUE = 0.0001;
+
+// Elastic charts will show any lone bin (i.e. a populated bin followed by empty bin)
+// as a circular marker instead of a bar
+// This provides a workaround by making the next bin not empty
+export const replaceHistogramDotsWithBars = (
+  originalHistogram: HistogramItem[] | undefined
+) => {
+  if (originalHistogram === undefined) return;
+  const histogram = [...originalHistogram];
+  {
+    for (let i = 0; i < histogram.length - 1; i++) {
+      if (
+        histogram[i].doc_count > 0 &&
+        histogram[i].doc_count !== CHART_PLACEHOLDER_VALUE &&
+        histogram[i + 1].doc_count === 0
+      ) {
+        histogram[i + 1].doc_count = CHART_PLACEHOLDER_VALUE;
+      }
+    }
+    return histogram;
+  }
+};
+
+export function CorrelationsChart({
+  field,
+  value,
+  histogram: originalHistogram,
+  markerValue,
+  markerPercentile,
+  overallHistogram,
+}: CorrelationsChartProps) {
+  const euiTheme = useTheme();
+
+  if (!Array.isArray(overallHistogram)) return <div />;
+  const annotationsDataValues: LineAnnotationDatum[] = [
+    { dataValue: markerValue, details: `${markerPercentile}th percentile` },
+  ];
+
+  const xMax = Math.max(...overallHistogram.map((d) => d.key)) ?? 0;
+
+  const durationFormatter = getDurationFormatter(xMax);
+
+  const histogram = replaceHistogramDotsWithBars(originalHistogram);
+
+  return (
+    <div style={{ overflow: 'hidden', textOverflow: 'ellipsis' }}>
+      <Chart
+        size={{
+          height: '250px',
+        }}
+      >
+        <Settings
+          rotation={0}
+          theme={chartTheme}
+          showLegend
+          legendPosition={Position.Bottom}
+        />
+        <LineAnnotation
+          id="annotation_1"
+          domainType={AnnotationDomainType.XDomain}
+          dataValues={annotationsDataValues}
+          style={annotationsStyle}
+          marker={`${markerPercentile}p`}
+          markerPosition={'top'}
+        />
+        <Axis
+          id="x-axis"
+          title=""
+          position={Position.Bottom}
+          tickFormat={(d) => durationFormatter(d).formatted}
+        />
+        <Axis
+          id="y-axis"
+          title={i18n.translate(
+            'xpack.apm.correlations.latency.chart.numberOfTransactionsLabel',
+            { defaultMessage: '# transactions' }
+          )}
+          position={Position.Left}
+          tickFormat={(d) =>
+            d === 0 || Number.isInteger(Math.log10(d)) ? d : ''
+          }
+        />
+        <AreaSeries
+          id={i18n.translate(
+            'xpack.apm.correlations.latency.chart.overallLatencyDistributionLabel',
+            { defaultMessage: 'Overall latency distribution' }
+          )}
+          xScaleType={ScaleType.Log}
+          yScaleType={ScaleType.Log}
+          data={overallHistogram}
+          curve={CurveType.CURVE_STEP_AFTER}
+          xAccessor="key"
+          yAccessors={['doc_count']}
+          color={euiTheme.eui.euiColorVis1}
+          fit="lookahead"
+        />
+        {Array.isArray(histogram) &&
+          field !== undefined &&
+          value !== undefined && (
+            <AreaSeries
+              id={i18n.translate(
+                'xpack.apm.correlations.latency.chart.selectedTermLatencyDistributionLabel',
+                {
+                  defaultMessage: '{fieldName}:{fieldValue}',
+                  values: {
+                    fieldName: field,
+                    fieldValue: value,
+                  },
+                }
+              )}
+              xScaleType={ScaleType.Log}
+              yScaleType={ScaleType.Log}
+              data={histogram}
+              curve={CurveType.CURVE_STEP_AFTER}
+              xAccessor="key"
+              yAccessors={['doc_count']}
+              color={euiTheme.eui.euiColorVis2}
+            />
+          )}
+      </Chart>
+      <EuiSpacer size="s" />
+    </div>
+  );
+}