[APM] Add correlations API (#78882)

x-pack/plugins/apm/public/components/app/Correlations/index.tsx

@@ -0,0 +1,85 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import React from 'react';
+import url from 'url';
+import { useParams } from 'react-router-dom';
+import { useLocation } from 'react-router-dom';
+import { EuiTitle, EuiListGroup } from '@elastic/eui';
+import { useUrlParams } from '../../../hooks/useUrlParams';
+import { useApmPluginContext } from '../../../hooks/useApmPluginContext';
+
+const SESSION_STORAGE_KEY = 'apm.debug.show_correlations';
+
+export function Correlations() {
+  const location = useLocation();
+  const { serviceName } = useParams<{ serviceName?: string }>();
+  const { urlParams, uiFilters } = useUrlParams();
+  const { core } = useApmPluginContext();
+  const { transactionName, transactionType, start, end } = urlParams;
+
+  if (
+    !location.search.includes('&_show_correlations') &&
+    sessionStorage.getItem(SESSION_STORAGE_KEY) !== 'true'
+  ) {
+    return null;
+  }
+
+  sessionStorage.setItem(SESSION_STORAGE_KEY, 'true');
+
+  const query = {
+    serviceName,
+    transactionName,
+    transactionType,
+    start,
+    end,
+    uiFilters: JSON.stringify(uiFilters),
+    fieldNames:
+      'user.username,user.id,host.ip,user_agent.name,kubernetes.pod.uuid,url.domain,container.id,service.node.name',
+  };
+
+  const listItems = [
+    {
+      label: 'Show correlations between two ranges',
+      href: url.format({
+        query: {
+          ...query,
+          gap: 24,
+        },
+        pathname: core.http.basePath.prepend(`/api/apm/correlations/ranges`),
+      }),
+      isDisabled: false,
+      iconType: 'tokenRange',
+      size: 's' as const,
+    },
+
+    {
+      label: 'Show correlations for slow transactions',
+      href: url.format({
+        query: {
+          ...query,
+          durationPercentile: 95,
+        },
+        pathname: core.http.basePath.prepend(
+          `/api/apm/correlations/slow_durations`
+        ),
+      }),
+      isDisabled: false,
+      iconType: 'clock',
+      size: 's' as const,
+    },
+  ];
+
+  return (
+    <>
+      <EuiTitle size="m">
+        <h2>Correlations</h2>
+      </EuiTitle>
+
+      <EuiListGroup listItems={listItems} />
+    </>
+  );
+}

x-pack/plugins/apm/public/components/app/ServiceOverview/index.tsx

@@ -21,6 +21,7 @@ import { useApmPluginContext } from '../../../hooks/useApmPluginContext';
 import { MLCallout } from './ServiceList/MLCallout';
 import { useLocalStorage } from '../../../hooks/useLocalStorage';
 import { useAnomalyDetectionJobs } from '../../../hooks/useAnomalyDetectionJobs';
+import { Correlations } from '../Correlations';
 
 const initialData = {
   items: [],
@@ -117,6 +118,9 @@ export function ServiceOverview() {
   return (
     <>
       <EuiSpacer />
+
+      <Correlations />
+
       <EuiFlexGroup>
         <EuiFlexItem grow={1}>
           <LocalUIFilters {...localFiltersConfig} />

x-pack/plugins/apm/public/components/app/TransactionDetails/index.tsx

@@ -31,6 +31,7 @@ import { fromQuery, toQuery } from '../../shared/Links/url_helpers';
 import { useUrlParams } from '../../../hooks/useUrlParams';
 import { LocalUIFilters } from '../../shared/LocalUIFilters';
 import { HeightRetainer } from '../../shared/HeightRetainer';
+import { Correlations } from '../Correlations';
 
 interface Sample {
   traceId: string;
@@ -111,6 +112,8 @@ export function TransactionDetails({
         </EuiTitle>
       </ApmHeader>
 
+      <Correlations />
+
       <EuiFlexGroup>
         <EuiFlexItem grow={1}>
           <LocalUIFilters {...localUIFiltersConfig} />

x-pack/plugins/apm/public/components/app/TransactionOverview/index.tsx

@@ -37,6 +37,7 @@ import { TransactionList } from './TransactionList';
 import { useRedirect } from './useRedirect';
 import { TRANSACTION_PAGE_LOAD } from '../../../../common/transaction_types';
 import { UserExperienceCallout } from './user_experience_callout';
+import { Correlations } from '../Correlations';
 
 function getRedirectLocation({
   urlParams,
@@ -117,6 +118,7 @@ export function TransactionOverview({ serviceName }: TransactionOverviewProps) {
 
   return (
     <>
+      <Correlations />
       <EuiSpacer />
       <EuiFlexGroup>
         <EuiFlexItem grow={1}>

x-pack/plugins/apm/server/lib/helpers/setup_request.ts

@@ -49,7 +49,15 @@ export interface SetupTimeRange {
 interface SetupRequestParams {
   query?: {
     _debug?: boolean;
+
+    /**
+     * Timestamp in ms since epoch
+     */
     start?: string;
+
+    /**
+     * Timestamp in ms since epoch
+     */
     end?: string;
     uiFilters?: string;
     processorEvent?: ProcessorEvent;

x-pack/plugins/apm/server/lib/transaction_groups/correlations/get_correlations_for_ranges.ts

@@ -0,0 +1,90 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { rangeFilter } from '../../../../common/utils/range_filter';
+import {
+  SERVICE_NAME,
+  TRANSACTION_NAME,
+  TRANSACTION_TYPE,
+} from '../../../../common/elasticsearch_fieldnames';
+import { ProcessorEvent } from '../../../../common/processor_event';
+import { Setup, SetupTimeRange } from '../../helpers/setup_request';
+import {
+  getSignificantTermsAgg,
+  formatAggregationResponse,
+} from './get_significant_terms_agg';
+import { SignificantTermsScoring } from './scoring_rt';
+
+export async function getCorrelationsForRanges({
+  serviceName,
+  transactionType,
+  transactionName,
+  scoring,
+  gapBetweenRanges,
+  fieldNames,
+  setup,
+}: {
+  serviceName: string | undefined;
+  transactionType: string | undefined;
+  transactionName: string | undefined;
+  scoring: SignificantTermsScoring;
+  gapBetweenRanges: number;
+  fieldNames: string[];
+  setup: Setup & SetupTimeRange;
+}) {
+  const { start, end, esFilter, apmEventClient } = setup;
+
+  const baseFilters = [...esFilter];
+
+  if (serviceName) {
+    baseFilters.push({ term: { [SERVICE_NAME]: serviceName } });
+  }
+
+  if (transactionType) {
+    baseFilters.push({ term: { [TRANSACTION_TYPE]: transactionType } });
+  }
+
+  if (transactionName) {
+    baseFilters.push({ term: { [TRANSACTION_NAME]: transactionName } });
+  }
+
+  const diff = end - start + gapBetweenRanges;
+  const baseRangeStart = start - diff;
+  const baseRangeEnd = end - diff;
+  const backgroundFilters = [
+    ...baseFilters,
+    { range: rangeFilter(baseRangeStart, baseRangeEnd) },
+  ];
+
+  const params = {
+    apm: { events: [ProcessorEvent.transaction] },
+    body: {
+      size: 0,
+      query: {
+        bool: { filter: [...baseFilters, { range: rangeFilter(start, end) }] },
+      },
+      aggs: getSignificantTermsAgg({
+        fieldNames,
+        backgroundFilters,
+        backgroundIsSuperset: false,
+        scoring,
+      }),
+    },
+  };
+
+  const response = await apmEventClient.search(params);
+
+  return {
+    message: `Showing significant fields between the ranges`,
+    firstRange: `${new Date(baseRangeStart).toISOString()} - ${new Date(
+      baseRangeEnd
+    ).toISOString()}`,
+    lastRange: `${new Date(start).toISOString()} - ${new Date(
+      end
+    ).toISOString()}`,
+    response: formatAggregationResponse(response.aggregations),
+  };
+}

x-pack/plugins/apm/server/lib/transaction_groups/correlations/get_correlations_for_slow_transactions.ts

@@ -0,0 +1,94 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { asDuration } from '../../../../common/utils/formatters';
+import { ESFilter } from '../../../../typings/elasticsearch';
+import { rangeFilter } from '../../../../common/utils/range_filter';
+import {
+  SERVICE_NAME,
+  TRANSACTION_DURATION,
+  TRANSACTION_NAME,
+  TRANSACTION_TYPE,
+} from '../../../../common/elasticsearch_fieldnames';
+import { ProcessorEvent } from '../../../../common/processor_event';
+import { Setup, SetupTimeRange } from '../../helpers/setup_request';
+import { getDurationForPercentile } from './get_duration_for_percentile';
+import {
+  formatAggregationResponse,
+  getSignificantTermsAgg,
+} from './get_significant_terms_agg';
+import { SignificantTermsScoring } from './scoring_rt';
+
+export async function getCorrelationsForSlowTransactions({
+  serviceName,
+  transactionType,
+  transactionName,
+  durationPercentile,
+  fieldNames,
+  scoring,
+  setup,
+}: {
+  serviceName: string | undefined;
+  transactionType: string | undefined;
+  transactionName: string | undefined;
+  scoring: SignificantTermsScoring;
+  durationPercentile: number;
+  fieldNames: string[];
+  setup: Setup & SetupTimeRange;
+}) {
+  const { start, end, esFilter, apmEventClient } = setup;
+
+  const backgroundFilters: ESFilter[] = [
+    ...esFilter,
+    { range: rangeFilter(start, end) },
+  ];
+
+  if (serviceName) {
+    backgroundFilters.push({ term: { [SERVICE_NAME]: serviceName } });
+  }
+
+  if (transactionType) {
+    backgroundFilters.push({ term: { [TRANSACTION_TYPE]: transactionType } });
+  }
+
+  if (transactionName) {
+    backgroundFilters.push({ term: { [TRANSACTION_NAME]: transactionName } });
+  }
+
+  const durationForPercentile = await getDurationForPercentile({
+    durationPercentile,
+    backgroundFilters,
+    setup,
+  });
+
+  const params = {
+    apm: { events: [ProcessorEvent.transaction] },
+    body: {
+      size: 0,
+      query: {
+        bool: {
+          // foreground filters
+          filter: [
+            ...backgroundFilters,
+            {
+              range: { [TRANSACTION_DURATION]: { gte: durationForPercentile } },
+            },
+          ],
+        },
+      },
+      aggs: getSignificantTermsAgg({ fieldNames, backgroundFilters, scoring }),
+    },
+  };
+
+  const response = await apmEventClient.search(params);
+
+  return {
+    message: `Showing significant fields for transactions slower than ${durationPercentile}th percentile (${asDuration(
+      durationForPercentile
+    )})`,
+    response: formatAggregationResponse(response.aggregations),
+  };
+}

x-pack/plugins/apm/server/lib/transaction_groups/correlations/get_duration_for_percentile.ts

@@ -0,0 +1,43 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { ESFilter } from '../../../../typings/elasticsearch';
+import { TRANSACTION_DURATION } from '../../../../common/elasticsearch_fieldnames';
+import { ProcessorEvent } from '../../../../common/processor_event';
+import { Setup, SetupTimeRange } from '../../helpers/setup_request';
+
+export async function getDurationForPercentile({
+  durationPercentile,
+  backgroundFilters,
+  setup,
+}: {
+  durationPercentile: number;
+  backgroundFilters: ESFilter[];
+  setup: Setup & SetupTimeRange;
+}) {
+  const { apmEventClient } = setup;
+  const res = await apmEventClient.search({
+    apm: {
+      events: [ProcessorEvent.transaction],
+    },
+    body: {
+      size: 0,
+      query: {
+        bool: { filter: backgroundFilters },
+      },
+      aggs: {
+        percentile: {
+          percentiles: {
+            field: TRANSACTION_DURATION,
+            percents: [durationPercentile],
+          },
+        },
+      },
+    },
+  });
+
+  return Object.values(res.aggregations?.percentile.values || {})[0];
+}