[ML] Fix custom index name settings, functional tests for APM Latency Correlation.

x-pack/plugins/apm/common/search_strategies/correlations/types.ts

@@ -19,7 +19,6 @@ export interface ResponseHit {
 }
 
 export interface SearchServiceParams {
-  index: string;
   environment?: string;
   kuery?: string;
   serviceName?: string;
@@ -31,6 +30,10 @@ export interface SearchServiceParams {
   percentileThresholdValue?: number;
 }
 
+export interface SearchServiceFetchParams extends SearchServiceParams {
+  index: string;
+}
+
 export interface SearchServiceValue {
   histogram: HistogramItem[];
   value: string;

x-pack/plugins/apm/public/components/app/correlations/correlations_chart.tsx

@@ -70,6 +70,11 @@ const chartTheme: PartialTheme = {
   },
 };
 
+// Log based axis cannot start a 0. Use a small positive number instead.
+const yAxisDomain = {
+  min: 0.00001,
+};
+
 interface CorrelationsChartProps {
   field?: string;
   value?: string;
@@ -140,7 +145,10 @@ export function CorrelationsChart({
   const histogram = replaceHistogramDotsWithBars(originalHistogram);
 
   return (
-    <div style={{ overflow: 'hidden', textOverflow: 'ellipsis' }}>
+    <div
+      data-test-subj="apmCorrelationsChart"
+      style={{ overflow: 'hidden', textOverflow: 'ellipsis' }}
+    >
       <Chart
         size={{
           height: '250px',
@@ -168,6 +176,7 @@ export function CorrelationsChart({
         />
         <Axis
           id="y-axis"
+          domain={yAxisDomain}
           title={i18n.translate(
             'xpack.apm.correlations.latency.chart.numberOfTransactionsLabel',
             { defaultMessage: '# transactions' }

x-pack/plugins/apm/public/components/app/correlations/index.tsx

@@ -133,6 +133,7 @@ export function Correlations() {
   return (
     <>
       <EuiButton
+        data-test-subj="apmViewCorrelationsButton"
         fill
         onClick={() => {
           setIsFlyoutVisible(true);
@@ -146,13 +147,17 @@ export function Correlations() {
       {isFlyoutVisible && (
         <EuiPortal>
           <EuiFlyout
+            data-test-subj="apmCorrelationsFlyout"
             size="l"
             ownFocus
             onClose={() => setIsFlyoutVisible(false)}
           >
             <EuiFlyoutHeader hasBorder aria-labelledby="correlations-flyout">
               <EuiTitle>
-                <h2 id="correlations-flyout">
+                <h2
+                  data-test-subj="apmCorrelationsFlyoutHeader"
+                  id="correlations-flyout"
+                >
                   {CORRELATIONS_TITLE}
                   &nbsp;
                   <EuiBetaBadge

x-pack/plugins/apm/public/components/app/correlations/ml_latency_correlations.tsx

@@ -6,8 +6,11 @@
  */
 
 import React, { useEffect, useMemo, useState } from 'react';
-import { useHistory, useParams } from 'react-router-dom';
+import { useHistory, useLocation, useParams } from 'react-router-dom';
 import {
+  EuiCode,
+  EuiAccordion,
+  EuiPanel,
   EuiIcon,
   EuiBasicTableColumn,
   EuiButton,
@@ -72,7 +75,11 @@ export function MlLatencyCorrelations({ onClose }: Props) {
     },
   } = useUrlParams();
 
+  const location = useLocation();
+  const displayLog = location.search.includes('debug=true');
+
   const {
+    log,
     error,
     histograms,
     percentileThresholdValue,
@@ -82,7 +89,6 @@ export function MlLatencyCorrelations({ onClose }: Props) {
     cancelFetch,
     overallHistogram: originalOverallHistogram,
   } = useCorrelations({
-    index: 'apm-*',
     ...{
       ...{
         environment,
@@ -292,9 +298,10 @@ export function MlLatencyCorrelations({ onClose }: Props) {
         </EuiFlexItem>
         <EuiFlexItem>
           <EuiFlexGroup direction="column" gutterSize="none">
-            <EuiFlexItem>
+            <EuiFlexItem data-test-subj="apmCorrelationsLatencyCorrelationsProgressTitle">
               <EuiText size="xs" color="subdued">
                 <FormattedMessage
+                  data-test-subj="apmCorrelationsLatencyCorrelationsProgressTitle"
                   id="xpack.apm.correlations.latencyCorrelations.progressTitle"
                   defaultMessage="Progress: {progress}%"
                   values={{ progress: Math.round(progress * 100) }}
@@ -323,7 +330,7 @@ export function MlLatencyCorrelations({ onClose }: Props) {
       {overallHistogram !== undefined ? (
         <>
           <EuiTitle size="xxs">
-            <h4>
+            <h4 data-test-subj="apmCorrelationsLatencyCorrelationsChartTitle">
               {i18n.translate(
                 'xpack.apm.correlations.latencyCorrelations.chartTitle',
                 {
@@ -347,32 +354,50 @@ export function MlLatencyCorrelations({ onClose }: Props) {
         </>
       ) : null}
 
-      {histograms.length > 0 && selectedHistogram !== undefined && (
-        <CorrelationsTable
-          // @ts-ignore correlations don't have the same column format other tables have
-          columns={mlCorrelationColumns}
-          // @ts-expect-error correlations don't have the same significant term other tables have
-          significantTerms={histogramTerms}
-          status={FETCH_STATUS.SUCCESS}
-          setSelectedSignificantTerm={setSelectedSignificantTerm}
-          selectedTerm={{
-            fieldName: selectedHistogram.field,
-            fieldValue: selectedHistogram.value,
-          }}
-          onFilter={onClose}
-        />
+      <div data-test-subj="apmCorrelationsTable">
+        {histograms.length > 0 && selectedHistogram !== undefined && (
+          <CorrelationsTable
+            // @ts-ignore correlations don't have the same column format other tables have
+            columns={mlCorrelationColumns}
+            // @ts-expect-error correlations don't have the same significant term other tables have
+            significantTerms={histogramTerms}
+            status={FETCH_STATUS.SUCCESS}
+            setSelectedSignificantTerm={setSelectedSignificantTerm}
+            selectedTerm={{
+              fieldName: selectedHistogram.field,
+              fieldValue: selectedHistogram.value,
+            }}
+            onFilter={onClose}
+          />
+        )}
+        {histograms.length < 1 && progress > 0.99 ? (
+          <>
+            <EuiSpacer size="m" />
+            <EuiText textAlign="center">
+              <FormattedMessage
+                id="xpack.apm.correlations.latencyCorrelations.noCorrelationsText"
+                defaultMessage="No significant correlations found"
+              />
+            </EuiText>
+          </>
+        ) : null}
+      </div>
+      {log.length > 0 && displayLog && (
+        <EuiAccordion id="accordion1" buttonContent="Log">
+          <EuiPanel color="subdued">
+            {log.map((d, i) => {
+              const splitItem = d.split(': ');
+              return (
+                <p key={i}>
+                  <small>
+                    <EuiCode>{splitItem[0]}</EuiCode> {splitItem[1]}
+                  </small>
+                </p>
+              );
+            })}
+          </EuiPanel>
+        </EuiAccordion>
       )}
-      {histograms.length < 1 && progress > 0.99 ? (
-        <>
-          <EuiSpacer size="m" />
-          <EuiText textAlign="center">
-            <FormattedMessage
-              id="xpack.apm.correlations.latencyCorrelations.noCorrelationsText"
-              defaultMessage="No significant correlations found"
-            />
-          </EuiText>
-        </>
-      ) : null}
     </>
   );
 }

x-pack/plugins/apm/public/components/app/correlations/use_correlations.ts

@@ -21,7 +21,6 @@ import { useKibana } from '../../../../../../../src/plugins/kibana_react/public'
 import { ApmPluginStartDeps } from '../../../plugin';
 
 interface CorrelationsOptions {
-  index: string;
   environment?: string;
   kuery?: string;
   serviceName?: string;
@@ -106,6 +105,7 @@ export const useCorrelations = (params: CorrelationsOptions) => {
   };
 
   return {
+    log: rawResponse?.log ?? [],
     error,
     histograms: rawResponse?.values ?? [],
     percentileThresholdValue:

x-pack/plugins/apm/public/components/app/service_inventory/service_list/index.tsx

@@ -117,6 +117,7 @@ export function getServiceColumns({
               )}
               <EuiFlexItem className="apmServiceList__serviceNameContainer">
                 <AppLink
+                  data-test-subj="apmServiceListAppLink"
                   serviceName={serviceName}
                   transactionType={transactionType}
                   className="eui-textTruncate"

x-pack/plugins/apm/public/components/routing/templates/apm_service_template.tsx

@@ -93,7 +93,9 @@ function TemplateWithContext({
               <EuiFlexGroup alignItems="center">
                 <EuiFlexItem grow={false}>
                   <EuiTitle size="l">
-                    <h1>{serviceName}</h1>
+                    <h1 data-test-subj="apmMainTemplateHeaderServiceName">
+                      {serviceName}
+                    </h1>
                   </EuiTitle>
                 </EuiFlexItem>
                 <EuiFlexItem grow={false}>

x-pack/plugins/apm/server/lib/search_strategies/correlations/async_search_service.ts

@@ -7,6 +7,7 @@
 
 import { shuffle, range } from 'lodash';
 import type { ElasticsearchClient } from 'src/core/server';
+import type { ApmIndicesConfig } from '../../settings/apm_indices/get_apm_indices';
 import { fetchTransactionDurationFieldCandidates } from './query_field_candidates';
 import { fetchTransactionDurationFieldValuePairs } from './query_field_value_pairs';
 import { fetchTransactionDurationPercentiles } from './query_percentiles';
@@ -16,6 +17,7 @@ import { fetchTransactionDurationRanges, HistogramItem } from './query_ranges';
 import type {
   AsyncSearchProviderProgress,
   SearchServiceParams,
+  SearchServiceFetchParams,
   SearchServiceValue,
 } from '../../../../common/search_strategies/correlations/types';
 import { computeExpectationsAndRanges } from './utils/aggregation_utils';
@@ -28,7 +30,9 @@ const currentTimeAsString = () => new Date().toISOString();
 
 export const asyncSearchServiceProvider = (
   esClient: ElasticsearchClient,
-  params: SearchServiceParams
+  getApmIndices: () => Promise<ApmIndicesConfig>,
+  searchServiceParams: SearchServiceParams,
+  includeFrozen: boolean
 ) => {
   let isCancelled = false;
   let isRunning = true;
@@ -64,6 +68,12 @@ export const asyncSearchServiceProvider = (
 
   const fetchCorrelations = async () => {
     try {
+      const indices = await getApmIndices();
+      const params: SearchServiceFetchParams = {
+        ...searchServiceParams,
+        index: indices['apm_oss.transactionIndices'],
+      };
+
       // 95th percentile to be displayed as a marker in the log log chart
       const {
         totalDocs,

x-pack/plugins/apm/server/lib/search_strategies/correlations/get_query_with_params.ts

@@ -12,7 +12,7 @@ import * as t from 'io-ts';
 
 import type { estypes } from '@elastic/elasticsearch';
 import { TRANSACTION_DURATION } from '../../../../common/elasticsearch_fieldnames';
-import type { SearchServiceParams } from '../../../../common/search_strategies/correlations/types';
+import type { SearchServiceFetchParams } from '../../../../common/search_strategies/correlations/types';
 import { rangeRt } from '../../../routes/default_api_types';
 
 import { Setup, SetupTimeRange } from '../../helpers/setup_request';
@@ -43,7 +43,7 @@ export const getTermsQuery = (
 };
 
 interface QueryParams {
-  params: SearchServiceParams;
+  params: SearchServiceFetchParams;
   fieldName?: string;
   fieldValue?: string;
 }

x-pack/plugins/apm/server/lib/search_strategies/correlations/query_correlation.ts

@@ -10,7 +10,7 @@ import type { estypes } from '@elastic/elasticsearch';
 import type { ElasticsearchClient } from 'src/core/server';
 
 import { TRANSACTION_DURATION } from '../../../../common/elasticsearch_fieldnames';
-import type { SearchServiceParams } from '../../../../common/search_strategies/correlations/types';
+import type { SearchServiceFetchParams } from '../../../../common/search_strategies/correlations/types';
 
 import { getQueryWithParams } from './get_query_with_params';
 
@@ -40,7 +40,7 @@ export interface BucketCorrelation {
 }
 
 export const getTransactionDurationCorrelationRequest = (
-  params: SearchServiceParams,
+  params: SearchServiceFetchParams,
   expectations: number[],
   ranges: estypes.AggregationsAggregationRange[],
   fractions: number[],
@@ -95,7 +95,7 @@ export const getTransactionDurationCorrelationRequest = (
 
 export const fetchTransactionDurationCorrelation = async (
   esClient: ElasticsearchClient,
-  params: SearchServiceParams,
+  params: SearchServiceFetchParams,
   expectations: number[],
   ranges: estypes.AggregationsAggregationRange[],
   fractions: number[],

x-pack/plugins/apm/server/lib/search_strategies/correlations/query_field_candidates.ts

@@ -9,7 +9,7 @@ import type { estypes } from '@elastic/elasticsearch';
 
 import type { ElasticsearchClient } from 'src/core/server';
 
-import type { SearchServiceParams } from '../../../../common/search_strategies/correlations/types';
+import type { SearchServiceFetchParams } from '../../../../common/search_strategies/correlations/types';
 
 import { getQueryWithParams } from './get_query_with_params';
 import { Field } from './query_field_value_pairs';
@@ -37,7 +37,7 @@ export const hasPrefixToInclude = (fieldName: string) => {
 };
 
 export const getRandomDocsRequest = (
-  params: SearchServiceParams
+  params: SearchServiceFetchParams
 ): estypes.SearchRequest => ({
   index: params.index,
   body: {
@@ -56,7 +56,7 @@ export const getRandomDocsRequest = (
 
 export const fetchTransactionDurationFieldCandidates = async (
   esClient: ElasticsearchClient,
-  params: SearchServiceParams
+  params: SearchServiceFetchParams
 ): Promise<{ fieldCandidates: Field[] }> => {
   const { index } = params;
   // Get all fields with keyword mapping

x-pack/plugins/apm/server/lib/search_strategies/correlations/query_field_value_pairs.ts

@@ -11,7 +11,7 @@ import type { estypes } from '@elastic/elasticsearch';
 
 import type {
   AsyncSearchProviderProgress,
-  SearchServiceParams,
+  SearchServiceFetchParams,
 } from '../../../../common/search_strategies/correlations/types';
 
 import { getQueryWithParams } from './get_query_with_params';
@@ -26,7 +26,7 @@ type FieldValuePairs = FieldValuePair[];
 export type Field = string;
 
 export const getTermsAggRequest = (
-  params: SearchServiceParams,
+  params: SearchServiceFetchParams,
   fieldName: string
 ): estypes.SearchRequest => ({
   index: params.index,
@@ -46,7 +46,7 @@ export const getTermsAggRequest = (
 
 export const fetchTransactionDurationFieldValuePairs = async (
   esClient: ElasticsearchClient,
-  params: SearchServiceParams,
+  params: SearchServiceFetchParams,
   fieldCandidates: Field[],
   progress: AsyncSearchProviderProgress
 ): Promise<FieldValuePairs> => {