[ML] Jest unit tests for APM Latency Correlations. (elastic#103907)

Adds jest unit tests for APM Latency Correlations code.

Writing the tests surfaced some minor glitches fixed as part of this PR:
- Fixes a typo in the name for the fetchTransactionDurationPercentiles() function.
- Avoids adding a @timestamp filter if neither start/end are set as parameters for getQueryWithParams().
- Adds a check to only push to ranges arrays if it's length is already greater than 0.
- Makes the check against from more strict otherwise it wouldn't be added as an attribute if 0.
- Fixes progress calculation for field/value pair fetching.
- Removes leading 0 from fractions since an ES update got merged.
- Removes deprecated use of track_total_hits.

x-pack/plugins/apm/server/lib/search_strategies/correlations/async_search_service.ts

@@ -9,7 +9,7 @@ import { shuffle, range } from 'lodash';
 import type { ElasticsearchClient } from 'src/core/server';
 import { fetchTransactionDurationFieldCandidates } from './query_field_candidates';
 import { fetchTransactionDurationFieldValuePairs } from './query_field_value_pairs';
-import { fetchTransactionDurationPecentiles } from './query_percentiles';
+import { fetchTransactionDurationPercentiles } from './query_percentiles';
 import { fetchTransactionDurationCorrelation } from './query_correlation';
 import { fetchTransactionDurationHistogramRangesteps } from './query_histogram_rangesteps';
 import { fetchTransactionDurationRanges, HistogramItem } from './query_ranges';
@@ -59,7 +59,7 @@ export const asyncSearchServiceProvider = (
   const fetchCorrelations = async () => {
     try {
       // 95th percentile to be displayed as a marker in the log log chart
-      const percentileThreshold = await fetchTransactionDurationPecentiles(
+      const percentileThreshold = await fetchTransactionDurationPercentiles(
         esClient,
         params,
         params.percentileThreshold ? [params.percentileThreshold] : undefined
@@ -93,7 +93,7 @@ export const asyncSearchServiceProvider = (
 
       // Create an array of ranges [2, 4, 6, ..., 98]
       const percents = Array.from(range(2, 100, 2));
-      const percentilesRecords = await fetchTransactionDurationPecentiles(
+      const percentilesRecords = await fetchTransactionDurationPercentiles(
         esClient,
         params,
         percents

x-pack/plugins/apm/server/lib/search_strategies/correlations/get_query_with_params.test.ts

@@ -0,0 +1,92 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { getQueryWithParams } from './get_query_with_params';
+
+describe('correlations', () => {
+  describe('getQueryWithParams', () => {
+    it('returns the most basic query filtering on processor.event=transaction', () => {
+      const query = getQueryWithParams({ params: { index: 'apm-*' } });
+      expect(query).toEqual({
+        bool: {
+          filter: [{ term: { 'processor.event': 'transaction' } }],
+        },
+      });
+    });
+
+    it('returns a query considering additional params', () => {
+      const query = getQueryWithParams({
+        params: {
+          index: 'apm-*',
+          serviceName: 'actualServiceName',
+          transactionName: 'actualTransactionName',
+          start: '01-01-2021',
+          end: '31-01-2021',
+          environment: 'dev',
+          percentileThresholdValue: 75,
+        },
+      });
+      expect(query).toEqual({
+        bool: {
+          filter: [
+            { term: { 'processor.event': 'transaction' } },
+            {
+              term: {
+                'service.name': 'actualServiceName',
+              },
+            },
+            {
+              term: {
+                'transaction.name': 'actualTransactionName',
+              },
+            },
+            {
+              range: {
+                '@timestamp': {
+                  gte: '01-01-2021',
+                  lte: '31-01-2021',
+                },
+              },
+            },
+            {
+              term: {
+                'service.environment': 'dev',
+              },
+            },
+            {
+              range: {
+                'transaction.duration.us': {
+                  gte: 75,
+                },
+              },
+            },
+          ],
+        },
+      });
+    });
+
+    it('returns a query considering a custom field/value pair', () => {
+      const query = getQueryWithParams({
+        params: { index: 'apm-*' },
+        fieldName: 'actualFieldName',
+        fieldValue: 'actualFieldValue',
+      });
+      expect(query).toEqual({
+        bool: {
+          filter: [
+            { term: { 'processor.event': 'transaction' } },
+            {
+              term: {
+                actualFieldName: 'actualFieldValue',
+              },
+            },
+          ],
+        },
+      });
+    });
+  });
+});

x-pack/plugins/apm/server/lib/search_strategies/correlations/get_query_with_params.ts

@@ -43,6 +43,10 @@ const getRangeQuery = (
   start?: string,
   end?: string
 ): estypes.QueryDslQueryContainer[] => {
+  if (start === undefined && end === undefined) {
+    return [];
+  }
+
   return [
     {
       range: {

x-pack/plugins/apm/server/lib/search_strategies/correlations/query_correlation.test.ts

@@ -0,0 +1,103 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { estypes } from '@elastic/elasticsearch';
+
+import type { ElasticsearchClient } from 'src/core/server';
+
+import {
+  fetchTransactionDurationCorrelation,
+  getTransactionDurationCorrelationRequest,
+  BucketCorrelation,
+} from './query_correlation';
+
+const params = { index: 'apm-*' };
+const expectations = [1, 3, 5];
+const ranges = [{ to: 1 }, { from: 1, to: 3 }, { from: 3, to: 5 }, { from: 5 }];
+const fractions = [1, 2, 4, 5];
+const totalDocCount = 1234;
+
+describe('query_correlation', () => {
+  describe('getTransactionDurationCorrelationRequest', () => {
+    it('applies options to the returned query with aggregations for correlations and k-test', () => {
+      const query = getTransactionDurationCorrelationRequest(
+        params,
+        expectations,
+        ranges,
+        fractions,
+        totalDocCount
+      );
+
+      expect(query.index).toBe(params.index);
+
+      expect(query?.body?.aggs?.latency_ranges?.range?.field).toBe(
+        'transaction.duration.us'
+      );
+      expect(query?.body?.aggs?.latency_ranges?.range?.ranges).toEqual(ranges);
+
+      expect(
+        (query?.body?.aggs?.transaction_duration_correlation as {
+          bucket_correlation: BucketCorrelation;
+        })?.bucket_correlation.function.count_correlation.indicator
+      ).toEqual({
+        fractions,
+        expectations,
+        doc_count: totalDocCount,
+      });
+
+      expect(
+        (query?.body?.aggs?.ks_test as any)?.bucket_count_ks_test?.fractions
+      ).toEqual(fractions);
+    });
+  });
+
+  describe('fetchTransactionDurationCorrelation', () => {
+    it('returns the data from the aggregations', async () => {
+      const latencyRangesBuckets = [{ to: 1 }, { from: 1, to: 2 }, { from: 2 }];
+      const transactionDurationCorrelationValue = 0.45;
+      const KsTestLess = 0.01;
+
+      const esClientSearchMock = jest.fn((req: estypes.SearchRequest): {
+        body: estypes.SearchResponse;
+      } => {
+        return {
+          body: ({
+            aggregations: {
+              latency_ranges: {
+                buckets: latencyRangesBuckets,
+              },
+              transaction_duration_correlation: {
+                value: transactionDurationCorrelationValue,
+              },
+              ks_test: { less: KsTestLess },
+            },
+          } as unknown) as estypes.SearchResponse,
+        };
+      });
+
+      const esClientMock = ({
+        search: esClientSearchMock,
+      } as unknown) as ElasticsearchClient;
+
+      const resp = await fetchTransactionDurationCorrelation(
+        esClientMock,
+        params,
+        expectations,
+        ranges,
+        fractions,
+        totalDocCount
+      );
+
+      expect(resp).toEqual({
+        correlation: transactionDurationCorrelationValue,
+        ksTest: KsTestLess,
+        ranges: latencyRangesBuckets,
+      });
+      expect(esClientSearchMock).toHaveBeenCalledTimes(1);
+    });
+  });
+});

x-pack/plugins/apm/server/lib/search_strategies/correlations/query_correlation.ts

@@ -26,7 +26,7 @@ interface ResponseHit {
   _source: ResponseHitSource;
 }
 
-interface BucketCorrelation {
+export interface BucketCorrelation {
   buckets_path: string;
   function: {
     count_correlation: {
@@ -80,8 +80,7 @@ export const getTransactionDurationCorrelationRequest = (
       // KS test p value = ks_test.less
       ks_test: {
         bucket_count_ks_test: {
-          // Remove 0 after https://github.com/elastic/elasticsearch/pull/74624 is merged
-          fractions: [0, ...fractions],
+          fractions,
           buckets_path: 'latency_ranges>_count',
           alternative: ['less', 'greater', 'two_sided'],
         },

x-pack/plugins/apm/server/lib/search_strategies/correlations/query_field_candidates.test.ts

@@ -0,0 +1,145 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { estypes } from '@elastic/elasticsearch';
+
+import type { ElasticsearchClient } from 'src/core/server';
+
+import {
+  fetchTransactionDurationFieldCandidates,
+  getRandomDocsRequest,
+  hasPrefixToInclude,
+  shouldBeExcluded,
+} from './query_field_candidates';
+
+const params = { index: 'apm-*' };
+
+describe('query_field_candidates', () => {
+  describe('shouldBeExcluded', () => {
+    it('does not exclude a completely custom field name', () => {
+      expect(shouldBeExcluded('myFieldName')).toBe(false);
+    });
+
+    it(`excludes a field if it's one of FIELDS_TO_EXCLUDE_AS_CANDIDATE`, () => {
+      expect(shouldBeExcluded('transaction.type')).toBe(true);
+    });
+
+    it(`excludes a field if it's prefixed with one of FIELD_PREFIX_TO_EXCLUDE_AS_CANDIDATE`, () => {
+      expect(shouldBeExcluded('observer.myFieldName')).toBe(true);
+    });
+  });
+
+  describe('hasPrefixToInclude', () => {
+    it('identifies if a field name is prefixed to be included', () => {
+      expect(hasPrefixToInclude('myFieldName')).toBe(false);
+      expect(hasPrefixToInclude('somePrefix.myFieldName')).toBe(false);
+      expect(hasPrefixToInclude('cloud.myFieldName')).toBe(true);
+      expect(hasPrefixToInclude('labels.myFieldName')).toBe(true);
+      expect(hasPrefixToInclude('user_agent.myFieldName')).toBe(true);
+    });
+  });
+
+  describe('getRandomDocsRequest', () => {
+    it('returns the most basic request body for a sample of random documents', () => {
+      const req = getRandomDocsRequest(params);
+
+      expect(req).toEqual({
+        body: {
+          _source: false,
+          fields: ['*'],
+          query: {
+            function_score: {
+              query: {
+                bool: {
+                  filter: [
+                    {
+                      term: {
+                        'processor.event': 'transaction',
+                      },
+                    },
+                  ],
+                },
+              },
+              random_score: {},
+            },
+          },
+          size: 1000,
+        },
+        index: params.index,
+      });
+    });
+  });
+
+  describe('fetchTransactionDurationFieldCandidates', () => {
+    it('returns field candidates and total hits', async () => {
+      const esClientFieldCapsMock = jest.fn(() => ({
+        body: {
+          fields: {
+            myIpFieldName: { ip: {} },
+            myKeywordFieldName: { keyword: {} },
+            myUnpopulatedKeywordFieldName: { keyword: {} },
+            myNumericFieldName: { number: {} },
+          },
+        },
+      }));
+      const esClientSearchMock = jest.fn((req: estypes.SearchRequest): {
+        body: estypes.SearchResponse;
+      } => {
+        return {
+          body: ({
+            hits: {
+              hits: [
+                {
+                  fields: {
+                    myIpFieldName: '1.1.1.1',
+                    myKeywordFieldName: 'myKeywordFieldValue',
+                    myNumericFieldName: 1234,
+                  },
+                },
+              ],
+            },
+          } as unknown) as estypes.SearchResponse,
+        };
+      });
+
+      const esClientMock = ({
+        fieldCaps: esClientFieldCapsMock,
+        search: esClientSearchMock,
+      } as unknown) as ElasticsearchClient;
+
+      const resp = await fetchTransactionDurationFieldCandidates(
+        esClientMock,
+        params
+      );
+
+      expect(resp).toEqual({
+        fieldCandidates: [
+          // default field candidates
+          'service.version',
+          'service.node.name',
+          'service.framework.version',
+          'service.language.version',
+          'service.runtime.version',
+          'kubernetes.pod.name',
+          'kubernetes.pod.uid',
+          'container.id',
+          'source.ip',
+          'client.ip',
+          'host.ip',
+          'service.environment',
+          'process.args',
+          'http.response.status_code',
+          // field candidates identified by sample documents
+          'myIpFieldName',
+          'myKeywordFieldName',
+        ],
+      });
+      expect(esClientFieldCapsMock).toHaveBeenCalledTimes(1);
+      expect(esClientSearchMock).toHaveBeenCalledTimes(1);
+    });
+  });
+});