[7.x] [APM] Garbage collection metrics charts (#47023) (#47675)

* [APM] Garbage collection metrics charts

Closes #36320.

* Review feedback

* Display average of delta in gc chart

x-pack/legacy/plugins/apm/common/elasticsearch_fieldnames.ts

@@ -58,6 +58,10 @@ export const METRIC_JAVA_NON_HEAP_MEMORY_COMMITTED =
   'jvm.memory.non_heap.committed';
 export const METRIC_JAVA_NON_HEAP_MEMORY_USED = 'jvm.memory.non_heap.used';
 export const METRIC_JAVA_THREAD_COUNT = 'jvm.thread.count';
+export const METRIC_JAVA_GC_COUNT = 'jvm.gc.count';
+export const METRIC_JAVA_GC_TIME = 'jvm.gc.time';
+
+export const LABEL_NAME = 'labels.name';
 
 export const HOST_NAME = 'host.hostname';
 export const CONTAINER_ID = 'container.id';

x-pack/legacy/plugins/apm/public/components/app/ServiceNodeMetrics/index.tsx

@@ -61,8 +61,7 @@ export function ServiceNodeMetrics() {
     }
   }, [serviceName, serviceNodeName]);
 
-  const isLoading =
-    status === FETCH_STATUS.LOADING || status === FETCH_STATUS.PENDING;
+  const isLoading = status === FETCH_STATUS.LOADING;
 
   return (
     <div>

x-pack/legacy/plugins/apm/public/components/shared/charts/MetricsChart/index.tsx

@@ -12,7 +12,9 @@ import {
   asDynamicBytes,
   asPercent,
   getFixedByteFormatter,
-  asDecimal
+  asDecimal,
+  asTime,
+  asInteger
 } from '../../../../utils/formatters';
 import { Coordinate } from '../../../../../typings/timeseries';
 import { isValidCoordinateValue } from '../../../../utils/isValidCoordinateValue';
@@ -64,6 +66,13 @@ function getYTickFormatter(chart: GenericMetricsChart) {
     case 'percent': {
       return (y: number | null | undefined) => asPercent(y || 0, 1);
     }
+    case 'time': {
+      return (y: number | null | undefined) => asTime(y);
+    }
+    case 'integer': {
+      return (y: number | null | undefined) =>
+        isValidCoordinateValue(y) ? asInteger(y) : y;
+    }
     default: {
       return (y: number | null | undefined) =>
         isValidCoordinateValue(y) ? asDecimal(y) : y;
@@ -79,6 +88,13 @@ function getTooltipFormatter({ yUnit }: GenericMetricsChart) {
     case 'percent': {
       return (c: Coordinate) => asPercent(c.y || 0, 1);
     }
+    case 'time': {
+      return (c: Coordinate) => asTime(c.y);
+    }
+    case 'integer': {
+      return (c: Coordinate) =>
+        isValidCoordinateValue(c.y) ? asInteger(c.y) : c.y;
+    }
     default: {
       return (c: Coordinate) =>
         isValidCoordinateValue(c.y) ? asDecimal(c.y) : c.y;

x-pack/legacy/plugins/apm/server/lib/metrics/by_agent/java/gc/fetchAndTransformGcMetrics.ts

@@ -0,0 +1,159 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { idx } from '@kbn/elastic-idx';
+import { sum } from 'lodash';
+import theme from '@elastic/eui/dist/eui_theme_light.json';
+import { Setup } from '../../../../helpers/setup_request';
+import { getMetricsDateHistogramParams } from '../../../../helpers/metrics';
+import { ChartBase } from '../../../types';
+import { getMetricsProjection } from '../../../../../../common/projections/metrics';
+import { mergeProjection } from '../../../../../../common/projections/util/merge_projection';
+import {
+  SERVICE_AGENT_NAME,
+  LABEL_NAME,
+  METRIC_JAVA_GC_COUNT,
+  METRIC_JAVA_GC_TIME
+} from '../../../../../../common/elasticsearch_fieldnames';
+
+const colors = [
+  theme.euiColorVis0,
+  theme.euiColorVis1,
+  theme.euiColorVis2,
+  theme.euiColorVis3,
+  theme.euiColorVis4,
+  theme.euiColorVis5,
+  theme.euiColorVis6
+];
+
+export async function fetchAndTransformGcMetrics({
+  setup,
+  serviceName,
+  serviceNodeName,
+  chartBase,
+  fieldName
+}: {
+  setup: Setup;
+  serviceName: string;
+  serviceNodeName?: string;
+  chartBase: ChartBase;
+  fieldName: typeof METRIC_JAVA_GC_COUNT | typeof METRIC_JAVA_GC_TIME;
+}) {
+  const { start, end, client } = setup;
+
+  const projection = getMetricsProjection({
+    setup,
+    serviceName,
+    serviceNodeName
+  });
+
+  // GC rate and time are reported by the agents as monotonically
+  // increasing counters, which means that we have to calculate
+  // the delta in an es query. In the future agent might start
+  // reporting deltas.
+  const params = mergeProjection(projection, {
+    body: {
+      size: 0,
+      query: {
+        bool: {
+          filter: [
+            ...projection.body.query.bool.filter,
+            { exists: { field: fieldName } },
+            { term: { [SERVICE_AGENT_NAME]: 'java' } }
+          ]
+        }
+      },
+      aggs: {
+        per_pool: {
+          terms: {
+            field: `${LABEL_NAME}`
+          },
+          aggs: {
+            over_time: {
+              date_histogram: getMetricsDateHistogramParams(start, end),
+              aggs: {
+                // get the max value
+                max: {
+                  max: {
+                    field: fieldName
+                  }
+                },
+                // get the derivative, which is the delta y
+                derivative: {
+                  derivative: {
+                    buckets_path: 'max'
+                  }
+                },
+                // if a gc counter is reset, the delta will be >0 and
+                // needs to be excluded
+                value: {
+                  bucket_script: {
+                    buckets_path: { value: 'derivative' },
+                    script: 'params.value > 0.0 ? params.value : 0.0'
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  });
+
+  const response = await client.search(params);
+
+  const aggregations = idx(response, _ => _.aggregations);
+
+  if (!aggregations) {
+    return {
+      ...chartBase,
+      totalHits: 0,
+      series: []
+    };
+  }
+
+  const series = aggregations.per_pool.buckets.map((poolBucket, i) => {
+    const label = poolBucket.key;
+    const timeseriesData = poolBucket.over_time;
+
+    const data = (idx(timeseriesData, _ => _.buckets) || []).map(bucket => {
+      // derivative/value will be undefined for the first hit and if the `max` value is null
+      const y =
+        'value' in bucket && bucket.value.value !== null
+          ? bucket.value.value
+          : null;
+
+      return {
+        y,
+        x: bucket.key
+      };
+    });
+
+    const values = data.map(coordinate => coordinate.y).filter(y => y !== null);
+
+    const overallValue = sum(values) / values.length;
+
+    return {
+      title: label,
+      key: label,
+      type: chartBase.type,
+      color: colors[i],
+      overallValue,
+      data
+    };
+  });
+
+  return {
+    ...chartBase,
+    totalHits: response.hits.total,
+    series
+  };
+}

x-pack/legacy/plugins/apm/server/lib/metrics/by_agent/java/gc/getGcRateChart.ts

@@ -0,0 +1,47 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import theme from '@elastic/eui/dist/eui_theme_light.json';
+import { i18n } from '@kbn/i18n';
+import { METRIC_JAVA_GC_COUNT } from '../../../../../../common/elasticsearch_fieldnames';
+import { Setup } from '../../../../helpers/setup_request';
+import { fetchAndTransformGcMetrics } from './fetchAndTransformGcMetrics';
+import { ChartBase } from '../../../types';
+
+const series = {
+  [METRIC_JAVA_GC_COUNT]: {
+    title: i18n.translate('xpack.apm.agentMetrics.java.gcRate', {
+      defaultMessage: 'GC count'
+    }),
+    color: theme.euiColorVis0
+  }
+};
+
+const chartBase: ChartBase = {
+  title: i18n.translate('xpack.apm.agentMetrics.java.gcCountChartTitle', {
+    defaultMessage: 'Garbage collection count'
+  }),
+  key: 'gc_count_line_chart',
+  type: 'linemark',
+  yUnit: 'integer',
+  series
+};
+
+const getGcRateChart = (
+  setup: Setup,
+  serviceName: string,
+  serviceNodeName?: string
+) => {
+  return fetchAndTransformGcMetrics({
+    setup,
+    serviceName,
+    serviceNodeName,
+    chartBase,
+    fieldName: METRIC_JAVA_GC_COUNT
+  });
+};
+
+export { getGcRateChart };

x-pack/legacy/plugins/apm/server/lib/metrics/by_agent/java/gc/getGcTimeChart.ts

@@ -0,0 +1,47 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import theme from '@elastic/eui/dist/eui_theme_light.json';
+import { i18n } from '@kbn/i18n';
+import { METRIC_JAVA_GC_TIME } from '../../../../../../common/elasticsearch_fieldnames';
+import { Setup } from '../../../../helpers/setup_request';
+import { fetchAndTransformGcMetrics } from './fetchAndTransformGcMetrics';
+import { ChartBase } from '../../../types';
+
+const series = {
+  [METRIC_JAVA_GC_TIME]: {
+    title: i18n.translate('xpack.apm.agentMetrics.java.gcTime', {
+      defaultMessage: 'GC time'
+    }),
+    color: theme.euiColorVis0
+  }
+};
+
+const chartBase: ChartBase = {
+  title: i18n.translate('xpack.apm.agentMetrics.java.gcTimeChartTitle', {
+    defaultMessage: 'Garbage collection time'
+  }),
+  key: 'gc_time_line_chart',
+  type: 'linemark',
+  yUnit: 'time',
+  series
+};
+
+const getGcTimeChart = (
+  setup: Setup,
+  serviceName: string,
+  serviceNodeName?: string
+) => {
+  return fetchAndTransformGcMetrics({
+    setup,
+    serviceName,
+    serviceNodeName,
+    chartBase,
+    fieldName: METRIC_JAVA_GC_TIME
+  });
+};
+
+export { getGcTimeChart };

x-pack/legacy/plugins/apm/server/lib/metrics/by_agent/java/index.ts

@@ -10,6 +10,8 @@ import { getNonHeapMemoryChart } from './non_heap_memory';
 import { getThreadCountChart } from './thread_count';
 import { getCPUChartData } from '../shared/cpu';
 import { getMemoryChartData } from '../shared/memory';
+import { getGcRateChart } from './gc/getGcRateChart';
+import { getGcTimeChart } from './gc/getGcTimeChart';
 
 export async function getJavaMetricsCharts(
   setup: Setup,
@@ -21,7 +23,9 @@ export async function getJavaMetricsCharts(
     getMemoryChartData(setup, serviceName, serviceNodeName),
     getHeapMemoryChart(setup, serviceName, serviceNodeName),
     getNonHeapMemoryChart(setup, serviceName, serviceNodeName),
-    getThreadCountChart(setup, serviceName, serviceNodeName)
+    getThreadCountChart(setup, serviceName, serviceNodeName),
+    getGcRateChart(setup, serviceName, serviceNodeName),
+    getGcTimeChart(setup, serviceName, serviceNodeName)
   ]);
 
   return { charts };

x-pack/legacy/plugins/apm/typings/elasticsearch.ts

@@ -30,7 +30,9 @@ declare module 'elasticsearch' {
     | 'filters'
     | 'cardinality'
     | 'sampler'
-    | 'value_count';
+    | 'value_count'
+    | 'derivative'
+    | 'bucket_script';
 
   type AggOptions = AggregationOptionMap & {
     [key: string]: any;
@@ -139,6 +141,13 @@ declare module 'elasticsearch' {
           value: number;
         };
         sampler: SamplerAggregation<AggregationOption[AggregationName]>;
+        derivative: BucketAggregation<
+          AggregationOption[AggregationName],
+          number
+        >;
+        bucket_script: {
+          value: number | null;
+        };
       }[AggregationType & keyof AggregationOption[AggregationName]];
     }
   >;