[ML] AIOps: Adds dip support for log rate analysis in observability a…

…lert details page. (elastic#162476) Adds support for analysing dips for log rate analysis on Observability's alert details page. - This removes the filter against `Comparator.GT/Comparator.GT_OR_EQ` which hides log rate analysis for log threshold alerts. Instead an `analysisType` is passed on to log rate analysis: When analysing dips the baseline and deviation ranges will be switched for both the analysis and the brush labels on the document count chart. - Updates all references to "spikes" only in UI text, e.g. the AI Assistant prompt.
walterra · Aug 1, 2023 · 0dabaca · 0dabaca
1 parent ad33d7a
commit 0dabaca
Show file tree

Hide file tree

Showing 21 changed files with 137 additions and 59 deletions.
diff --git a/x-pack/packages/ml/aiops_components/src/document_count_chart/document_count_chart.tsx b/x-pack/packages/ml/aiops_components/src/document_count_chart/document_count_chart.tsx
@@ -20,23 +20,22 @@ import {
   XYChartElementEvent,
   XYBrushEvent,
 } from '@elastic/charts';
+import {
+  BarStyleAccessor,
+  RectAnnotationSpec,
+} from '@elastic/charts/dist/chart_types/xy_chart/utils/specs';
 
 import { i18n } from '@kbn/i18n';
 import { IUiSettingsClient } from '@kbn/core/public';
 import { getSnappedWindowParameters, getWindowParameters } from '@kbn/aiops-utils';
 import type { WindowParameters } from '@kbn/aiops-utils';
 import { MULTILAYER_TIME_AXIS_STYLE } from '@kbn/charts-plugin/common';
 
-import {
-  BarStyleAccessor,
-  RectAnnotationSpec,
-} from '@elastic/charts/dist/chart_types/xy_chart/utils/specs';
-
 import type { DataPublicPluginStart } from '@kbn/data-plugin/public';
 import type { ChartsPluginStart } from '@kbn/charts-plugin/public';
 import type { FieldFormatsStart } from '@kbn/field-formats-plugin/public';
-import { BrushBadge } from './brush_badge';
 import { DualBrush, DualBrushAnnotation } from '../..';
+import { BrushBadge } from './brush_badge';
 
 declare global {
   interface Window {

diff --git a/x-pack/packages/ml/aiops_utils/src/get_window_parameters.ts b/x-pack/packages/ml/aiops_utils/src/get_window_parameters.ts
@@ -8,7 +8,7 @@
 import { isPopulatedObject } from '@kbn/ml-is-populated-object';
 
 /**
- * Time range definition for baseline and deviation to be used by spike log analysis.
+ * Time range definition for baseline and deviation to be used by log rate analysis.
  *
  * @export
  * @interface WindowParameters
@@ -54,12 +54,12 @@ export const isWindowParameters = (arg: unknown): arg is WindowParameters =>
  * 2. The historical time window prior to the click to use as a baseline.
  *
  * The philosophy here is that charts are displayed with different granularities according to their
- * overall time window. We select the log spike and historical time windows inline with the
+ * overall time window. We select the log deviation and historical time windows inline with the
  * overall time window.
  *
  * The algorithm for doing this is based on the typical granularities that exist in machine data.
  *
- * @param clickTime timestamp of the clicked log rate spike.
+ * @param clickTime timestamp of the clicked log rate deviation.
  * @param minTime minimum timestamp of the time window to be analysed
  * @param maxTime maximum timestamp of the time window to be analysed
  * @returns WindowParameters
@@ -103,7 +103,7 @@ export const getWindowParameters = (
  * Converts window paramaters from the brushes to “snap” the brushes to the chart histogram bar width and ensure timestamps
  * correspond to bucket timestamps
  *
- * @param windowParameters time range definition for baseline and deviation to be used by spike log analysis
+ * @param windowParameters time range definition for baseline and deviation to be used by log rate analysis
  * @param snapTimestamps time range definition that always corresponds to histogram bucket timestamps
  * @returns WindowParameters
  */

diff --git a/x-pack/plugins/aiops/common/api/log_rate_analysis/schema.ts b/x-pack/plugins/aiops/common/api/log_rate_analysis/schema.ts
@@ -19,7 +19,7 @@ export const aiopsLogRateAnalysisSchema = schema.object({
   baselineMax: schema.number(),
   deviationMin: schema.number(),
   deviationMax: schema.number(),
-  /** The index to query for log rate spikes */
+  /** The index to query for log rate analysis */
   index: schema.string(),
   /** Settings to override headers derived compression and flush fix */
   compressResponse: schema.maybe(schema.boolean()),

diff --git a/x-pack/plugins/aiops/common/constants.ts b/x-pack/plugins/aiops/common/constants.ts
@@ -5,7 +5,14 @@
  * 2.0.
  */
 
-export const SPIKE_ANALYSIS_THRESHOLD = 0.02;
+export const LOG_RATE_ANALYSIS_P_VALUE_THRESHOLD = 0.02;
+
+export const LOG_RATE_ANALYSIS_TYPE = {
+  SPIKE: 'spike',
+  DIP: 'dip',
+} as const;
+export type LogRateAnalysisType =
+  typeof LOG_RATE_ANALYSIS_TYPE[keyof typeof LOG_RATE_ANALYSIS_TYPE];
 
 // For the technical preview of Log Rate Analysis we use a hard coded seed.
 // In future versions we might use a user specific seed or let the user costumise it.

diff --git a/...ublic/components/document_count_content/document_count_content/document_count_content.tsx b/...ublic/components/document_count_content/document_count_content/document_count_content.tsx
@@ -8,14 +8,14 @@
 import React, { type FC } from 'react';
 
 import { EuiFlexGroup, EuiFlexItem } from '@elastic/eui';
-
-import type { WindowParameters } from '@kbn/aiops-utils';
-
 import {
   BarStyleAccessor,
   RectAnnotationSpec,
 } from '@elastic/charts/dist/chart_types/xy_chart/utils/specs';
+
+import type { WindowParameters } from '@kbn/aiops-utils';
 import { DocumentCountChart, type DocumentCountChartPoint } from '@kbn/aiops-components';
+
 import { useAiopsAppContext } from '../../../hooks/use_aiops_app_context';
 import { DocumentCountStats } from '../../../get_document_stats';
 

diff --git a/...blic/components/log_rate_analysis/log_rate_analysis_content/log_rate_analysis_content.tsx b/...blic/components/log_rate_analysis/log_rate_analysis_content/log_rate_analysis_content.tsx
@@ -18,6 +18,8 @@ import type { Dictionary } from '@kbn/ml-url-state';
 import type { WindowParameters } from '@kbn/aiops-utils';
 import type { SignificantTerm } from '@kbn/ml-agg-utils';
 
+import { LOG_RATE_ANALYSIS_TYPE, type LogRateAnalysisType } from '../../../../common/constants';
+
 import { useData } from '../../../hooks/use_data';
 
 import { DocumentCountContent } from '../../document_count_content/document_count_content';
@@ -46,6 +48,8 @@ export function getDocumentCountStatsSplitLabel(
 export interface LogRateAnalysisContentProps {
   /** The data view to analyze. */
   dataView: DataView;
+  /** The type of analysis, whether it's a spike or dip */
+  analysisType?: LogRateAnalysisType;
   setGlobalState?: (params: Dictionary<unknown>) => void;
   /** Timestamp for the start of the range for initial analysis */
   initialAnalysisStart?: number | WindowParameters;
@@ -64,6 +68,7 @@ export interface LogRateAnalysisContentProps {
 
 export const LogRateAnalysisContent: FC<LogRateAnalysisContentProps> = ({
   dataView,
+  analysisType = LOG_RATE_ANALYSIS_TYPE.SPIKE,
   setGlobalState,
   initialAnalysisStart: incomingInitialAnalysisStart,
   timeRange,
@@ -94,7 +99,7 @@ export const LogRateAnalysisContent: FC<LogRateAnalysisContentProps> = ({
 
   const { documentStats, earliest, latest } = useData(
     dataView,
-    'explain_log_rage_spikes',
+    'log_rate_analysis',
     esSearchQuery,
     setGlobalState,
     currentSelectedSignificantTerm,
@@ -148,6 +153,7 @@ export const LogRateAnalysisContent: FC<LogRateAnalysisContentProps> = ({
       {earliest !== undefined && latest !== undefined && windowParameters !== undefined && (
         <LogRateAnalysisResults
           dataView={dataView}
+          analysisType={analysisType}
           earliest={earliest}
           isBrushCleared={isBrushCleared}
           latest={latest}
@@ -171,7 +177,7 @@ export const LogRateAnalysisContent: FC<LogRateAnalysisContentProps> = ({
             <h2>
               <FormattedMessage
                 id="xpack.aiops.logRateAnalysis.page.emptyPromptTitle"
-                defaultMessage="Click a spike in the histogram chart to start the analysis."
+                defaultMessage="Click a spike or dip in the histogram chart to start the analysis."
               />
             </h2>
           }
@@ -180,7 +186,7 @@ export const LogRateAnalysisContent: FC<LogRateAnalysisContentProps> = ({
             <p>
               <FormattedMessage
                 id="xpack.aiops.logRateAnalysis.page.emptyPromptBody"
-                defaultMessage="The log rate analysis feature identifies statistically significant field/value combinations that contribute to a log rate spike or drop."
+                defaultMessage="The log rate analysis feature identifies statistically significant field/value combinations that contribute to a log rate spike or dip."
               />
             </p>
           }

diff --git a/...ponents/log_rate_analysis/log_rate_analysis_content/log_rate_analysis_content_wrapper.tsx b/...ponents/log_rate_analysis/log_rate_analysis_content/log_rate_analysis_content_wrapper.tsx
@@ -20,6 +20,7 @@ import { DatePickerContextProvider } from '@kbn/ml-date-picker';
 import { UI_SETTINGS } from '@kbn/data-plugin/common';
 import { toMountPoint, wrapWithTheme } from '@kbn/kibana-react-plugin/public';
 
+import { LOG_RATE_ANALYSIS_TYPE, type LogRateAnalysisType } from '../../../../common/constants';
 import { timeSeriesDataViewWarning } from '../../../application/utils/time_series_dataview_check';
 import { AiopsAppContext, type AiopsAppDependencies } from '../../../hooks/use_aiops_app_context';
 import { DataSourceContext } from '../../../hooks/use_data_source';
@@ -34,6 +35,8 @@ const localStorage = new Storage(window.localStorage);
 export interface LogRateAnalysisContentWrapperProps {
   /** The data view to analyze. */
   dataView: DataView;
+  /** The type of analysis, whether it's a spike or dip */
+  analysisType?: LogRateAnalysisType;
   /** Option to make main histogram sticky */
   stickyHistogram?: boolean;
   /** App dependencies */
@@ -55,6 +58,7 @@ export interface LogRateAnalysisContentWrapperProps {
 
 export const LogRateAnalysisContentWrapper: FC<LogRateAnalysisContentWrapperProps> = ({
   dataView,
+  analysisType = LOG_RATE_ANALYSIS_TYPE.SPIKE,
   appDependencies,
   setGlobalState,
   initialAnalysisStart,
@@ -89,6 +93,7 @@ export const LogRateAnalysisContentWrapper: FC<LogRateAnalysisContentWrapperProp
               <DatePickerContextProvider {...datePickerDeps}>
                 <LogRateAnalysisContent
                   dataView={dataView}
+                  analysisType={analysisType}
                   setGlobalState={setGlobalState}
                   initialAnalysisStart={initialAnalysisStart}
                   timeRange={timeRange}

diff --git a/x-pack/plugins/aiops/public/components/log_rate_analysis/log_rate_analysis_page.tsx b/x-pack/plugins/aiops/public/components/log_rate_analysis/log_rate_analysis_page.tsx
@@ -84,7 +84,7 @@ export const LogRateAnalysisPage: FC<Props> = ({ stickyHistogram }) => {
 
   const { timefilter } = useData(
     dataView,
-    'explain_log_rage_spikes',
+    'log_rate_analysis',
     searchQuery,
     setGlobalState,
     currentSelectedSignificantTerm,

diff --git a/x-pack/plugins/aiops/public/components/log_rate_analysis/log_rate_analysis_results.tsx b/x-pack/plugins/aiops/public/components/log_rate_analysis/log_rate_analysis_results.tsx
@@ -30,6 +30,7 @@ import { FormattedMessage } from '@kbn/i18n-react';
 import type { SignificantTerm, SignificantTermGroup } from '@kbn/ml-agg-utils';
 
 import { useAiopsAppContext } from '../../hooks/use_aiops_app_context';
+import { LOG_RATE_ANALYSIS_TYPE, type LogRateAnalysisType } from '../../../common/constants';
 import { initialState, streamReducer } from '../../../common/api/stream_reducer';
 import type { AiopsApiLogRateAnalysis } from '../../../common/api';
 import {
@@ -79,6 +80,8 @@ export interface LogRateAnalysisResultsData {
 interface LogRateAnalysisResultsProps {
   /** The data view to analyze. */
   dataView: DataView;
+  /** The type of analysis, whether it's a spike or dip */
+  analysisType?: LogRateAnalysisType;
   /** Start timestamp filter */
   earliest: number;
   /** End timestamp filter */
@@ -104,6 +107,7 @@ interface LogRateAnalysisResultsProps {
 
 export const LogRateAnalysisResults: FC<LogRateAnalysisResultsProps> = ({
   dataView,
+  analysisType = LOG_RATE_ANALYSIS_TYPE.SPIKE,
   earliest,
   isBrushCleared,
   latest,
@@ -170,7 +174,16 @@ export const LogRateAnalysisResults: FC<LogRateAnalysisResultsProps> = ({
       index: dataView.getIndexPattern(),
       grouping: true,
       flushFix: true,
-      ...windowParameters,
+      // If analysis type is `spike`, pass on window parameters as is,
+      // if it's `dip`, swap baseline and deviation.
+      ...(analysisType === LOG_RATE_ANALYSIS_TYPE.SPIKE
+        ? windowParameters
+        : {
+            baselineMin: windowParameters.deviationMin,
+            baselineMax: windowParameters.deviationMax,
+            deviationMin: windowParameters.baselineMin,
+            deviationMax: windowParameters.baselineMax,
+          }),
       overrides,
       sampleProbability,
     },
@@ -384,7 +397,7 @@ export const LogRateAnalysisResults: FC<LogRateAnalysisResultsProps> = ({
             <p>
               <FormattedMessage
                 id="xpack.aiops.logRateAnalysis.page.noResultsPromptBody"
-                defaultMessage="Try to adjust the baseline and deviation time ranges and rerun the analysis. If you still get no results, there might be no statistically significant entities contributing to this spike in log rates."
+                defaultMessage="Try to adjust the baseline and deviation time ranges and rerun the analysis. If you still get no results, there might be no statistically significant entities contributing to this deviation in log rate."
               />
             </p>
           }

diff --git a/x-pack/plugins/aiops/public/index.ts b/x-pack/plugins/aiops/public/index.ts
@@ -13,6 +13,8 @@ export function plugin() {
   return new AiopsPlugin();
 }
 
+export { LOG_RATE_ANALYSIS_TYPE, type LogRateAnalysisType } from '../common/constants';
+
 export type { AiopsAppDependencies } from './hooks/use_aiops_app_context';
 export type { LogRateAnalysisAppStateProps } from './components/log_rate_analysis';
 export type { LogCategorizationAppStateProps } from './components/log_categorization';

diff --git a/x-pack/plugins/aiops/server/routes/queries/fetch_significant_term_p_values.ts b/x-pack/plugins/aiops/server/routes/queries/fetch_significant_term_p_values.ts
@@ -15,7 +15,10 @@ import {
   type RandomSamplerWrapper,
 } from '@kbn/ml-random-sampler-utils';
 
-import { SPIKE_ANALYSIS_THRESHOLD, RANDOM_SAMPLER_SEED } from '../../../common/constants';
+import {
+  LOG_RATE_ANALYSIS_P_VALUE_THRESHOLD,
+  RANDOM_SAMPLER_SEED,
+} from '../../../common/constants';
 import type { AiopsLogRateAnalysisSchema } from '../../../common/api/log_rate_analysis';
 
 import { isRequestAbortedError } from '../../lib/is_request_aborted_error';
@@ -171,7 +174,7 @@ export const fetchSignificantTermPValues = async (
         0.25 * Math.min(Math.max((bucket.score - 6.908) / 6.908, 0), 1) +
         0.25 * Math.min(Math.max((bucket.score - 13.816) / 101.314, 0), 1);
 
-      if (typeof pValue === 'number' && pValue < SPIKE_ANALYSIS_THRESHOLD) {
+      if (typeof pValue === 'number' && pValue < LOG_RATE_ANALYSIS_P_VALUE_THRESHOLD) {
         result.push({
           fieldName,
           fieldValue: String(bucket.key),