[Response Ops][Alerting] Adding evaluation threshold to alert payload…

… for ES query rule (#171571)

Resolves #166986

## Summary

Adding `kibana.alert.evalution.threshold` to the alert payload for the
ES query rule. This is the field that's shown in the alert details view
in Observability. To show this, we add `ALERT_EVALUATION_CONDITIONS` to
the stack alerts mapping, using the same mapping type as the
observability rule types. This is typed as a `scaled_float` which is
expecting a single value, so the threshold is set in the alert payload
only when the threshold is a single value. I will open a followup issue
for handling multi-valued thresholds.
#172714

<img width="1064" alt="Screenshot 2023-11-20 at 1 10 05 PM"
src="https://github.com/elastic/kibana/assets/13104637/e265a9e8-4bbf-4d3e-a6bc-e69b774c7574">


## To Verify

Create an ES query rule with a single threshold that triggers an alert
and give it a Metrics or Logs visibility. Let it run and then look at
the alert details for the alert from the Observability alert table. The
`Expected Value` row should be populated.

packages/kbn-alerts-as-data-utils/src/schemas/generated/stack_schema.ts

@@ -70,6 +70,7 @@ const StackAlertRequired = rt.type({
 });
 const StackAlertOptional = rt.partial({
   'kibana.alert.evaluation.conditions': schemaString,
+  'kibana.alert.evaluation.threshold': schemaStringOrNumber,
   'kibana.alert.evaluation.value': schemaString,
   'kibana.alert.title': schemaString,
 });

x-pack/plugins/observability/public/utils/format_alert_evaluation_value.test.ts

@@ -11,6 +11,13 @@ describe('formatAlertEvaluationValue', () => {
   it('returns - when there is no evaluationValue passed', () => {
     expect(formatAlertEvaluationValue('apm.transaction_error_rate', undefined)).toBe('-');
   });
+  it('returns - when there is null evaluationValue passed', () => {
+    // @ts-expect-error
+    expect(formatAlertEvaluationValue('apm.transaction_error_rate', null)).toBe('-');
+  });
+  it('returns the evaluation value when the value is 0', () => {
+    expect(formatAlertEvaluationValue('.es-query', 0)).toBe(0);
+  });
   it('returns the evaluation value when ruleTypeId in unknown aka unformatted', () => {
     expect(formatAlertEvaluationValue('unknown.rule.type', 2000)).toBe(2000);
   });

x-pack/plugins/observability/public/utils/format_alert_evaluation_value.ts

@@ -12,7 +12,7 @@ import {
 } from './get_alert_evaluation_unit_type_by_rule_type_id';
 
 export const formatAlertEvaluationValue = (ruleTypeId?: string, evaluationValue?: number) => {
-  if (!evaluationValue || !ruleTypeId) return '-';
+  if (null == evaluationValue || !ruleTypeId) return '-';
   const unitType = getAlertEvaluationUnitTypeByRuleTypeId(ruleTypeId);
   switch (unitType) {
     case ALERT_EVALUATION_UNIT_TYPE.DURATION:

x-pack/plugins/stack_alerts/server/rule_types/constants.ts

@@ -5,23 +5,28 @@
  * 2.0.
  */
 import { IRuleTypeAlerts } from '@kbn/alerting-plugin/server';
-import { StackAlert } from '@kbn/alerts-as-data-utils';
-import { ALERT_EVALUATION_VALUE } from '@kbn/rule-data-utils';
+import { ALERT_EVALUATION_THRESHOLD, ALERT_EVALUATION_VALUE } from '@kbn/rule-data-utils';
 import { ALERT_NAMESPACE } from '@kbn/rule-data-utils';
+import { StackAlertType } from './types';
 
 export const STACK_AAD_INDEX_NAME = 'stack';
 
 export const ALERT_TITLE = `${ALERT_NAMESPACE}.title` as const;
 // kibana.alert.evaluation.conditions - human readable string that shows the conditions set by the user
 export const ALERT_EVALUATION_CONDITIONS = `${ALERT_NAMESPACE}.evaluation.conditions` as const;
 
-export const STACK_ALERTS_AAD_CONFIG: IRuleTypeAlerts<StackAlert> = {
+export const STACK_ALERTS_AAD_CONFIG: IRuleTypeAlerts<StackAlertType> = {
   context: STACK_AAD_INDEX_NAME,
   mappings: {
     fieldMap: {
       [ALERT_TITLE]: { type: 'keyword', array: false, required: false },
       [ALERT_EVALUATION_CONDITIONS]: { type: 'keyword', array: false, required: false },
       [ALERT_EVALUATION_VALUE]: { type: 'keyword', array: false, required: false },
+      [ALERT_EVALUATION_THRESHOLD]: {
+        type: 'scaled_float',
+        scaling_factor: 100,
+        required: false,
+      },
     },
   },
   shouldWrite: true,

x-pack/plugins/stack_alerts/server/rule_types/es_query/executor.test.ts

@@ -299,6 +299,7 @@ describe('es_query executor', () => {
         payload: {
           'kibana.alert.evaluation.conditions':
             'Number of matching documents is greater than or equal to 200',
+          'kibana.alert.evaluation.threshold': 200,
           'kibana.alert.evaluation.value': '491',
           'kibana.alert.reason':
             'Document count is 491 in the last 5m. Alert when greater than or equal to 200.',
@@ -311,6 +312,64 @@ describe('es_query executor', () => {
       expect(mockSetLimitReached).toHaveBeenCalledWith(false);
     });
 
+    it('should create alert if compare function returns true for ungrouped alert for multi threshold param', async () => {
+      mockFetchEsQuery.mockResolvedValueOnce({
+        parsedResults: {
+          results: [
+            {
+              group: 'all documents',
+              count: 491,
+              hits: [],
+            },
+          ],
+          truncated: false,
+        },
+        link: 'https://localhost:5601/app/management/insightsAndAlerting/triggersActions/rule/test-rule-id',
+      });
+      await executor(coreMock, {
+        ...defaultExecutorOptions,
+        // @ts-expect-error
+        params: {
+          ...defaultProps,
+          threshold: [200, 500],
+          thresholdComparator: 'between' as Comparator,
+        },
+      });
+
+      expect(mockReport).toHaveBeenCalledTimes(1);
+      expect(mockReport).toHaveBeenNthCalledWith(1, {
+        actionGroup: 'query matched',
+        context: {
+          conditions: 'Number of matching documents is between 200 and 500',
+          date: new Date(mockNow).toISOString(),
+          hits: [],
+          link: 'https://localhost:5601/app/management/insightsAndAlerting/triggersActions/rule/test-rule-id',
+          message: 'Document count is 491 in the last 5m. Alert when between 200 and 500.',
+          title: "rule 'test-rule-name' matched query",
+          value: 491,
+        },
+        id: 'query matched',
+        state: {
+          dateEnd: new Date(mockNow).toISOString(),
+          dateStart: new Date(mockNow).toISOString(),
+          latestTimestamp: undefined,
+        },
+        payload: {
+          'kibana.alert.evaluation.conditions':
+            'Number of matching documents is between 200 and 500',
+          'kibana.alert.evaluation.threshold': null,
+          'kibana.alert.evaluation.value': '491',
+          'kibana.alert.reason':
+            'Document count is 491 in the last 5m. Alert when between 200 and 500.',
+          'kibana.alert.title': "rule 'test-rule-name' matched query",
+          'kibana.alert.url':
+            'https://localhost:5601/app/management/insightsAndAlerting/triggersActions/rule/test-rule-id',
+        },
+      });
+      expect(mockSetLimitReached).toHaveBeenCalledTimes(1);
+      expect(mockSetLimitReached).toHaveBeenCalledWith(false);
+    });
+
     it('should create as many alerts as number of results in parsedResults for grouped alert', async () => {
       mockFetchEsQuery.mockResolvedValueOnce({
         parsedResults: {
@@ -371,6 +430,7 @@ describe('es_query executor', () => {
         payload: {
           'kibana.alert.evaluation.conditions':
             'Number of matching documents for group "host-1" is greater than or equal to 200',
+          'kibana.alert.evaluation.threshold': 200,
           'kibana.alert.evaluation.value': '291',
           'kibana.alert.reason':
             'Document count is 291 in the last 5m for host-1. Alert when greater than or equal to 200.',
@@ -401,6 +461,7 @@ describe('es_query executor', () => {
         payload: {
           'kibana.alert.evaluation.conditions':
             'Number of matching documents for group "host-2" is greater than or equal to 200',
+          'kibana.alert.evaluation.threshold': 200,
           'kibana.alert.evaluation.value': '477',
           'kibana.alert.reason':
             'Document count is 477 in the last 5m for host-2. Alert when greater than or equal to 200.',
@@ -431,6 +492,7 @@ describe('es_query executor', () => {
         payload: {
           'kibana.alert.evaluation.conditions':
             'Number of matching documents for group "host-3" is greater than or equal to 200',
+          'kibana.alert.evaluation.threshold': 200,
           'kibana.alert.evaluation.value': '999',
           'kibana.alert.reason':
             'Document count is 999 in the last 5m for host-3. Alert when greater than or equal to 200.',
@@ -482,6 +544,7 @@ describe('es_query executor', () => {
         id: 'query matched',
         payload: {
           'kibana.alert.evaluation.conditions': 'Query matched documents',
+          'kibana.alert.evaluation.threshold': 0,
           'kibana.alert.evaluation.value': '198',
           'kibana.alert.reason':
             'Document count is 198 in the last 5m. Alert when greater than or equal to 0.',
@@ -586,6 +649,7 @@ describe('es_query executor', () => {
         payload: {
           'kibana.alert.evaluation.conditions':
             'Number of matching documents is NOT greater than or equal to 500',
+          'kibana.alert.evaluation.threshold': 500,
           'kibana.alert.evaluation.value': '0',
           'kibana.alert.reason':
             'Document count is 0 in the last 5m. Alert when greater than or equal to 500.',
@@ -645,6 +709,7 @@ describe('es_query executor', () => {
         payload: {
           'kibana.alert.evaluation.conditions':
             'Number of matching documents for group "host-1" is NOT greater than or equal to 200',
+          'kibana.alert.evaluation.threshold': 200,
           'kibana.alert.evaluation.value': '0',
           'kibana.alert.reason':
             'Document count is 0 in the last 5m for host-1. Alert when greater than or equal to 200.',
@@ -668,6 +733,7 @@ describe('es_query executor', () => {
         payload: {
           'kibana.alert.evaluation.conditions':
             'Number of matching documents for group "host-2" is NOT greater than or equal to 200',
+          'kibana.alert.evaluation.threshold': 200,
           'kibana.alert.evaluation.value': '0',
           'kibana.alert.reason':
             'Document count is 0 in the last 5m for host-2. Alert when greater than or equal to 200.',
@@ -720,6 +786,7 @@ describe('es_query executor', () => {
         },
         payload: {
           'kibana.alert.evaluation.conditions': 'Query did NOT match documents',
+          'kibana.alert.evaluation.threshold': 0,
           'kibana.alert.evaluation.value': '0',
           'kibana.alert.reason': 'Document count is 0 in the last 5m. Alert when greater than 0.',
           'kibana.alert.title': "rule 'test-rule-name' recovered",

x-pack/plugins/stack_alerts/server/rule_types/es_query/executor.ts

@@ -8,7 +8,12 @@ import { sha256 } from 'js-sha256';
 import { i18n } from '@kbn/i18n';
 import { CoreSetup } from '@kbn/core/server';
 import { isGroupAggregation, UngroupedGroupId } from '@kbn/triggers-actions-ui-plugin/common';
-import { ALERT_EVALUATION_VALUE, ALERT_REASON, ALERT_URL } from '@kbn/rule-data-utils';
+import {
+  ALERT_EVALUATION_THRESHOLD,
+  ALERT_EVALUATION_VALUE,
+  ALERT_REASON,
+  ALERT_URL,
+} from '@kbn/rule-data-utils';
 
 import { ComparatorFns } from '../../../common';
 import {
@@ -161,6 +166,7 @@ export async function executor(core: CoreSetup, options: ExecutorOptions<EsQuery
         [ALERT_TITLE]: actionContext.title,
         [ALERT_EVALUATION_CONDITIONS]: actionContext.conditions,
         [ALERT_EVALUATION_VALUE]: `${actionContext.value}`,
+        [ALERT_EVALUATION_THRESHOLD]: params.threshold?.length === 1 ? params.threshold[0] : null,
       },
     });
     if (!isGroupAgg) {
@@ -211,6 +217,7 @@ export async function executor(core: CoreSetup, options: ExecutorOptions<EsQuery
         [ALERT_TITLE]: recoveryContext.title,
         [ALERT_EVALUATION_CONDITIONS]: recoveryContext.conditions,
         [ALERT_EVALUATION_VALUE]: `${recoveryContext.value}`,
+        [ALERT_EVALUATION_THRESHOLD]: params.threshold?.length === 1 ? params.threshold[0] : null,
       },
     });
   }

x-pack/plugins/stack_alerts/server/rule_types/es_query/rule_type.test.ts

@@ -703,6 +703,7 @@ describe('ruleType', () => {
           payload: expect.objectContaining({
             'kibana.alert.evaluation.conditions':
               'Number of matching documents is greater than or equal to 3',
+            'kibana.alert.evaluation.threshold': 3,
             'kibana.alert.evaluation.value': '3',
             'kibana.alert.reason': expect.any(String),
             'kibana.alert.title': "rule 'rule-name' matched query",
@@ -797,6 +798,7 @@ describe('ruleType', () => {
           id: 'query matched',
           payload: expect.objectContaining({
             'kibana.alert.evaluation.conditions': 'Query matched documents',
+            'kibana.alert.evaluation.threshold': 0,
             'kibana.alert.evaluation.value': '3',
             'kibana.alert.reason': expect.any(String),
             'kibana.alert.title': "rule 'rule-name' matched query",

x-pack/plugins/stack_alerts/server/rule_types/es_query/rule_type.ts

@@ -9,7 +9,6 @@ import { i18n } from '@kbn/i18n';
 import { CoreSetup, DEFAULT_APP_CATEGORIES } from '@kbn/core/server';
 import { extractReferences, injectReferences } from '@kbn/data-plugin/common';
 import { ES_QUERY_ID, STACK_ALERTS_FEATURE_ID } from '@kbn/rule-data-utils';
-import { StackAlert } from '@kbn/alerts-as-data-utils';
 import { STACK_ALERTS_AAD_CONFIG } from '..';
 import { RuleType } from '../../types';
 import { ActionContext } from './action_context';
@@ -23,6 +22,7 @@ import { ExecutorOptions } from './types';
 import { ActionGroupId } from './constants';
 import { executor } from './executor';
 import { isSearchSourceRule } from './util';
+import { StackAlertType } from '../types';
 
 export function getRuleType(
   core: CoreSetup
@@ -34,7 +34,7 @@ export function getRuleType(
   ActionContext,
   typeof ActionGroupId,
   never,
-  StackAlert
+  StackAlertType
 > {
   const ruleTypeName = i18n.translate('xpack.stackAlerts.esQuery.alertTypeTitle', {
     defaultMessage: 'Elasticsearch query',

x-pack/plugins/stack_alerts/server/rule_types/es_query/types.ts

@@ -5,11 +5,11 @@
  * 2.0.
  */
 
-import { StackAlert } from '@kbn/alerts-as-data-utils';
 import { RuleExecutorOptions, RuleTypeParams } from '../../types';
 import { ActionContext } from './action_context';
 import { EsQueryRuleParams, EsQueryRuleState } from './rule_type_params';
 import { ActionGroupId } from './constants';
+import { StackAlertType } from '../types';
 
 export type OnlyEsQueryRuleParams = Omit<EsQueryRuleParams, 'searchConfiguration' | 'esqlQuery'> & {
   searchType: 'esQuery';
@@ -37,5 +37,5 @@ export type ExecutorOptions<P extends RuleTypeParams> = RuleExecutorOptions<
   {},
   ActionContext,
   typeof ActionGroupId,
-  StackAlert
+  StackAlertType
 >;

x-pack/plugins/stack_alerts/server/rule_types/index_threshold/rule_type.ts

@@ -12,7 +12,6 @@ import {
   TIME_SERIES_BUCKET_SELECTOR_FIELD,
 } from '@kbn/triggers-actions-ui-plugin/server';
 import { isGroupAggregation } from '@kbn/triggers-actions-ui-plugin/common';
-import { StackAlert } from '@kbn/alerts-as-data-utils';
 import {
   ALERT_EVALUATION_VALUE,
   ALERT_REASON,
@@ -23,13 +22,14 @@ import { ComparatorFns, getComparatorScript, getHumanReadableComparator } from '
 import { ActionContext, BaseActionContext, addMessages } from './action_context';
 import { Params, ParamsSchema } from './rule_type_params';
 import { RuleType, RuleExecutorOptions, StackAlertsStartDeps } from '../../types';
+import { StackAlertType } from '../types';
 
 export const ID = '.index-threshold';
 export const ActionGroupId = 'threshold met';
 
 export function getRuleType(
   data: Promise<StackAlertsStartDeps['triggersActionsUi']['data']>
-): RuleType<Params, never, {}, {}, ActionContext, typeof ActionGroupId, never, StackAlert> {
+): RuleType<Params, never, {}, {}, ActionContext, typeof ActionGroupId, never, StackAlertType> {
   const ruleTypeName = i18n.translate('xpack.stackAlerts.indexThreshold.alertTypeTitle', {
     defaultMessage: 'Index threshold',
   });
@@ -212,7 +212,14 @@ export function getRuleType(
   };
 
   async function executor(
-    options: RuleExecutorOptions<Params, {}, {}, ActionContext, typeof ActionGroupId, StackAlert>
+    options: RuleExecutorOptions<
+      Params,
+      {},
+      {},
+      ActionContext,
+      typeof ActionGroupId,
+      StackAlertType
+    >
   ) {
     const {
       rule: { id: ruleId, name },

x-pack/plugins/stack_alerts/server/rule_types/types.ts

@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+import { StackAlert } from '@kbn/alerts-as-data-utils';
 import { CoreSetup, Logger } from '@kbn/core/server';
 import { AlertingSetup, StackAlertsStartDeps } from '../types';
 
@@ -14,3 +15,8 @@ export interface RegisterRuleTypesParams {
   alerting: AlertingSetup;
   core: CoreSetup;
 }
+
+export type StackAlertType = Omit<StackAlert, 'kibana.alert.evaluation.threshold'> & {
+  // Defining a custom type for this because the schema generation script doesn't allow explicit null values
+  'kibana.alert.evaluation.threshold'?: string | number | null;
+};

x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group3/builtin_alert_types/es_query/esql_only.ts

@@ -99,6 +99,7 @@ export default function ruleTests({ getService }: FtrProviderContext) {
       expect(alertDoc[ALERT_REASON]).to.match(messagePattern);
       expect(alertDoc['kibana.alert.title']).to.be("rule 'always fire' matched query");
       expect(alertDoc['kibana.alert.evaluation.conditions']).to.be('Query matched documents');
+      expect(alertDoc['kibana.alert.evaluation.threshold']).to.eql(0);
       const value = parseInt(alertDoc['kibana.alert.evaluation.value'], 10);
       expect(value).greaterThan(0);
       expect(alertDoc[ALERT_URL]).to.contain('/s/space1/app/');

x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group3/builtin_alert_types/es_query/rule.ts

@@ -169,6 +169,7 @@ export default function ruleTests({ getService }: FtrProviderContext) {
         expect(alertDoc['kibana.alert.evaluation.conditions']).to.be(
           'Number of matching documents is greater than -1'
         );
+        expect(alertDoc['kibana.alert.evaluation.threshold']).to.eql(-1);
         const value = parseInt(alertDoc['kibana.alert.evaluation.value'], 10);
         expect(value).greaterThan(0);
         expect(alertDoc[ALERT_URL]).to.contain('/s/space1/app/');

x-pack/test_serverless/api_integration/test_suites/common/alerting/alert_documents.ts

@@ -141,6 +141,7 @@ export default function ({ getService }: FtrProviderContext) {
         [SPACE_IDS]: ['default'],
         ['kibana.alert.title']: "rule 'always fire' matched query",
         ['kibana.alert.evaluation.conditions']: 'Number of matching documents is greater than -1',
+        ['kibana.alert.evaluation.threshold']: -1,
         ['kibana.alert.evaluation.value']: '0',
         [ALERT_ACTION_GROUP]: 'query matched',
         [ALERT_FLAPPING]: false,

x-pack/test_serverless/api_integration/test_suites/common/alerting/summary_actions.ts

@@ -167,6 +167,7 @@ export default function ({ getService }: FtrProviderContext) {
         [EVENT_KIND]: 'signal',
         ['kibana.alert.title']: "rule 'always fire' matched query",
         ['kibana.alert.evaluation.conditions']: 'Number of matching documents is greater than -1',
+        ['kibana.alert.evaluation.threshold']: -1,
         ['kibana.alert.evaluation.value']: '0',
         [ALERT_ACTION_GROUP]: 'query matched',
         [ALERT_FLAPPING]: false,
@@ -291,6 +292,7 @@ export default function ({ getService }: FtrProviderContext) {
         [EVENT_KIND]: 'signal',
         ['kibana.alert.title']: "rule 'always fire' matched query",
         ['kibana.alert.evaluation.conditions']: 'Number of matching documents is greater than -1',
+        ['kibana.alert.evaluation.threshold']: -1,
         ['kibana.alert.evaluation.value']: '0',
         [ALERT_ACTION_GROUP]: 'query matched',
         [ALERT_FLAPPING]: false,
@@ -507,6 +509,7 @@ export default function ({ getService }: FtrProviderContext) {
         [EVENT_KIND]: 'signal',
         ['kibana.alert.title']: "rule 'always fire' matched query",
         ['kibana.alert.evaluation.conditions']: 'Number of matching documents is greater than -1',
+        ['kibana.alert.evaluation.threshold']: -1,
         ['kibana.alert.evaluation.value']: '0',
         [ALERT_ACTION_GROUP]: 'query matched',
         [ALERT_FLAPPING]: false,