[Alerting] Configurable number of hits for ES query alert (#90089)

* Adding size parameter to ES query alert * Can't use const inside validation * Updating docs * Fixing functional test * License Co-authored-by: Kibana Machine <[email protected]>
elastic · Feb 9, 2021 · 5f8de69 · 5f8de69
1 parent 1f5d52e
commit 5f8de69
Show file tree

Hide file tree

Showing 16 changed files with 382 additions and 8 deletions.
diff --git a/docs/user/alerting/alert-types.asciidoc b/docs/user/alerting/alert-types.asciidoc
@@ -130,12 +130,13 @@ image::images/alert-types-es-query-select.png[Choosing an ES query alert type]
 [float]
 ==== Defining the conditions
 
-The ES query alert has 4 clauses that define the condition to detect.
+The ES query alert has 5 clauses that define the condition to detect.
 
 [role="screenshot"]
 image::images/alert-types-es-query-conditions.png[Four clauses define the condition to detect]
 
 Index:: This clause requires an *index or index pattern* and a *time field* that will be used for the *time window*.
+Size:: This clause specifies the number of documents to pass to the configured actions when the the threshold condition is met.
 ES query:: This clause specifies the ES DSL query to execute. The number of documents that match this query will be evaulated against the threshold
 condition. Aggregations are not supported at this time. 
 Threshold:: This clause defines a threshold value and a comparison operator  (`is above`, `is above or equals`, `is below`, `is below or equals`, or `is between`). The number of documents that match the specified query is compared to this threshold.

diff --git a/docs/user/alerting/images/alert-types-es-query-conditions.png b/docs/user/alerting/images/alert-types-es-query-conditions.png
diff --git a/x-pack/plugins/stack_alerts/public/alert_types/es_query/expression.test.tsx b/x-pack/plugins/stack_alerts/public/alert_types/es_query/expression.test.tsx
@@ -126,6 +126,7 @@ describe('EsQueryAlertTypeExpression', () => {
       index: ['test-index'],
       timeField: '@timestamp',
       esQuery: `{\n  \"query\":{\n    \"match_all\" : {}\n  }\n}`,
+      size: 100,
       thresholdComparator: '>',
       threshold: [0],
       timeWindowSize: 15,
@@ -137,6 +138,7 @@ describe('EsQueryAlertTypeExpression', () => {
     const errors = {
       index: [],
       esQuery: [],
+      size: [],
       timeField: [],
       timeWindowSize: [],
     };
@@ -169,6 +171,7 @@ describe('EsQueryAlertTypeExpression', () => {
   test('should render EsQueryAlertTypeExpression with expected components', async () => {
     const wrapper = await setup(getAlertParams());
     expect(wrapper.find('[data-test-subj="indexSelectPopover"]').exists()).toBeTruthy();
+    expect(wrapper.find('[data-test-subj="sizeValueExpression"]').exists()).toBeTruthy();
     expect(wrapper.find('[data-test-subj="queryJsonEditor"]').exists()).toBeTruthy();
     expect(wrapper.find('[data-test-subj="testQuerySuccess"]').exists()).toBeFalsy();
     expect(wrapper.find('[data-test-subj="testQueryError"]').exists()).toBeFalsy();

diff --git a/x-pack/plugins/stack_alerts/public/alert_types/es_query/expression.tsx b/x-pack/plugins/stack_alerts/public/alert_types/es_query/expression.tsx
@@ -30,6 +30,7 @@ import {
   COMPARATORS,
   ThresholdExpression,
   ForLastExpression,
+  ValueExpression,
   AlertTypeParamsExpressionProps,
 } from '../../../../triggers_actions_ui/public';
 import { validateExpression } from './validation';
@@ -45,6 +46,7 @@ const DEFAULT_VALUES = {
     "match_all" : {}
   }
 }`,
+  SIZE: 100,
   TIME_WINDOW_SIZE: 5,
   TIME_WINDOW_UNIT: 'm',
   THRESHOLD: [1000],
@@ -53,6 +55,7 @@ const DEFAULT_VALUES = {
 const expressionFieldsWithValidation = [
   'index',
   'esQuery',
+  'size',
   'timeField',
   'threshold0',
   'threshold1',
@@ -74,6 +77,7 @@ export const EsQueryAlertTypeExpression: React.FunctionComponent<
     index,
     timeField,
     esQuery,
+    size,
     thresholdComparator,
     threshold,
     timeWindowSize,
@@ -83,6 +87,7 @@ export const EsQueryAlertTypeExpression: React.FunctionComponent<
   const getDefaultParams = () => ({
     ...alertParams,
     esQuery: esQuery ?? DEFAULT_VALUES.QUERY,
+    size: size ?? DEFAULT_VALUES.SIZE,
     timeWindowSize: timeWindowSize ?? DEFAULT_VALUES.TIME_WINDOW_SIZE,
     timeWindowUnit: timeWindowUnit ?? DEFAULT_VALUES.TIME_WINDOW_UNIT,
     threshold: threshold ?? DEFAULT_VALUES.THRESHOLD,
@@ -214,7 +219,7 @@ export const EsQueryAlertTypeExpression: React.FunctionComponent<
         <h5>
           <FormattedMessage
             id="xpack.stackAlerts.esQuery.ui.selectIndex"
-            defaultMessage="Select an index"
+            defaultMessage="Select an index and size"
           />
         </h5>
       </EuiTitle>
@@ -234,6 +239,7 @@ export const EsQueryAlertTypeExpression: React.FunctionComponent<
               ...alertParams,
               index: indices,
               esQuery: DEFAULT_VALUES.QUERY,
+              size: DEFAULT_VALUES.SIZE,
               thresholdComparator: DEFAULT_VALUES.THRESHOLD_COMPARATOR,
               timeWindowSize: DEFAULT_VALUES.TIME_WINDOW_SIZE,
               timeWindowUnit: DEFAULT_VALUES.TIME_WINDOW_UNIT,
@@ -246,6 +252,19 @@ export const EsQueryAlertTypeExpression: React.FunctionComponent<
         }}
         onTimeFieldChange={(updatedTimeField: string) => setParam('timeField', updatedTimeField)}
       />
+      <ValueExpression
+        description={i18n.translate('xpack.stackAlerts.esQuery.ui.sizeExpression', {
+          defaultMessage: 'Size',
+        })}
+        data-test-subj="sizeValueExpression"
+        value={size}
+        errors={errors.size}
+        display="fullWidth"
+        popupPosition={'upLeft'}
+        onChangeSelectedValue={(updatedValue) => {
+          setParam('size', updatedValue);
+        }}
+      />
       <EuiSpacer />
       <EuiTitle size="xs">
         <h5>

diff --git a/x-pack/plugins/stack_alerts/public/alert_types/es_query/types.ts b/x-pack/plugins/stack_alerts/public/alert_types/es_query/types.ts
@@ -17,6 +17,7 @@ export interface EsQueryAlertParams extends AlertTypeParams {
   index: string[];
   timeField?: string;
   esQuery: string;
+  size: number;
   thresholdComparator?: string;
   threshold: number[];
   timeWindowSize: number;

diff --git a/x-pack/plugins/stack_alerts/public/alert_types/es_query/validation.test.ts b/x-pack/plugins/stack_alerts/public/alert_types/es_query/validation.test.ts
@@ -13,6 +13,7 @@ describe('expression params validation', () => {
     const initialParams: EsQueryAlertParams = {
       index: [],
       esQuery: `{\n  \"query\":{\n    \"match_all\" : {}\n  }\n}`,
+      size: 100,
       timeWindowSize: 1,
       timeWindowUnit: 's',
       threshold: [0],
@@ -25,6 +26,7 @@ describe('expression params validation', () => {
     const initialParams: EsQueryAlertParams = {
       index: ['test'],
       esQuery: `{\n  \"query\":{\n    \"match_all\" : {}\n  }\n}`,
+      size: 100,
       timeWindowSize: 1,
       timeWindowUnit: 's',
       threshold: [0],
@@ -37,6 +39,7 @@ describe('expression params validation', () => {
     const initialParams: EsQueryAlertParams = {
       index: ['test'],
       esQuery: `{\n  \"query\":{\n    \"match_all\" : {}\n  }\n`,
+      size: 100,
       timeWindowSize: 1,
       timeWindowUnit: 's',
       threshold: [0],
@@ -49,6 +52,7 @@ describe('expression params validation', () => {
     const initialParams: EsQueryAlertParams = {
       index: ['test'],
       esQuery: `{\n  \"aggs\":{\n    \"match_all\" : {}\n  }\n}`,
+      size: 100,
       timeWindowSize: 1,
       timeWindowUnit: 's',
       threshold: [0],
@@ -61,6 +65,7 @@ describe('expression params validation', () => {
     const initialParams: EsQueryAlertParams = {
       index: ['test'],
       esQuery: `{\n  \"query\":{\n    \"match_all\" : {}\n  }\n}`,
+      size: 100,
       threshold: [],
       timeWindowSize: 1,
       timeWindowUnit: 's',
@@ -74,6 +79,7 @@ describe('expression params validation', () => {
     const initialParams: EsQueryAlertParams = {
       index: ['test'],
       esQuery: `{\n  \"query\":{\n    \"match_all\" : {}\n  }\n}`,
+      size: 100,
       threshold: [1],
       timeWindowSize: 1,
       timeWindowUnit: 's',
@@ -87,6 +93,7 @@ describe('expression params validation', () => {
     const initialParams: EsQueryAlertParams = {
       index: ['test'],
       esQuery: `{\n  \"query\":{\n    \"match_all\" : {}\n  }\n}`,
+      size: 100,
       threshold: [10, 1],
       timeWindowSize: 1,
       timeWindowUnit: 's',
@@ -97,4 +104,34 @@ describe('expression params validation', () => {
       'Threshold 1 must be > Threshold 0.'
     );
   });
+
+  test('if size property is < 0 should return proper error message', () => {
+    const initialParams: EsQueryAlertParams = {
+      index: ['test'],
+      esQuery: `{\n  \"query\":{\n    \"match_all\" : {}\n  }\n`,
+      size: -1,
+      timeWindowSize: 1,
+      timeWindowUnit: 's',
+      threshold: [0],
+    };
+    expect(validateExpression(initialParams).errors.size.length).toBeGreaterThan(0);
+    expect(validateExpression(initialParams).errors.size[0]).toBe(
+      'Size must be between 0 and 10,000.'
+    );
+  });
+
+  test('if size property is > 10000 should return proper error message', () => {
+    const initialParams: EsQueryAlertParams = {
+      index: ['test'],
+      esQuery: `{\n  \"query\":{\n    \"match_all\" : {}\n  }\n`,
+      size: 25000,
+      timeWindowSize: 1,
+      timeWindowUnit: 's',
+      threshold: [0],
+    };
+    expect(validateExpression(initialParams).errors.size.length).toBeGreaterThan(0);
+    expect(validateExpression(initialParams).errors.size[0]).toBe(
+      'Size must be between 0 and 10,000.'
+    );
+  });
 });
diff --git a/x-pack/plugins/stack_alerts/public/alert_types/es_query/validation.ts b/x-pack/plugins/stack_alerts/public/alert_types/es_query/validation.ts
@@ -10,12 +10,21 @@ import { EsQueryAlertParams } from './types';
 import { ValidationResult, builtInComparators } from '../../../../triggers_actions_ui/public';
 
 export const validateExpression = (alertParams: EsQueryAlertParams): ValidationResult => {
-  const { index, timeField, esQuery, threshold, timeWindowSize, thresholdComparator } = alertParams;
+  const {
+    index,
+    timeField,
+    esQuery,
+    size,
+    threshold,
+    timeWindowSize,
+    thresholdComparator,
+  } = alertParams;
   const validationResult = { errors: {} };
   const errors = {
     index: new Array<string>(),
     timeField: new Array<string>(),
     esQuery: new Array<string>(),
+    size: new Array<string>(),
     threshold0: new Array<string>(),
     threshold1: new Array<string>(),
     thresholdComparator: new Array<string>(),
@@ -94,5 +103,20 @@ export const validateExpression = (alertParams: EsQueryAlertParams): ValidationR
       })
     );
   }
+  if (!size) {
+    errors.size.push(
+      i18n.translate('xpack.stackAlerts.esQuery.ui.validation.error.requiredSizeText', {
+        defaultMessage: 'Size is required.',
+      })
+    );
+  }
+  if ((size && size < 0) || size > 10000) {
+    errors.size.push(
+      i18n.translate('xpack.stackAlerts.esQuery.ui.validation.error.invalidSizeRangeText', {
+        defaultMessage: 'Size must be between 0 and {max, number}.',
+        values: { max: 10000 },
+      })
+    );
+  }
   return validationResult;
 };
diff --git a/x-pack/plugins/stack_alerts/server/alert_types/es_query/action_context.test.ts b/x-pack/plugins/stack_alerts/server/alert_types/es_query/action_context.test.ts
@@ -14,6 +14,7 @@ describe('ActionContext', () => {
       index: ['[index]'],
       timeField: '[timeField]',
       esQuery: `{\n  \"query\":{\n    \"match_all\" : {}\n  }\n}`,
+      size: 100,
       timeWindowSize: 5,
       timeWindowUnit: 'm',
       thresholdComparator: '>',
@@ -41,6 +42,7 @@ describe('ActionContext', () => {
       index: ['[index]'],
       timeField: '[timeField]',
       esQuery: `{\n  \"query\":{\n    \"match_all\" : {}\n  }\n}`,
+      size: 100,
       timeWindowSize: 5,
       timeWindowUnit: 'm',
       thresholdComparator: 'between',

diff --git a/x-pack/plugins/stack_alerts/server/alert_types/es_query/alert_type.test.ts b/x-pack/plugins/stack_alerts/server/alert_types/es_query/alert_type.test.ts
@@ -57,6 +57,10 @@ describe('alertType', () => {
             "description": "The string representation of the ES query.",
             "name": "esQuery",
           },
+          Object {
+            "description": "The number of hits to retrieve for each query.",
+            "name": "size",
+          },
           Object {
             "description": "An array of values to use as the threshold; 'between' and 'notBetween' require two values, the others require one.",
             "name": "threshold",
@@ -75,6 +79,7 @@ describe('alertType', () => {
       index: ['index-name'],
       timeField: 'time-field',
       esQuery: `{\n  \"query\":{\n    \"match_all\" : {}\n  }\n}`,
+      size: 100,
       timeWindowSize: 5,
       timeWindowUnit: 'm',
       thresholdComparator: '<',
@@ -92,6 +97,7 @@ describe('alertType', () => {
       index: ['index-name'],
       timeField: 'time-field',
       esQuery: `{\n  \"query\":{\n    \"match_all\" : {}\n  }\n}`,
+      size: 100,
       timeWindowSize: 5,
       timeWindowUnit: 'm',
       thresholdComparator: 'between',

diff --git a/x-pack/plugins/stack_alerts/server/alert_types/es_query/alert_type.ts b/x-pack/plugins/stack_alerts/server/alert_types/es_query/alert_type.ts
@@ -23,8 +23,6 @@ import { ESSearchHit } from '../../../../../typings/elasticsearch';
 
 export const ES_QUERY_ID = '.es-query';
 
-const DEFAULT_MAX_HITS_PER_EXECUTION = 1000;
-
 const ActionGroupId = 'query matched';
 const ConditionMetAlertInstanceId = 'query matched';
 
@@ -88,6 +86,13 @@ export function getAlertType(
     }
   );
 
+  const actionVariableContextSizeLabel = i18n.translate(
+    'xpack.stackAlerts.esQuery.actionVariableContextSizeLabel',
+    {
+      defaultMessage: 'The number of hits to retrieve for each query.',
+    }
+  );
+
   const actionVariableContextThresholdLabel = i18n.translate(
     'xpack.stackAlerts.esQuery.actionVariableContextThresholdLabel',
     {
@@ -130,6 +135,7 @@ export function getAlertType(
       params: [
         { name: 'index', description: actionVariableContextIndexLabel },
         { name: 'esQuery', description: actionVariableContextQueryLabel },
+        { name: 'size', description: actionVariableContextSizeLabel },
         { name: 'threshold', description: actionVariableContextThresholdLabel },
         { name: 'thresholdComparator', description: actionVariableContextThresholdComparatorLabel },
       ],
@@ -160,7 +166,7 @@ export function getAlertType(
     }
 
     // During each alert execution, we run the configured query, get a hit count
-    // (hits.total) and retrieve up to DEFAULT_MAX_HITS_PER_EXECUTION hits. We
+    // (hits.total) and retrieve up to params.size hits. We
     // evaluate the threshold condition using the value of hits.total. If the threshold
     // condition is met, the hits are counted toward the query match and we update
     // the alert state with the timestamp of the latest hit. In the next execution
@@ -200,7 +206,7 @@ export function getAlertType(
       from: dateStart,
       to: dateEnd,
       filter,
-      size: DEFAULT_MAX_HITS_PER_EXECUTION,
+      size: params.size,
       sortOrder: 'desc',
       searchAfterSortId: undefined,
       timeField: params.timeField,

diff --git a/x-pack/plugins/stack_alerts/server/alert_types/es_query/alert_type_params.test.ts b/x-pack/plugins/stack_alerts/server/alert_types/es_query/alert_type_params.test.ts
@@ -7,12 +7,17 @@
 
 import { TypeOf } from '@kbn/config-schema';
 import type { Writable } from '@kbn/utility-types';
-import { EsQueryAlertParamsSchema, EsQueryAlertParams } from './alert_type_params';
+import {
+  EsQueryAlertParamsSchema,
+  EsQueryAlertParams,
+  ES_QUERY_MAX_HITS_PER_EXECUTION,
+} from './alert_type_params';
 
 const DefaultParams: Writable<Partial<EsQueryAlertParams>> = {
   index: ['index-name'],
   timeField: 'time-field',
   esQuery: `{\n  \"query\":{\n    \"match_all\" : {}\n  }\n}`,
+  size: 100,
   timeWindowSize: 5,
   timeWindowUnit: 'm',
   thresholdComparator: '>',
@@ -99,6 +104,28 @@ describe('alertType Params validate()', () => {
     );
   });
 
+  it('fails for invalid size', async () => {
+    delete params.size;
+    expect(onValidate()).toThrowErrorMatchingInlineSnapshot(
+      `"[size]: expected value of type [number] but got [undefined]"`
+    );
+
+    params.size = 'foo';
+    expect(onValidate()).toThrowErrorMatchingInlineSnapshot(
+      `"[size]: expected value of type [number] but got [string]"`
+    );
+
+    params.size = -1;
+    expect(onValidate()).toThrowErrorMatchingInlineSnapshot(
+      `"[size]: Value must be equal to or greater than [0]."`
+    );
+
+    params.size = ES_QUERY_MAX_HITS_PER_EXECUTION + 1;
+    expect(onValidate()).toThrowErrorMatchingInlineSnapshot(
+      `"[size]: Value must be equal to or lower than [10000]."`
+    );
+  });
+
   it('fails for invalid timeWindowSize', async () => {
     delete params.timeWindowSize;
     expect(onValidate()).toThrowErrorMatchingInlineSnapshot(