[Security Solution][Detection Engine] log ES requests when running ru…

…le preview (#191107) ## Summary **Status:** works only for **ES|QL and EQL** rule types When clicking on "Show Elasticsearch requests, ran during rule executions" preview would return logged Elasticsearch queries that can be used to debug/explore rule execution. Each rule execution accordion has time rule execution started and its duration. Upon opening accordion: it will display ES requests with their description and duration. **NOTE**: Only search requests are returned, not the requests that create actual alerts Feature flag: **loggingRequestsEnabled** On week Demo([internal link](https://drive.google.com/drive/folders/1l-cDhbiMxykNH6BzIxFAnLeibmV9a4Cz)) ### Video demo (older UI) https://github.com/user-attachments/assets/26f963da-c528-447c-9efd-350b4d42b52c ### Up to date UI #### UI control <img width="733" alt="Screenshot 2024-09-11 at 12 39 07" src="https://github.com/user-attachments/assets/c2b1304d-6f93-4e8e-92f9-a6a0b53cefc7"> #### List of executions and code blocks <img width="770" alt="Screenshot 2024-09-11 at 12 38 23" src="https://github.com/user-attachments/assets/48b5aa12-174c-46f5-b0bc-a141833b225b"> ### Checklist Delete any items that are not applicable to this PR. - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios - [x] [Flaky Test Runner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1) was used on any tests changed 🎉 All tests passed! - [kibana-flaky-test-suite-runner#6909](https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/6909) [✅] [Serverless] Security Solution Detection Engine - Cypress: 100/100 tests passed. [✅] Security Solution Detection Engine - Cypress: 100/100 tests passed. FTR tests - https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/6918 --------- Co-authored-by: kibanamachine <[email protected]>
elastic · Sep 19, 2024 · 60176bc · 60176bc
1 parent e524ed6
commit 60176bc
Show file tree

Hide file tree

Showing 46 changed files with 1,268 additions and 165 deletions.
diff --git a/...ck/plugins/security_solution/common/api/detection_engine/rule_preview/rule_preview.gen.ts b/...ck/plugins/security_solution/common/api/detection_engine/rule_preview/rule_preview.gen.ts
@@ -15,6 +15,7 @@
  */
 
 import { z } from '@kbn/zod';
+import { BooleanFromString } from '@kbn/zod-helpers';
 
 import {
   EqlRuleCreateProps,
@@ -34,6 +35,13 @@ export const RulePreviewParams = z.object({
   timeframeEnd: z.string().datetime(),
 });
 
+export type RulePreviewLoggedRequest = z.infer<typeof RulePreviewLoggedRequest>;
+export const RulePreviewLoggedRequest = z.object({
+  request: NonEmptyString,
+  description: NonEmptyString.optional(),
+  duration: z.number().int().optional(),
+});
+
 export type RulePreviewLogs = z.infer<typeof RulePreviewLogs>;
 export const RulePreviewLogs = z.object({
   errors: z.array(NonEmptyString),
@@ -43,7 +51,17 @@ export const RulePreviewLogs = z.object({
    */
   duration: z.number().int(),
   startedAt: NonEmptyString.optional(),
+  requests: z.array(RulePreviewLoggedRequest).optional(),
+});
+
+export type RulePreviewRequestQuery = z.infer<typeof RulePreviewRequestQuery>;
+export const RulePreviewRequestQuery = z.object({
+  /**
+   * Enables logging and returning in response ES queries, performed during rule execution
+   */
+  enable_logged_requests: BooleanFromString.optional(),
 });
+export type RulePreviewRequestQueryInput = z.input<typeof RulePreviewRequestQuery>;
 
 export type RulePreviewRequestBody = z.infer<typeof RulePreviewRequestBody>;
 export const RulePreviewRequestBody = z.discriminatedUnion('type', [

diff --git a/...ugins/security_solution/common/api/detection_engine/rule_preview/rule_preview.schema.yaml b/...ugins/security_solution/common/api/detection_engine/rule_preview/rule_preview.schema.yaml
@@ -11,6 +11,13 @@ paths:
       summary: Preview rule alerts generated on specified time range
       tags:
         - Rule preview API
+      parameters:
+        - name: enable_logged_requests
+          in: query
+          description: Enables logging and returning in response ES queries, performed during rule execution
+          required: false
+          schema:
+            type: boolean
       requestBody:
         description: An object containing tags to add or remove and alert ids the changes will be applied
         required: true
@@ -94,6 +101,18 @@ components:
           format: date-time
       required: [invocationCount, timeframeEnd]
 
+    RulePreviewLoggedRequest:
+      type: object  
+      properties:
+        request:
+          $ref: '../../model/primitives.schema.yaml#/components/schemas/NonEmptyString'
+        description:
+          $ref: '../../model/primitives.schema.yaml#/components/schemas/NonEmptyString'
+        duration:
+          type: integer
+      required:
+        - request
+
     RulePreviewLogs:
       type: object
       properties:
@@ -110,6 +129,10 @@ components:
           description: Execution duration in milliseconds
         startedAt:
           $ref: '../../model/primitives.schema.yaml#/components/schemas/NonEmptyString'
+        requests:
+          type: array
+          items:
+            $ref: '#/components/schemas/RulePreviewLoggedRequest'
       required:
         - errors
         - warnings

diff --git a/x-pack/plugins/security_solution/common/api/quickstart_client.gen.ts b/x-pack/plugins/security_solution/common/api/quickstart_client.gen.ts
@@ -99,6 +99,7 @@ import type {
   GetRuleExecutionResultsResponse,
 } from './detection_engine/rule_monitoring/rule_execution_logs/get_rule_execution_results/get_rule_execution_results_route.gen';
 import type {
+  RulePreviewRequestQueryInput,
   RulePreviewRequestBodyInput,
   RulePreviewResponse,
 } from './detection_engine/rule_preview/rule_preview.gen';
@@ -1763,6 +1764,7 @@ detection engine rules.
         },
         method: 'POST',
         body: props.body,
+        query: props.query,
       })
       .catch(catchAxiosErrorFormatAndThrow);
   }
@@ -2160,6 +2162,7 @@ export interface ResolveTimelineProps {
   query: ResolveTimelineRequestQueryInput;
 }
 export interface RulePreviewProps {
+  query: RulePreviewRequestQueryInput;
   body: RulePreviewRequestBodyInput;
 }
 export interface SearchAlertsProps {

diff --git a/x-pack/plugins/security_solution/common/experimental_features.ts b/x-pack/plugins/security_solution/common/experimental_features.ts
@@ -138,6 +138,11 @@ export const allowedExperimentalValues = Object.freeze({
    */
   esqlRulesDisabled: false,
 
+  /**
+   * enables logging requests during rule preview
+   */
+  loggingRequestsEnabled: false,
+
   /**
    * Enables Protection Updates tab in the Endpoint Policy Details page
    */

diff --git a/...solution/docs/openapi/ess/security_solution_detections_api_2023_10_31.bundled.schema.yaml b/...solution/docs/openapi/ess/security_solution_detections_api_2023_10_31.bundled.schema.yaml
@@ -891,6 +891,15 @@ paths:
   /api/detection_engine/rules/preview:
     post:
       operationId: RulePreview
+      parameters:
+        - description: >-
+            Enables logging and returning in response ES queries, performed
+            during rule execution
+          in: query
+          name: enable_logged_requests
+          required: false
+          schema:
+            type: boolean
       requestBody:
         content:
           application/json:
@@ -5178,6 +5187,17 @@ components:
         - $ref: '#/components/schemas/MachineLearningRulePatchProps'
         - $ref: '#/components/schemas/NewTermsRulePatchProps'
         - $ref: '#/components/schemas/EsqlRulePatchProps'
+    RulePreviewLoggedRequest:
+      type: object
+      properties:
+        description:
+          $ref: '#/components/schemas/NonEmptyString'
+        duration:
+          type: integer
+        request:
+          $ref: '#/components/schemas/NonEmptyString'
+      required:
+        - request
     RulePreviewLogs:
       type: object
       properties:
@@ -5188,6 +5208,10 @@ components:
           items:
             $ref: '#/components/schemas/NonEmptyString'
           type: array
+        requests:
+          items:
+            $ref: '#/components/schemas/RulePreviewLoggedRequest'
+          type: array
         startedAt:
           $ref: '#/components/schemas/NonEmptyString'
         warnings:

diff --git a/...n/docs/openapi/serverless/security_solution_detections_api_2023_10_31.bundled.schema.yaml b/...n/docs/openapi/serverless/security_solution_detections_api_2023_10_31.bundled.schema.yaml
@@ -476,6 +476,15 @@ paths:
   /api/detection_engine/rules/preview:
     post:
       operationId: RulePreview
+      parameters:
+        - description: >-
+            Enables logging and returning in response ES queries, performed
+            during rule execution
+          in: query
+          name: enable_logged_requests
+          required: false
+          schema:
+            type: boolean
       requestBody:
         content:
           application/json:
@@ -4331,6 +4340,17 @@ components:
         - $ref: '#/components/schemas/MachineLearningRulePatchProps'
         - $ref: '#/components/schemas/NewTermsRulePatchProps'
         - $ref: '#/components/schemas/EsqlRulePatchProps'
+    RulePreviewLoggedRequest:
+      type: object
+      properties:
+        description:
+          $ref: '#/components/schemas/NonEmptyString'
+        duration:
+          type: integer
+        request:
+          $ref: '#/components/schemas/NonEmptyString'
+      required:
+        - request
     RulePreviewLogs:
       type: object
       properties:
@@ -4341,6 +4361,10 @@ components:
           items:
             $ref: '#/components/schemas/NonEmptyString'
           type: array
+        requests:
+          items:
+            $ref: '#/components/schemas/RulePreviewLoggedRequest'
+          type: array
         startedAt:
           $ref: '#/components/schemas/NonEmptyString'
         warnings: