Adds date time query and return fields for timestamps and overrides (e…

…lastic#79911) ## Summary Fixes elastic#79865 Also fixes: * Timestamp override not being pushed down into threshold rules to use * Timestamp override not being used for lastValidDate * The return format of the date time might have been different depending on the customer mapping for both the override and the regular @timestamp so this fixes that as well. * Fixes one small type issue with fields. ### Checklist Delete any items that are not applicable to this PR. - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios
FrankHassanabad · Oct 8, 2020 · 7b7d931 · 7b7d931
1 parent 35dbb66
commit 7b7d931
Show file tree

Hide file tree

Showing 10 changed files with 207 additions and 34 deletions.
diff --git a/x-pack/plugins/security_solution/common/detection_engine/types.ts b/x-pack/plugins/security_solution/common/detection_engine/types.ts
@@ -35,6 +35,7 @@ export interface BaseHit<T> {
   _index: string;
   _id: string;
   _source: T;
+  fields?: Record<string, SearchTypes[]>;
 }
 
 export interface EqlSequence<T> {

diff --git a/.../plugins/security_solution/server/lib/detection_engine/signals/build_events_query.test.ts b/.../plugins/security_solution/server/lib/detection_engine/signals/build_events_query.test.ts
@@ -23,6 +23,12 @@ describe('create_signals', () => {
       size: 100,
       ignoreUnavailable: true,
       body: {
+        docvalue_fields: [
+          {
+            field: '@timestamp',
+            format: 'strict_date_optional_time',
+          },
+        ],
         query: {
           bool: {
             filter: [
@@ -37,6 +43,7 @@ describe('create_signals', () => {
                             range: {
                               '@timestamp': {
                                 gte: 'now-5m',
+                                format: 'strict_date_optional_time',
                               },
                             },
                           },
@@ -51,6 +58,7 @@ describe('create_signals', () => {
                             range: {
                               '@timestamp': {
                                 lte: 'today',
+                                format: 'strict_date_optional_time',
                               },
                             },
                           },
@@ -94,6 +102,12 @@ describe('create_signals', () => {
       size: 100,
       ignoreUnavailable: true,
       body: {
+        docvalue_fields: [
+          {
+            field: '@timestamp',
+            format: 'strict_date_optional_time',
+          },
+        ],
         query: {
           bool: {
             filter: [
@@ -108,6 +122,7 @@ describe('create_signals', () => {
                             range: {
                               '@timestamp': {
                                 gte: 'now-5m',
+                                format: 'strict_date_optional_time',
                               },
                             },
                           },
@@ -122,6 +137,7 @@ describe('create_signals', () => {
                             range: {
                               '@timestamp': {
                                 lte: 'today',
+                                format: 'strict_date_optional_time',
                               },
                             },
                           },
@@ -166,6 +182,12 @@ describe('create_signals', () => {
       size: 100,
       ignoreUnavailable: true,
       body: {
+        docvalue_fields: [
+          {
+            field: '@timestamp',
+            format: 'strict_date_optional_time',
+          },
+        ],
         query: {
           bool: {
             filter: [
@@ -180,6 +202,7 @@ describe('create_signals', () => {
                             range: {
                               '@timestamp': {
                                 gte: 'now-5m',
+                                format: 'strict_date_optional_time',
                               },
                             },
                           },
@@ -194,6 +217,7 @@ describe('create_signals', () => {
                             range: {
                               '@timestamp': {
                                 lte: 'today',
+                                format: 'strict_date_optional_time',
                               },
                             },
                           },
@@ -239,6 +263,12 @@ describe('create_signals', () => {
       size: 100,
       ignoreUnavailable: true,
       body: {
+        docvalue_fields: [
+          {
+            field: '@timestamp',
+            format: 'strict_date_optional_time',
+          },
+        ],
         query: {
           bool: {
             filter: [
@@ -253,6 +283,7 @@ describe('create_signals', () => {
                             range: {
                               '@timestamp': {
                                 gte: 'now-5m',
+                                format: 'strict_date_optional_time',
                               },
                             },
                           },
@@ -267,6 +298,7 @@ describe('create_signals', () => {
                             range: {
                               '@timestamp': {
                                 lte: 'today',
+                                format: 'strict_date_optional_time',
                               },
                             },
                           },
@@ -311,6 +343,12 @@ describe('create_signals', () => {
       size: 100,
       ignoreUnavailable: true,
       body: {
+        docvalue_fields: [
+          {
+            field: '@timestamp',
+            format: 'strict_date_optional_time',
+          },
+        ],
         query: {
           bool: {
             filter: [
@@ -325,6 +363,7 @@ describe('create_signals', () => {
                             range: {
                               '@timestamp': {
                                 gte: 'now-5m',
+                                format: 'strict_date_optional_time',
                               },
                             },
                           },
@@ -339,6 +378,7 @@ describe('create_signals', () => {
                             range: {
                               '@timestamp': {
                                 lte: 'today',
+                                format: 'strict_date_optional_time',
                               },
                             },
                           },
@@ -390,6 +430,7 @@ describe('create_signals', () => {
       size: 100,
       ignoreUnavailable: true,
       body: {
+        docvalue_fields: [{ field: '@timestamp', format: 'strict_date_optional_time' }],
         query: {
           bool: {
             filter: [
@@ -404,6 +445,7 @@ describe('create_signals', () => {
                             range: {
                               '@timestamp': {
                                 gte: 'now-5m',
+                                format: 'strict_date_optional_time',
                               },
                             },
                           },
@@ -418,6 +460,7 @@ describe('create_signals', () => {
                             range: {
                               '@timestamp': {
                                 lte: 'today',
+                                format: 'strict_date_optional_time',
                               },
                             },
                           },

diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.ts
@@ -28,6 +28,25 @@ export const buildEventsSearchQuery = ({
   timestampOverride,
 }: BuildEventsSearchQuery) => {
   const timestamp = timestampOverride ?? '@timestamp';
+  const docFields =
+    timestampOverride != null
+      ? [
+          {
+            field: '@timestamp',
+            format: 'strict_date_optional_time',
+          },
+          {
+            field: timestampOverride,
+            format: 'strict_date_optional_time',
+          },
+        ]
+      : [
+          {
+            field: '@timestamp',
+            format: 'strict_date_optional_time',
+          },
+        ];
+
   const filterWithTime = [
     filter,
     {
@@ -40,6 +59,7 @@ export const buildEventsSearchQuery = ({
                   range: {
                     [timestamp]: {
                       gte: from,
+                      format: 'strict_date_optional_time',
                     },
                   },
                 },
@@ -54,6 +74,7 @@ export const buildEventsSearchQuery = ({
                   range: {
                     [timestamp]: {
                       lte: to,
+                      format: 'strict_date_optional_time',
                     },
                   },
                 },
@@ -65,12 +86,14 @@ export const buildEventsSearchQuery = ({
       },
     },
   ];
+
   const searchQuery = {
     allowNoIndices: true,
     index,
     size,
     ignoreUnavailable: true,
     body: {
+      docvalue_fields: docFields,
       query: {
         bool: {
           filter: [

diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.ts
@@ -102,7 +102,7 @@ export const buildRule = ({
     created_by: createdBy,
     updated_by: updatedBy,
     threat: ruleParams.threat ?? [],
-    timestamp_override: ruleParams.timestampOverride, // TODO: Timestamp Override via timestamp_override
+    timestamp_override: ruleParams.timestampOverride,
     throttle,
     version: ruleParams.version,
     created_at: createdAt,
@@ -155,7 +155,7 @@ export const buildRuleWithoutOverrides = (
     created_by: ruleSO.attributes.createdBy,
     updated_by: ruleSO.attributes.updatedBy,
     threat: ruleParams.threat ?? [],
-    timestamp_override: ruleParams.timestampOverride, // TODO: Timestamp Override via timestamp_override
+    timestamp_override: ruleParams.timestampOverride,
     throttle: ruleSO.attributes.throttle,
     version: ruleParams.version,
     created_at: ruleSO.attributes.createdAt,

diff --git a/...k/plugins/security_solution/server/lib/detection_engine/signals/find_threshold_signals.ts b/...k/plugins/security_solution/server/lib/detection_engine/signals/find_threshold_signals.ts
@@ -6,7 +6,10 @@
 
 import { isEmpty } from 'lodash/fp';
 
-import { Threshold } from '../../../../common/detection_engine/schemas/common/schemas';
+import {
+  Threshold,
+  TimestampOverrideOrUndefined,
+} from '../../../../common/detection_engine/schemas/common/schemas';
 import { singleSearchAfter } from './single_search_after';
 
 import { AlertServices } from '../../../../../alerts/server';
@@ -23,6 +26,7 @@ interface FindThresholdSignalsParams {
   filter: unknown;
   threshold: Threshold;
   buildRuleMessage: BuildRuleMessage;
+  timestampOverride: TimestampOverrideOrUndefined;
 }
 
 export const findThresholdSignals = async ({
@@ -34,6 +38,7 @@ export const findThresholdSignals = async ({
   filter,
   threshold,
   buildRuleMessage,
+  timestampOverride,
 }: FindThresholdSignalsParams): Promise<{
   searchResult: SignalSearchResponse;
   searchDuration: string;
@@ -54,7 +59,7 @@ export const findThresholdSignals = async ({
   return singleSearchAfter({
     aggregations,
     searchAfterSortId: undefined,
-    timestampOverride: undefined,
+    timestampOverride,
     index: inputIndexPattern,
     from,
     to,

diff --git a/...plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts b/...plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts
@@ -95,7 +95,10 @@ export const searchAfterAndBulkCreate = async ({
         });
         toReturn = mergeReturns([
           toReturn,
-          createSearchAfterReturnTypeFromResponse({ searchResult }),
+          createSearchAfterReturnTypeFromResponse({
+            searchResult,
+            timestampOverride: ruleParams.timestampOverride,
+          }),
           createSearchAfterReturnType({
             searchAfterTimes: [searchDuration],
             errors: searchErrors,

diff --git a/...k/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/...k/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts
@@ -99,6 +99,7 @@ export const signalRulesAlertType = ({
         from,
         ruleId,
         index,
+        eventCategoryOverride,
         filters,
         language,
         maxSignals,
@@ -114,6 +115,7 @@ export const signalRulesAlertType = ({
         threatIndex,
         threatMapping,
         threatLanguage,
+        timestampOverride,
         type,
         exceptionsList,
       } = params;
@@ -313,6 +315,7 @@ export const signalRulesAlertType = ({
             logger,
             filter: esFilter,
             threshold,
+            timestampOverride,
             buildRuleMessage,
           });
 
@@ -346,7 +349,10 @@ export const signalRulesAlertType = ({
           });
           result = mergeReturns([
             result,
-            createSearchAfterReturnTypeFromResponse({ searchResult: thresholdResults }),
+            createSearchAfterReturnTypeFromResponse({
+              searchResult: thresholdResults,
+              timestampOverride,
+            }),
             createSearchAfterReturnType({
               success,
               errors: [...errors, ...searchErrors],
@@ -464,12 +470,12 @@ export const signalRulesAlertType = ({
           const request = buildEqlSearchRequest(
             query,
             inputIndex,
-            params.from,
-            params.to,
+            from,
+            to,
             searchAfterSize,
-            params.timestampOverride,
+            timestampOverride,
             exceptionItems ?? [],
-            params.eventCategoryOverride
+            eventCategoryOverride
           );
           const response: EqlSignalSearchResponse = await services.callCluster(
             'transport.request',