[SIEM][Detection Engine] - Update DE to work with new exceptions sche…

…ma (#69715) (#69932)

## Summary

The purpose of this PR is to update the detection engine to work with the new exception list schema. Previously, exceptions lived directly in the rule, now they are stored separately, and the rule only stores a reference to said exception lists.

Changes are concentrated in two areas. First `x-pack/plugins/lists`:
- Updates the `EntryList` schema so it now takes a list `id` and `type` (as needed for the detections engine logic) as opposed to before where it was just a string for the `id`
- Updates the lists plugin unit tests to account for the updated `EntryList` schema
- Exposes the exceptions list client (in the lists plugin) to be used by the detection engine

Second in `x-pack/plugins/security_solution`:
- Updates the detection engine `exceptions_list` schema. Previously, exceptions sat directly on the rule itself, now that exceptions are stored separately, the `exceptions_list` property needs to only keep reference to the exceptions list
- Updates schema unit tests
- Updates routes to use new exceptions list schema
- Updates route unit tests
- Removes old exception list scripts that are no longer necessary

x-pack/plugins/lists/README.md

@@ -157,12 +157,14 @@ And you can attach exception list items like so:
     {
       "field": "actingProcess.file.signer",
       "operator": "included",
-      "match": "Elastic, N.V."
+      "type": "match",
+      "value": "Elastic, N.V."
     },
     {
       "field": "event.category",
       "operator": "included",
-      "match_any": [
+      "type": "match_any",
+      "value": [
         "process",
         "malware"
       ]

x-pack/plugins/lists/common/constants.mock.ts

@@ -46,10 +46,8 @@ export const EXISTS = 'exists';
 export const NESTED = 'nested';
 export const ENTRIES: EntriesArray = [
   {
-    entries: [
-      { field: 'some.not.nested.field', operator: 'included', type: 'match', value: 'some value' },
-    ],
-    field: 'some.field',
+    entries: [{ field: 'nested.field', operator: 'included', type: 'match', value: 'some value' }],
+    field: 'some.parentField',
     type: 'nested',
   },
   { field: 'some.not.nested.field', operator: 'included', type: 'match', value: 'some value' },

x-pack/plugins/lists/common/schemas/types/default_entries_array.test.ts

@@ -17,7 +17,8 @@ import { getEntriesArrayMock, getEntryMatchMock, getEntryNestedMock } from './en
 // it checks against every item in that union. Since entries consist of 5
 // different entry types, it returns 5 of these. To make more readable,
 // extracted here.
-const returnedSchemaError = `"Array<({| field: string, operator: "excluded" | "included", type: "match", value: string |} | {| field: string, operator: "excluded" | "included", type: "match_any", value: DefaultStringArray |} | {| field: string, operator: "excluded" | "included", type: "list", value: DefaultStringArray |} | {| field: string, operator: "excluded" | "included", type: "exists" |} | {| entries: Array<({| field: string, operator: "excluded" | "included", type: "match", value: string |} | {| field: string, operator: "excluded" | "included", type: "match_any", value: DefaultStringArray |} | {| field: string, operator: "excluded" | "included", type: "list", value: DefaultStringArray |} | {| field: string, operator: "excluded" | "included", type: "exists" |})>, field: string, type: "nested" |})>"`;
+const returnedSchemaError =
+  '"Array<({| field: string, operator: "excluded" | "included", type: "match", value: string |} | {| field: string, operator: "excluded" | "included", type: "match_any", value: DefaultStringArray |} | {| field: string, list: {| id: string, type: "ip" | "keyword" |}, operator: "excluded" | "included", type: "list" |} | {| field: string, operator: "excluded" | "included", type: "exists" |} | {| entries: Array<{| field: string, operator: "excluded" | "included", type: "match", value: string |}>, field: string, type: "nested" |})>"';
 
 describe('default_entries_array', () => {
   test('it should validate an empty array', () => {

x-pack/plugins/lists/common/schemas/types/default_namespace.test.ts

@@ -0,0 +1,61 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { pipe } from 'fp-ts/lib/pipeable';
+import { left } from 'fp-ts/lib/Either';
+
+import { foldLeftRight, getPaths } from '../../siem_common_deps';
+
+import { DefaultNamespace } from './default_namespace';
+
+describe('default_namespace', () => {
+  test('it should validate "single"', () => {
+    const payload = 'single';
+    const decoded = DefaultNamespace.decode(payload);
+    const message = pipe(decoded, foldLeftRight);
+
+    expect(getPaths(left(message.errors))).toEqual([]);
+    expect(message.schema).toEqual(payload);
+  });
+
+  test('it should validate "agnostic"', () => {
+    const payload = 'agnostic';
+    const decoded = DefaultNamespace.decode(payload);
+    const message = pipe(decoded, foldLeftRight);
+
+    expect(getPaths(left(message.errors))).toEqual([]);
+    expect(message.schema).toEqual(payload);
+  });
+
+  test('it defaults to "single" if "undefined"', () => {
+    const payload = undefined;
+    const decoded = DefaultNamespace.decode(payload);
+    const message = pipe(decoded, foldLeftRight);
+
+    expect(getPaths(left(message.errors))).toEqual([]);
+    expect(message.schema).toEqual('single');
+  });
+
+  test('it defaults to "single" if "null"', () => {
+    const payload = null;
+    const decoded = DefaultNamespace.decode(payload);
+    const message = pipe(decoded, foldLeftRight);
+
+    expect(getPaths(left(message.errors))).toEqual([]);
+    expect(message.schema).toEqual('single');
+  });
+
+  test('it should NOT validate if not "single" or "agnostic"', () => {
+    const payload = 'something else';
+    const decoded = DefaultNamespace.decode(payload);
+    const message = pipe(decoded, foldLeftRight);
+
+    expect(getPaths(left(message.errors))).toEqual([
+      `Invalid value "something else" supplied to "DefaultNamespace"`,
+    ]);
+    expect(message.schema).toEqual({});
+  });
+});

x-pack/plugins/lists/common/schemas/types/default_namespace.ts

@@ -7,7 +7,7 @@
 import * as t from 'io-ts';
 import { Either } from 'fp-ts/lib/Either';
 
-const namespaceType = t.keyof({ agnostic: null, single: null });
+export const namespaceType = t.keyof({ agnostic: null, single: null });
 
 type NamespaceType = t.TypeOf<typeof namespaceType>;
 

x-pack/plugins/lists/common/schemas/types/entries.mock.ts

@@ -9,10 +9,12 @@ import {
   EXISTS,
   FIELD,
   LIST,
+  LIST_ID,
   MATCH,
   MATCH_ANY,
   NESTED,
   OPERATOR,
+  TYPE,
 } from '../../constants.mock';
 
 import {
@@ -40,9 +42,9 @@ export const getEntryMatchAnyMock = (): EntryMatchAny => ({
 
 export const getEntryListMock = (): EntryList => ({
   field: FIELD,
+  list: { id: LIST_ID, type: TYPE },
   operator: OPERATOR,
   type: LIST,
-  value: [ENTRY_VALUE],
 });
 
 export const getEntryExistsMock = (): EntryExists => ({
@@ -52,7 +54,7 @@ export const getEntryExistsMock = (): EntryExists => ({
 });
 
 export const getEntryNestedMock = (): EntryNested => ({
-  entries: [getEntryMatchMock(), getEntryExistsMock()],
+  entries: [getEntryMatchMock(), getEntryMatchMock()],
   field: FIELD,
   type: NESTED,
 });

x-pack/plugins/lists/common/schemas/types/entries.test.ts

@@ -251,16 +251,16 @@ describe('Entries', () => {
       expect(message.schema).toEqual(payload);
     });
 
-    test('it should not validate when "value" is not string array', () => {
-      const payload: Omit<EntryList, 'value'> & { value: string } = {
+    test('it should not validate when "list" is not expected value', () => {
+      const payload: Omit<EntryList, 'list'> & { list: string } = {
         ...getEntryListMock(),
-        value: 'someListId',
+        list: 'someListId',
       };
       const decoded = entriesList.decode(payload);
       const message = pipe(decoded, foldLeftRight);
 
       expect(getPaths(left(message.errors))).toEqual([
-        'Invalid value "someListId" supplied to "value"',
+        'Invalid value "someListId" supplied to "list"',
       ]);
       expect(message.schema).toEqual({});
     });
@@ -338,6 +338,20 @@ describe('Entries', () => {
       expect(message.schema).toEqual({});
     });
 
+    test('it should NOT validate when "entries" contains an entry item that is not type "match"', () => {
+      const payload: Omit<EntryNested, 'entries'> & {
+        entries: EntryMatchAny[];
+      } = { ...getEntryNestedMock(), entries: [getEntryMatchAnyMock()] };
+      const decoded = entriesNested.decode(payload);
+      const message = pipe(decoded, foldLeftRight);
+
+      expect(getPaths(left(message.errors))).toEqual([
+        'Invalid value "match_any" supplied to "entries,type"',
+        'Invalid value "["some host name"]" supplied to "entries,value"',
+      ]);
+      expect(message.schema).toEqual({});
+    });
+
     test('it should strip out extra keys', () => {
       const payload: EntryNested & {
         extraKey?: string;

x-pack/plugins/lists/common/schemas/types/entries.ts

@@ -8,7 +8,7 @@
 
 import * as t from 'io-ts';
 
-import { operator } from '../common/schemas';
+import { operator, type } from '../common/schemas';
 import { DefaultStringArray } from '../../siem_common_deps';
 
 export const entriesMatch = t.exact(
@@ -34,9 +34,9 @@ export type EntryMatchAny = t.TypeOf<typeof entriesMatchAny>;
 export const entriesList = t.exact(
   t.type({
     field: t.string,
+    list: t.exact(t.type({ id: t.string, type })),
     operator,
     type: t.keyof({ list: null }),
-    value: DefaultStringArray,
   })
 );
 export type EntryList = t.TypeOf<typeof entriesList>;
@@ -52,7 +52,7 @@ export type EntryExists = t.TypeOf<typeof entriesExists>;
 
 export const entriesNested = t.exact(
   t.type({
-    entries: t.array(t.union([entriesMatch, entriesMatchAny, entriesList, entriesExists])),
+    entries: t.array(entriesMatch),
     field: t.string,
     type: t.keyof({ nested: null }),
   })

x-pack/plugins/lists/common/schemas/types/index.ts

@@ -5,5 +5,6 @@
  */
 export * from './default_comments_array';
 export * from './default_entries_array';
+export * from './default_namespace';
 export * from './comments';
 export * from './entries';

x-pack/plugins/lists/server/index.ts

@@ -11,6 +11,7 @@ import { ListPlugin } from './plugin';
 
 // exporting these since its required at top level in siem plugin
 export { ListClient } from './services/lists/list_client';
+export { ExceptionListClient } from './services/exception_lists/exception_list_client';
 export { ListPluginSetup } from './types';
 
 export const config = { schema: ConfigSchema };

x-pack/plugins/lists/server/saved_objects/exception_list.ts

@@ -105,6 +105,16 @@ export const exceptionListItemMapping: SavedObjectsType['mappings'] = {
         field: {
           type: 'keyword',
         },
+        list: {
+          properties: {
+            id: {
+              type: 'keyword',
+            },
+            type: {
+              type: 'keyword',
+            },
+          },
+        },
         operator: {
           type: 'keyword',
         },

x-pack/plugins/lists/server/scripts/exception_lists/new/exception_list_item_with_list.json

@@ -0,0 +1,24 @@
+{
+  "list_id": "endpoint_list",
+  "item_id": "endpoint_list_item_lg_val_list",
+  "_tags": ["endpoint", "process", "malware", "os:windows"],
+  "tags": ["user added string for a tag", "malware"],
+  "type": "simple",
+  "description": "This is a sample exception list item with a large value list included",
+  "name": "Sample Endpoint Exception List Item with large value list",
+  "comments": [],
+  "entries": [
+    {
+      "field": "event.module",
+      "operator": "excluded",
+      "type": "match_any",
+      "value": ["zeek"]
+    },
+    {
+      "field": "source.ip",
+      "operator": "excluded",
+      "type": "list",
+      "list": { "id": "list-ip", "type": "ip" }
+    }
+  ]
+}

x-pack/plugins/lists/server/scripts/lists/new/list_ip_item.json

@@ -1,5 +1,5 @@
 {
   "id": "hand_inserted_item_id",
   "list_id": "list-ip",
-  "value": "127.0.0.1"
+  "value": "10.4.2.140"
 }

x-pack/plugins/security_solution/common/detection_engine/lists_common_deps.ts

@@ -0,0 +1,7 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export { EntriesArray, namespaceType } from '../../../lists/common/schemas';

x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts

@@ -341,40 +341,3 @@ export type Note = t.TypeOf<typeof note>;
 
 export const noteOrUndefined = t.union([note, t.undefined]);
 export type NoteOrUndefined = t.TypeOf<typeof noteOrUndefined>;
-
-// NOTE: Experimental list support not being shipped currently and behind a feature flag
-// TODO: Remove this comment once we lists have passed testing and is ready for the release
-export const list_field = t.string;
-export const list_values_operator = t.keyof({ included: null, excluded: null });
-export const list_values_type = t.keyof({ match: null, match_all: null, list: null, exists: null });
-export const list_values = t.exact(
-  t.intersection([
-    t.type({
-      name: t.string,
-    }),
-    t.partial({
-      id: t.string,
-      description: t.string,
-      created_at,
-    }),
-  ])
-);
-export const list = t.exact(
-  t.intersection([
-    t.type({
-      field: t.string,
-      values_operator: list_values_operator,
-      values_type: list_values_type,
-    }),
-    t.partial({ values: t.array(list_values) }),
-  ])
-);
-export const list_and = t.intersection([
-  list,
-  t.partial({
-    and: t.array(list),
-  }),
-]);
-
-export const listAndOrUndefined = t.union([t.array(list_and), t.undefined]);
-export type ListAndOrUndefined = t.TypeOf<typeof listAndOrUndefined>;

x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackaged_rules_schema.ts

@@ -40,16 +40,19 @@ import {
 } from '../common/schemas';
 /* eslint-enable @typescript-eslint/camelcase */
 
-import { DefaultStringArray } from '../types/default_string_array';
-import { DefaultActionsArray } from '../types/default_actions_array';
-import { DefaultBooleanFalse } from '../types/default_boolean_false';
-import { DefaultFromString } from '../types/default_from_string';
-import { DefaultIntervalString } from '../types/default_interval_string';
-import { DefaultMaxSignalsNumber } from '../types/default_max_signals_number';
-import { DefaultToString } from '../types/default_to_string';
-import { DefaultThreatArray } from '../types/default_threat_array';
-import { DefaultThrottleNull } from '../types/default_throttle_null';
-import { ListsDefaultArray, ListsDefaultArraySchema } from '../types/lists_default_array';
+import {
+  DefaultStringArray,
+  DefaultActionsArray,
+  DefaultBooleanFalse,
+  DefaultFromString,
+  DefaultIntervalString,
+  DefaultMaxSignalsNumber,
+  DefaultToString,
+  DefaultThreatArray,
+  DefaultThrottleNull,
+  DefaultListArray,
+  ListArray,
+} from '../types';
 
 /**
  * Big differences between this schema and the createRulesSchema
@@ -96,7 +99,7 @@ export const addPrepackagedRulesSchema = t.intersection([
       throttle: DefaultThrottleNull, // defaults to "null" if not set during decode
       references: DefaultStringArray, // defaults to empty array of strings if not set during decode
       note, // defaults to "undefined" if not set during decode
-      exceptions_list: ListsDefaultArray, // defaults to empty array if not set during decode
+      exceptions_list: DefaultListArray, // defaults to empty array if not set during decode
     })
   ),
 ]);
@@ -130,5 +133,5 @@ export type AddPrepackagedRulesSchemaDecoded = Omit<
   to: To;
   threat: Threat;
   throttle: ThrottleOrNull;
-  exceptions_list: ListsDefaultArraySchema;
+  exceptions_list: ListArray;
 };