[SIEM][Detection Engine] - Update DE to work with new exceptions sche…

…ma (elastic#69715)

* Updates list entry schema, exposes exception list client, updates tests

* create new de list schema and unit tests

* updated route unit tests and types to match new list schema

* updated existing DE exceptions code so it should now work as is with updated schema

* test and types cleanup

* cleanup

* update unit test

* updates per feedback

x-pack/plugins/lists/README.md

@@ -157,12 +157,14 @@ And you can attach exception list items like so:
     {
       "field": "actingProcess.file.signer",
       "operator": "included",
-      "match": "Elastic, N.V."
+      "type": "match",
+      "value": "Elastic, N.V."
     },
     {
       "field": "event.category",
       "operator": "included",
-      "match_any": [
+      "type": "match_any",
+      "value": [
         "process",
         "malware"
       ]

x-pack/plugins/lists/common/constants.mock.ts

@@ -46,10 +46,8 @@ export const EXISTS = 'exists';
 export const NESTED = 'nested';
 export const ENTRIES: EntriesArray = [
   {
-    entries: [
-      { field: 'some.not.nested.field', operator: 'included', type: 'match', value: 'some value' },
-    ],
-    field: 'some.field',
+    entries: [{ field: 'nested.field', operator: 'included', type: 'match', value: 'some value' }],
+    field: 'some.parentField',
     type: 'nested',
   },
   { field: 'some.not.nested.field', operator: 'included', type: 'match', value: 'some value' },

x-pack/plugins/lists/common/schemas/types/default_entries_array.test.ts

@@ -17,7 +17,8 @@ import { getEntriesArrayMock, getEntryMatchMock, getEntryNestedMock } from './en
 // it checks against every item in that union. Since entries consist of 5
 // different entry types, it returns 5 of these. To make more readable,
 // extracted here.
-const returnedSchemaError = `"Array<({| field: string, operator: "excluded" | "included", type: "match", value: string |} | {| field: string, operator: "excluded" | "included", type: "match_any", value: DefaultStringArray |} | {| field: string, operator: "excluded" | "included", type: "list", value: DefaultStringArray |} | {| field: string, operator: "excluded" | "included", type: "exists" |} | {| entries: Array<({| field: string, operator: "excluded" | "included", type: "match", value: string |} | {| field: string, operator: "excluded" | "included", type: "match_any", value: DefaultStringArray |} | {| field: string, operator: "excluded" | "included", type: "list", value: DefaultStringArray |} | {| field: string, operator: "excluded" | "included", type: "exists" |})>, field: string, type: "nested" |})>"`;
+const returnedSchemaError =
+  '"Array<({| field: string, operator: "excluded" | "included", type: "match", value: string |} | {| field: string, operator: "excluded" | "included", type: "match_any", value: DefaultStringArray |} | {| field: string, list: {| id: string, type: "ip" | "keyword" |}, operator: "excluded" | "included", type: "list" |} | {| field: string, operator: "excluded" | "included", type: "exists" |} | {| entries: Array<{| field: string, operator: "excluded" | "included", type: "match", value: string |}>, field: string, type: "nested" |})>"';
 
 describe('default_entries_array', () => {
   test('it should validate an empty array', () => {

x-pack/plugins/lists/common/schemas/types/default_namespace.test.ts

@@ -0,0 +1,61 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { pipe } from 'fp-ts/lib/pipeable';
+import { left } from 'fp-ts/lib/Either';
+
+import { foldLeftRight, getPaths } from '../../siem_common_deps';
+
+import { DefaultNamespace } from './default_namespace';
+
+describe('default_namespace', () => {
+  test('it should validate "single"', () => {
+    const payload = 'single';
+    const decoded = DefaultNamespace.decode(payload);
+    const message = pipe(decoded, foldLeftRight);
+
+    expect(getPaths(left(message.errors))).toEqual([]);
+    expect(message.schema).toEqual(payload);
+  });
+
+  test('it should validate "agnostic"', () => {
+    const payload = 'agnostic';
+    const decoded = DefaultNamespace.decode(payload);
+    const message = pipe(decoded, foldLeftRight);
+
+    expect(getPaths(left(message.errors))).toEqual([]);
+    expect(message.schema).toEqual(payload);
+  });
+
+  test('it defaults to "single" if "undefined"', () => {
+    const payload = undefined;
+    const decoded = DefaultNamespace.decode(payload);
+    const message = pipe(decoded, foldLeftRight);
+
+    expect(getPaths(left(message.errors))).toEqual([]);
+    expect(message.schema).toEqual('single');
+  });
+
+  test('it defaults to "single" if "null"', () => {
+    const payload = null;
+    const decoded = DefaultNamespace.decode(payload);
+    const message = pipe(decoded, foldLeftRight);
+
+    expect(getPaths(left(message.errors))).toEqual([]);
+    expect(message.schema).toEqual('single');
+  });
+
+  test('it should NOT validate if not "single" or "agnostic"', () => {
+    const payload = 'something else';
+    const decoded = DefaultNamespace.decode(payload);
+    const message = pipe(decoded, foldLeftRight);
+
+    expect(getPaths(left(message.errors))).toEqual([
+      `Invalid value "something else" supplied to "DefaultNamespace"`,
+    ]);
+    expect(message.schema).toEqual({});
+  });
+});

x-pack/plugins/lists/common/schemas/types/default_namespace.ts

@@ -7,7 +7,7 @@
 import * as t from 'io-ts';
 import { Either } from 'fp-ts/lib/Either';
 
-const namespaceType = t.keyof({ agnostic: null, single: null });
+export const namespaceType = t.keyof({ agnostic: null, single: null });
 
 type NamespaceType = t.TypeOf<typeof namespaceType>;
 

x-pack/plugins/lists/common/schemas/types/entries.mock.ts

@@ -9,10 +9,12 @@ import {
   EXISTS,
   FIELD,
   LIST,
+  LIST_ID,
   MATCH,
   MATCH_ANY,
   NESTED,
   OPERATOR,
+  TYPE,
 } from '../../constants.mock';
 
 import {
@@ -40,9 +42,9 @@ export const getEntryMatchAnyMock = (): EntryMatchAny => ({
 
 export const getEntryListMock = (): EntryList => ({
   field: FIELD,
+  list: { id: LIST_ID, type: TYPE },
   operator: OPERATOR,
   type: LIST,
-  value: [ENTRY_VALUE],
 });
 
 export const getEntryExistsMock = (): EntryExists => ({
@@ -52,7 +54,7 @@ export const getEntryExistsMock = (): EntryExists => ({
 });
 
 export const getEntryNestedMock = (): EntryNested => ({
-  entries: [getEntryMatchMock(), getEntryExistsMock()],
+  entries: [getEntryMatchMock(), getEntryMatchMock()],
   field: FIELD,
   type: NESTED,
 });

x-pack/plugins/lists/common/schemas/types/entries.test.ts

@@ -251,16 +251,16 @@ describe('Entries', () => {
       expect(message.schema).toEqual(payload);
     });
 
-    test('it should not validate when "value" is not string array', () => {
-      const payload: Omit<EntryList, 'value'> & { value: string } = {
+    test('it should not validate when "list" is not expected value', () => {
+      const payload: Omit<EntryList, 'list'> & { list: string } = {
         ...getEntryListMock(),
-        value: 'someListId',
+        list: 'someListId',
       };
       const decoded = entriesList.decode(payload);
       const message = pipe(decoded, foldLeftRight);
 
       expect(getPaths(left(message.errors))).toEqual([
-        'Invalid value "someListId" supplied to "value"',
+        'Invalid value "someListId" supplied to "list"',
       ]);
       expect(message.schema).toEqual({});
     });
@@ -338,6 +338,20 @@ describe('Entries', () => {
       expect(message.schema).toEqual({});
     });
 
+    test('it should NOT validate when "entries" contains an entry item that is not type "match"', () => {
+      const payload: Omit<EntryNested, 'entries'> & {
+        entries: EntryMatchAny[];
+      } = { ...getEntryNestedMock(), entries: [getEntryMatchAnyMock()] };
+      const decoded = entriesNested.decode(payload);
+      const message = pipe(decoded, foldLeftRight);
+
+      expect(getPaths(left(message.errors))).toEqual([
+        'Invalid value "match_any" supplied to "entries,type"',
+        'Invalid value "["some host name"]" supplied to "entries,value"',
+      ]);
+      expect(message.schema).toEqual({});
+    });
+
     test('it should strip out extra keys', () => {
       const payload: EntryNested & {
         extraKey?: string;

x-pack/plugins/lists/common/schemas/types/entries.ts

@@ -8,7 +8,7 @@
 
 import * as t from 'io-ts';
 
-import { operator } from '../common/schemas';
+import { operator, type } from '../common/schemas';
 import { DefaultStringArray } from '../../siem_common_deps';
 
 export const entriesMatch = t.exact(
@@ -34,9 +34,9 @@ export type EntryMatchAny = t.TypeOf<typeof entriesMatchAny>;
 export const entriesList = t.exact(
   t.type({
     field: t.string,
+    list: t.exact(t.type({ id: t.string, type })),
     operator,
     type: t.keyof({ list: null }),
-    value: DefaultStringArray,
   })
 );
 export type EntryList = t.TypeOf<typeof entriesList>;
@@ -52,7 +52,7 @@ export type EntryExists = t.TypeOf<typeof entriesExists>;
 
 export const entriesNested = t.exact(
   t.type({
-    entries: t.array(t.union([entriesMatch, entriesMatchAny, entriesList, entriesExists])),
+    entries: t.array(entriesMatch),
     field: t.string,
     type: t.keyof({ nested: null }),
   })

x-pack/plugins/lists/common/schemas/types/index.ts

@@ -5,5 +5,6 @@
  */
 export * from './default_comments_array';
 export * from './default_entries_array';
+export * from './default_namespace';
 export * from './comments';
 export * from './entries';

x-pack/plugins/lists/server/index.ts

@@ -11,6 +11,7 @@ import { ListPlugin } from './plugin';
 
 // exporting these since its required at top level in siem plugin
 export { ListClient } from './services/lists/list_client';
+export { ExceptionListClient } from './services/exception_lists/exception_list_client';
 export { ListPluginSetup } from './types';
 
 export const config = { schema: ConfigSchema };

x-pack/plugins/lists/server/saved_objects/exception_list.ts

@@ -105,6 +105,16 @@ export const exceptionListItemMapping: SavedObjectsType['mappings'] = {
         field: {
           type: 'keyword',
         },
+        list: {
+          properties: {
+            id: {
+              type: 'keyword',
+            },
+            type: {
+              type: 'keyword',
+            },
+          },
+        },
         operator: {
           type: 'keyword',
         },

x-pack/plugins/lists/server/scripts/exception_lists/new/exception_list_item_with_list.json

@@ -0,0 +1,24 @@
+{
+  "list_id": "endpoint_list",
+  "item_id": "endpoint_list_item_lg_val_list",
+  "_tags": ["endpoint", "process", "malware", "os:windows"],
+  "tags": ["user added string for a tag", "malware"],
+  "type": "simple",
+  "description": "This is a sample exception list item with a large value list included",
+  "name": "Sample Endpoint Exception List Item with large value list",
+  "comments": [],
+  "entries": [
+    {
+      "field": "event.module",
+      "operator": "excluded",
+      "type": "match_any",
+      "value": ["zeek"]
+    },
+    {
+      "field": "source.ip",
+      "operator": "excluded",
+      "type": "list",
+      "list": { "id": "list-ip", "type": "ip" }
+    }
+  ]
+}

x-pack/plugins/lists/server/scripts/lists/new/list_ip_item.json

@@ -1,5 +1,5 @@
 {
   "id": "hand_inserted_item_id",
   "list_id": "list-ip",
-  "value": "127.0.0.1"
+  "value": "10.4.2.140"
 }

x-pack/plugins/security_solution/common/detection_engine/lists_common_deps.ts

@@ -0,0 +1,7 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export { EntriesArray, namespaceType } from '../../../lists/common/schemas';

x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts

@@ -341,40 +341,3 @@ export type Note = t.TypeOf<typeof note>;
 
 export const noteOrUndefined = t.union([note, t.undefined]);
 export type NoteOrUndefined = t.TypeOf<typeof noteOrUndefined>;
-
-// NOTE: Experimental list support not being shipped currently and behind a feature flag
-// TODO: Remove this comment once we lists have passed testing and is ready for the release
-export const list_field = t.string;
-export const list_values_operator = t.keyof({ included: null, excluded: null });
-export const list_values_type = t.keyof({ match: null, match_all: null, list: null, exists: null });
-export const list_values = t.exact(
-  t.intersection([
-    t.type({
-      name: t.string,
-    }),
-    t.partial({
-      id: t.string,
-      description: t.string,
-      created_at,
-    }),
-  ])
-);
-export const list = t.exact(
-  t.intersection([
-    t.type({
-      field: t.string,
-      values_operator: list_values_operator,
-      values_type: list_values_type,
-    }),
-    t.partial({ values: t.array(list_values) }),
-  ])
-);
-export const list_and = t.intersection([
-  list,
-  t.partial({
-    and: t.array(list),
-  }),
-]);
-
-export const listAndOrUndefined = t.union([t.array(list_and), t.undefined]);
-export type ListAndOrUndefined = t.TypeOf<typeof listAndOrUndefined>;

x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackaged_rules_schema.ts

@@ -40,16 +40,19 @@ import {
 } from '../common/schemas';
 /* eslint-enable @typescript-eslint/camelcase */
 
-import { DefaultStringArray } from '../types/default_string_array';
-import { DefaultActionsArray } from '../types/default_actions_array';
-import { DefaultBooleanFalse } from '../types/default_boolean_false';
-import { DefaultFromString } from '../types/default_from_string';
-import { DefaultIntervalString } from '../types/default_interval_string';
-import { DefaultMaxSignalsNumber } from '../types/default_max_signals_number';
-import { DefaultToString } from '../types/default_to_string';
-import { DefaultThreatArray } from '../types/default_threat_array';
-import { DefaultThrottleNull } from '../types/default_throttle_null';
-import { ListsDefaultArray, ListsDefaultArraySchema } from '../types/lists_default_array';
+import {
+  DefaultStringArray,
+  DefaultActionsArray,
+  DefaultBooleanFalse,
+  DefaultFromString,
+  DefaultIntervalString,
+  DefaultMaxSignalsNumber,
+  DefaultToString,
+  DefaultThreatArray,
+  DefaultThrottleNull,
+  DefaultListArray,
+  ListArray,
+} from '../types';
 
 /**
  * Big differences between this schema and the createRulesSchema
@@ -96,7 +99,7 @@ export const addPrepackagedRulesSchema = t.intersection([
       throttle: DefaultThrottleNull, // defaults to "null" if not set during decode
       references: DefaultStringArray, // defaults to empty array of strings if not set during decode
       note, // defaults to "undefined" if not set during decode
-      exceptions_list: ListsDefaultArray, // defaults to empty array if not set during decode
+      exceptions_list: DefaultListArray, // defaults to empty array if not set during decode
     })
   ),
 ]);
@@ -130,5 +133,5 @@ export type AddPrepackagedRulesSchemaDecoded = Omit<
   to: To;
   threat: Threat;
   throttle: ThrottleOrNull;
-  exceptions_list: ListsDefaultArraySchema;
+  exceptions_list: ListArray;
 };