[SIEM][Detection Engine] - Update UI to read rule exceptions_list

x-pack/plugins/lists/README.md

@@ -157,12 +157,14 @@ And you can attach exception list items like so:
     {
       "field": "actingProcess.file.signer",
       "operator": "included",
-      "match": "Elastic, N.V."
+      "type": "match",
+      "value": "Elastic, N.V."
     },
     {
       "field": "event.category",
       "operator": "included",
-      "match_any": [
+      "type": "match_any",
+      "value": [
         "process",
         "malware"
       ]

x-pack/plugins/lists/common/constants.mock.ts

@@ -46,10 +46,8 @@ export const EXISTS = 'exists';
 export const NESTED = 'nested';
 export const ENTRIES: EntriesArray = [
   {
-    entries: [
-      { field: 'some.not.nested.field', operator: 'included', type: 'match', value: 'some value' },
-    ],
-    field: 'some.field',
+    entries: [{ field: 'nested.field', operator: 'included', type: 'match', value: 'some value' }],
+    field: 'some.parentField',
     type: 'nested',
   },
   { field: 'some.not.nested.field', operator: 'included', type: 'match', value: 'some value' },

x-pack/plugins/lists/common/schemas/types/default_entries_array.test.ts

@@ -17,7 +17,8 @@ import { getEntriesArrayMock, getEntryMatchMock, getEntryNestedMock } from './en
 // it checks against every item in that union. Since entries consist of 5
 // different entry types, it returns 5 of these. To make more readable,
 // extracted here.
-const returnedSchemaError = `"Array<({| field: string, operator: "excluded" | "included", type: "match", value: string |} | {| field: string, operator: "excluded" | "included", type: "match_any", value: DefaultStringArray |} | {| field: string, operator: "excluded" | "included", type: "list", value: DefaultStringArray |} | {| field: string, operator: "excluded" | "included", type: "exists" |} | {| entries: Array<({| field: string, operator: "excluded" | "included", type: "match", value: string |} | {| field: string, operator: "excluded" | "included", type: "match_any", value: DefaultStringArray |} | {| field: string, operator: "excluded" | "included", type: "list", value: DefaultStringArray |} | {| field: string, operator: "excluded" | "included", type: "exists" |})>, field: string, type: "nested" |})>"`;
+const returnedSchemaError =
+  '"Array<({| field: string, operator: "excluded" | "included", type: "match", value: string |} | {| field: string, operator: "excluded" | "included", type: "match_any", value: DefaultStringArray |} | {| field: string, list: {| id: string, type: "ip" | "keyword" |}, operator: "excluded" | "included", type: "list" |} | {| field: string, operator: "excluded" | "included", type: "exists" |} | {| entries: Array<{| field: string, operator: "excluded" | "included", type: "match", value: string |}>, field: string, type: "nested" |})>"';
 
 describe('default_entries_array', () => {
   test('it should validate an empty array', () => {

x-pack/plugins/lists/common/schemas/types/default_namespace.test.ts

@@ -0,0 +1,61 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { pipe } from 'fp-ts/lib/pipeable';
+import { left } from 'fp-ts/lib/Either';
+
+import { foldLeftRight, getPaths } from '../../siem_common_deps';
+
+import { DefaultNamespace } from './default_namespace';
+
+describe('default_namespace', () => {
+  test('it should validate "single"', () => {
+    const payload = 'single';
+    const decoded = DefaultNamespace.decode(payload);
+    const message = pipe(decoded, foldLeftRight);
+
+    expect(getPaths(left(message.errors))).toEqual([]);
+    expect(message.schema).toEqual(payload);
+  });
+
+  test('it should validate "agnostic"', () => {
+    const payload = 'agnostic';
+    const decoded = DefaultNamespace.decode(payload);
+    const message = pipe(decoded, foldLeftRight);
+
+    expect(getPaths(left(message.errors))).toEqual([]);
+    expect(message.schema).toEqual(payload);
+  });
+
+  test('it defaults to "single" if "undefined"', () => {
+    const payload = undefined;
+    const decoded = DefaultNamespace.decode(payload);
+    const message = pipe(decoded, foldLeftRight);
+
+    expect(getPaths(left(message.errors))).toEqual([]);
+    expect(message.schema).toEqual('single');
+  });
+
+  test('it defaults to "single" if "null"', () => {
+    const payload = null;
+    const decoded = DefaultNamespace.decode(payload);
+    const message = pipe(decoded, foldLeftRight);
+
+    expect(getPaths(left(message.errors))).toEqual([]);
+    expect(message.schema).toEqual('single');
+  });
+
+  test('it should NOT validate if not "single" or "agnostic"', () => {
+    const payload = 'something else';
+    const decoded = DefaultNamespace.decode(payload);
+    const message = pipe(decoded, foldLeftRight);
+
+    expect(getPaths(left(message.errors))).toEqual([
+      `Invalid value "something else" supplied to "DefaultNamespace"`,
+    ]);
+    expect(message.schema).toEqual({});
+  });
+});

x-pack/plugins/lists/common/schemas/types/default_namespace.ts

@@ -7,7 +7,7 @@
 import * as t from 'io-ts';
 import { Either } from 'fp-ts/lib/Either';
 
-const namespaceType = t.keyof({ agnostic: null, single: null });
+export const namespaceType = t.keyof({ agnostic: null, single: null });
 
 type NamespaceType = t.TypeOf<typeof namespaceType>;
 

x-pack/plugins/lists/common/schemas/types/entries.mock.ts

@@ -9,10 +9,12 @@ import {
   EXISTS,
   FIELD,
   LIST,
+  LIST_ID,
   MATCH,
   MATCH_ANY,
   NESTED,
   OPERATOR,
+  TYPE,
 } from '../../constants.mock';
 
 import {
@@ -40,9 +42,9 @@ export const getEntryMatchAnyMock = (): EntryMatchAny => ({
 
 export const getEntryListMock = (): EntryList => ({
   field: FIELD,
+  list: { id: LIST_ID, type: TYPE },
   operator: OPERATOR,
   type: LIST,
-  value: [ENTRY_VALUE],
 });
 
 export const getEntryExistsMock = (): EntryExists => ({
@@ -52,7 +54,7 @@ export const getEntryExistsMock = (): EntryExists => ({
 });
 
 export const getEntryNestedMock = (): EntryNested => ({
-  entries: [getEntryMatchMock(), getEntryExistsMock()],
+  entries: [getEntryMatchMock(), getEntryMatchMock()],
   field: FIELD,
   type: NESTED,
 });

x-pack/plugins/lists/common/schemas/types/entries.test.ts

@@ -251,16 +251,16 @@ describe('Entries', () => {
       expect(message.schema).toEqual(payload);
     });
 
-    test('it should not validate when "value" is not string array', () => {
-      const payload: Omit<EntryList, 'value'> & { value: string } = {
+    test('it should not validate when "list" is not expected value', () => {
+      const payload: Omit<EntryList, 'list'> & { list: string } = {
         ...getEntryListMock(),
-        value: 'someListId',
+        list: 'someListId',
       };
       const decoded = entriesList.decode(payload);
       const message = pipe(decoded, foldLeftRight);
 
       expect(getPaths(left(message.errors))).toEqual([
-        'Invalid value "someListId" supplied to "value"',
+        'Invalid value "someListId" supplied to "list"',
       ]);
       expect(message.schema).toEqual({});
     });
@@ -338,6 +338,20 @@ describe('Entries', () => {
       expect(message.schema).toEqual({});
     });
 
+    test('it should NOT validate when "entries" contains an entry item that is not type "match"', () => {
+      const payload: Omit<EntryNested, 'entries'> & {
+        entries: EntryMatchAny[];
+      } = { ...getEntryNestedMock(), entries: [getEntryMatchAnyMock()] };
+      const decoded = entriesNested.decode(payload);
+      const message = pipe(decoded, foldLeftRight);
+
+      expect(getPaths(left(message.errors))).toEqual([
+        'Invalid value "match_any" supplied to "entries,type"',
+        'Invalid value "["some host name"]" supplied to "entries,value"',
+      ]);
+      expect(message.schema).toEqual({});
+    });
+
     test('it should strip out extra keys', () => {
       const payload: EntryNested & {
         extraKey?: string;

x-pack/plugins/lists/common/schemas/types/entries.ts

@@ -8,7 +8,7 @@
 
 import * as t from 'io-ts';
 
-import { operator } from '../common/schemas';
+import { operator, type } from '../common/schemas';
 import { DefaultStringArray } from '../../siem_common_deps';
 
 export const entriesMatch = t.exact(
@@ -34,9 +34,9 @@ export type EntryMatchAny = t.TypeOf<typeof entriesMatchAny>;
 export const entriesList = t.exact(
   t.type({
     field: t.string,
+    list: t.exact(t.type({ id: t.string, type })),
     operator,
     type: t.keyof({ list: null }),
-    value: DefaultStringArray,
   })
 );
 export type EntryList = t.TypeOf<typeof entriesList>;
@@ -52,7 +52,7 @@ export type EntryExists = t.TypeOf<typeof entriesExists>;
 
 export const entriesNested = t.exact(
   t.type({
-    entries: t.array(t.union([entriesMatch, entriesMatchAny, entriesList, entriesExists])),
+    entries: t.array(entriesMatch),
     field: t.string,
     type: t.keyof({ nested: null }),
   })

x-pack/plugins/lists/common/schemas/types/index.ts

@@ -5,5 +5,6 @@
  */
 export * from './default_comments_array';
 export * from './default_entries_array';
+export * from './default_namespace';
 export * from './comments';
 export * from './entries';

x-pack/plugins/lists/public/exceptions/hooks/use_exception_list.test.tsx

@@ -41,7 +41,7 @@ describe('useExceptionList', () => {
         useExceptionList({
           filterOptions: { filter: '', tags: [] },
           http: mockKibanaHttpService,
-          lists: [{ id: 'myListId', namespaceType: 'single' }],
+          lists: [{ id: 'myListId', namespace_type: 'single' }],
           onError: onErrorMock,
           pagination: {
             page: 1,
@@ -76,7 +76,7 @@ describe('useExceptionList', () => {
         useExceptionList({
           filterOptions: { filter: '', tags: [] },
           http: mockKibanaHttpService,
-          lists: [{ id: 'myListId', namespaceType: 'single' }],
+          lists: [{ id: 'myListId', namespace_type: 'single' }],
           onError: onErrorMock,
           onSuccess: onSuccessMock,
           pagination: {
@@ -131,7 +131,7 @@ describe('useExceptionList', () => {
           initialProps: {
             filterOptions: { filter: '', tags: [] },
             http: mockKibanaHttpService,
-            lists: [{ id: 'myListId', namespaceType: 'single' }],
+            lists: [{ id: 'myListId', namespace_type: 'single' }],
             onError: onErrorMock,
             onSuccess: onSuccessMock,
             pagination: {
@@ -146,7 +146,7 @@ describe('useExceptionList', () => {
       rerender({
         filterOptions: { filter: '', tags: [] },
         http: mockKibanaHttpService,
-        lists: [{ id: 'newListId', namespaceType: 'single' }],
+        lists: [{ id: 'newListId', namespace_type: 'single' }],
         onError: onErrorMock,
         onSuccess: onSuccessMock,
         pagination: {
@@ -173,7 +173,7 @@ describe('useExceptionList', () => {
         useExceptionList({
           filterOptions: { filter: '', tags: [] },
           http: mockKibanaHttpService,
-          lists: [{ id: 'myListId', namespaceType: 'single' }],
+          lists: [{ id: 'myListId', namespace_type: 'single' }],
           onError: onErrorMock,
           pagination: {
             page: 1,
@@ -210,7 +210,7 @@ describe('useExceptionList', () => {
           useExceptionList({
             filterOptions: { filter: '', tags: [] },
             http: mockKibanaHttpService,
-            lists: [{ id: 'myListId', namespaceType: 'single' }],
+            lists: [{ id: 'myListId', namespace_type: 'single' }],
             onError: onErrorMock,
             pagination: {
               page: 1,
@@ -238,7 +238,7 @@ describe('useExceptionList', () => {
           useExceptionList({
             filterOptions: { filter: '', tags: [] },
             http: mockKibanaHttpService,
-            lists: [{ id: 'myListId', namespaceType: 'single' }],
+            lists: [{ id: 'myListId', namespace_type: 'single' }],
             onError: onErrorMock,
             pagination: {
               page: 1,

x-pack/plugins/lists/public/exceptions/hooks/use_exception_list.tsx

@@ -73,25 +73,25 @@ export const useExceptionList = ({
         let exceptions: ExceptionListItemSchema[] = [];
         let exceptionListsReturned: ExceptionList[] = [];
 
-        const fetchData = async ({ id, namespaceType }: ExceptionIdentifiers): Promise<void> => {
+        const fetchData = async ({ id, namespace_type }: ExceptionIdentifiers): Promise<void> => {
           try {
             setLoading(true);
 
             const {
               list_id,
-              namespace_type,
+              namespace_type: namespaceType,
               ...restOfExceptionList
             } = await fetchExceptionListById({
               http,
               id,
-              namespaceType,
+              namespaceType: namespace_type,
               signal: abortCtrl.signal,
             });
             const fetchListItemsResult = await fetchExceptionListItemsByListId({
               filterOptions,
               http,
               listId: list_id,
-              namespaceType: namespace_type,
+              namespaceType,
               pagination,
               signal: abortCtrl.signal,
             });
@@ -147,8 +147,8 @@ export const useExceptionList = ({
         // TODO: Workaround for now. Once api updated, we can pass in array of lists to fetch
         await Promise.all(
           lists.map(
-            ({ id, namespaceType }: ExceptionIdentifiers): Promise<void> =>
-              fetchData({ id, namespaceType })
+            ({ id, namespace_type }: ExceptionIdentifiers): Promise<void> =>
+              fetchData({ id, namespace_type })
           )
         );
 

x-pack/plugins/lists/public/exceptions/types.ts

@@ -59,7 +59,7 @@ export interface UseExceptionListProps {
 
 export interface ExceptionIdentifiers {
   id: string;
-  namespaceType: NamespaceType;
+  namespace_type: NamespaceType;
   type?: string;
 }
 

x-pack/plugins/lists/server/index.ts

@@ -11,6 +11,7 @@ import { ListPlugin } from './plugin';
 
 // exporting these since its required at top level in siem plugin
 export { ListClient } from './services/lists/list_client';
+export { ExceptionListClient } from './services/exception_lists/exception_list_client';
 export { ListPluginSetup } from './types';
 
 export const config = { schema: ConfigSchema };

x-pack/plugins/lists/server/saved_objects/exception_list.ts

@@ -105,6 +105,16 @@ export const exceptionListItemMapping: SavedObjectsType['mappings'] = {
         field: {
           type: 'keyword',
         },
+        list: {
+          properties: {
+            id: {
+              type: 'keyword',
+            },
+            type: {
+              type: 'keyword',
+            },
+          },
+        },
         operator: {
           type: 'keyword',
         },

x-pack/plugins/lists/server/scripts/exception_lists/new/exception_list_item_with_list.json

@@ -0,0 +1,24 @@
+{
+  "list_id": "endpoint_list",
+  "item_id": "endpoint_list_item_lg_val_list",
+  "_tags": ["endpoint", "process", "malware", "os:windows"],
+  "tags": ["user added string for a tag", "malware"],
+  "type": "simple",
+  "description": "This is a sample exception list item with a large value list included",
+  "name": "Sample Endpoint Exception List Item with large value list",
+  "comments": [],
+  "entries": [
+    {
+      "field": "event.module",
+      "operator": "excluded",
+      "type": "match_any",
+      "value": ["zeek"]
+    },
+    {
+      "field": "source.ip",
+      "operator": "excluded",
+      "type": "list",
+      "list": { "id": "list-ip", "type": "ip" }
+    }
+  ]
+}

x-pack/plugins/lists/server/scripts/lists/new/list_ip_item.json

@@ -1,5 +1,5 @@
 {
   "id": "hand_inserted_item_id",
   "list_id": "list-ip",
-  "value": "127.0.0.1"
+  "value": "10.4.2.140"
 }

x-pack/plugins/security_solution/common/detection_engine/lists_common_deps.ts

@@ -0,0 +1,7 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export { EntriesArray, namespaceType } from '../../../lists/common/schemas';