[SIEM][Detections] Adds large list support using REST endpoints

.eslintrc.js

@@ -730,6 +730,101 @@ module.exports = {
       },
     },
 
+    /**
+     * Lists overrides
+     */
+    {
+      // typescript and javascript for front and back end
+      files: ['x-pack/plugins/lists/**/*.{js,ts,tsx}'],
+      plugins: ['eslint-plugin-node'],
+      env: {
+        mocha: true,
+        jest: true,
+      },
+      rules: {
+        'accessor-pairs': 'error',
+        'array-callback-return': 'error',
+        'no-array-constructor': 'error',
+        complexity: 'error',
+        'consistent-return': 'error',
+        'func-style': ['error', 'expression'],
+        'import/order': [
+          'error',
+          {
+            groups: ['builtin', 'external', 'internal', 'parent', 'sibling', 'index'],
+            'newlines-between': 'always',
+          },
+        ],
+        'sort-imports': [
+          'error',
+          {
+            ignoreDeclarationSort: true,
+          },
+        ],
+        'node/no-deprecated-api': 'error',
+        'no-bitwise': 'error',
+        'no-continue': 'error',
+        'no-dupe-keys': 'error',
+        'no-duplicate-case': 'error',
+        'no-duplicate-imports': 'error',
+        'no-empty-character-class': 'error',
+        'no-empty-pattern': 'error',
+        'no-ex-assign': 'error',
+        'no-extend-native': 'error',
+        'no-extra-bind': 'error',
+        'no-extra-boolean-cast': 'error',
+        'no-extra-label': 'error',
+        'no-func-assign': 'error',
+        'no-implicit-globals': 'error',
+        'no-implied-eval': 'error',
+        'no-invalid-regexp': 'error',
+        'no-inner-declarations': 'error',
+        'no-lone-blocks': 'error',
+        'no-multi-assign': 'error',
+        'no-misleading-character-class': 'error',
+        'no-new-symbol': 'error',
+        'no-obj-calls': 'error',
+        'no-param-reassign': 'error',
+        'no-process-exit': 'error',
+        'no-prototype-builtins': 'error',
+        'no-return-await': 'error',
+        'no-self-compare': 'error',
+        'no-shadow-restricted-names': 'error',
+        'no-sparse-arrays': 'error',
+        'no-this-before-super': 'error',
+        'no-undef': 'error',
+        'no-unreachable': 'error',
+        'no-unsafe-finally': 'error',
+        'no-useless-call': 'error',
+        'no-useless-catch': 'error',
+        'no-useless-concat': 'error',
+        'no-useless-computed-key': 'error',
+        'no-useless-escape': 'error',
+        'no-useless-rename': 'error',
+        'no-useless-return': 'error',
+        'no-void': 'error',
+        'one-var-declaration-per-line': 'error',
+        'prefer-object-spread': 'error',
+        'prefer-promise-reject-errors': 'error',
+        'prefer-rest-params': 'error',
+        'prefer-spread': 'error',
+        'prefer-template': 'error',
+        'require-atomic-updates': 'error',
+        'symbol-description': 'error',
+        'vars-on-top': 'error',
+        '@typescript-eslint/explicit-member-accessibility': 'error',
+        '@typescript-eslint/no-this-alias': 'error',
+        '@typescript-eslint/no-explicit-any': 'error',
+        '@typescript-eslint/no-useless-constructor': 'error',
+        '@typescript-eslint/unified-signatures': 'error',
+        '@typescript-eslint/explicit-function-return-type': 'error',
+        '@typescript-eslint/no-non-null-assertion': 'error',
+        '@typescript-eslint/no-unused-vars': 'error',
+        'no-template-curly-in-string': 'error',
+        'sort-keys': 'error',
+      },
+    },
+
     /**
      * Lens overrides
      */

.github/CODEOWNERS

@@ -222,7 +222,7 @@
 /x-pack/test/detection_engine_api_integration @elastic/siem
 /x-pack/test/api_integration/apis/siem @elastic/siem
 /x-pack/plugins/case @elastic/siem
+/x-pack/plugins/lists @elastic/siem
 
 # Security Intelligence And Analytics
-/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/prepackaged_rules @elastic/security-intelligence-analytics
 /x-pack/plugins/siem/server/lib/detection_engine/rules/prepackaged_rules @elastic/security-intelligence-analytics

x-pack/plugins/lists/common/constants.ts

@@ -0,0 +1,12 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+/**
+ * Lists routes
+ */
+export const LIST_URL = `/api/lists`;
+export const LIST_INDEX = `${LIST_URL}/index`;
+export const LIST_ITEM_URL = `${LIST_URL}/items`;

x-pack/plugins/lists/common/schemas/common/index.ts

@@ -0,0 +1,7 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export * from './schemas';

x-pack/plugins/lists/common/schemas/common/schemas.ts

@@ -0,0 +1,62 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+/* eslint-disable @typescript-eslint/camelcase */
+
+import * as t from 'io-ts';
+
+import { NonEmptyString } from '../types/non_empty_string';
+
+export const name = t.string;
+export type Name = t.TypeOf<typeof name>;
+export const nameOrUndefined = t.union([name, t.undefined]);
+export type NameOrUndefined = t.TypeOf<typeof nameOrUndefined>;
+
+export const description = t.string;
+export type Description = t.TypeOf<typeof description>;
+export const descriptionOrUndefined = t.union([description, t.undefined]);
+export type DescriptionOrUndefined = t.TypeOf<typeof descriptionOrUndefined>;
+
+export const list_id = NonEmptyString;
+export const list_idOrUndefined = t.union([list_id, t.undefined]);
+export type List_idOrUndefined = t.TypeOf<typeof list_idOrUndefined>;
+
+export const item = t.string;
+export const created_at = t.string; // TODO: Make this into an ISO Date string check
+export const updated_at = t.string; // TODO: Make this into an ISO Date string check
+export const updated_by = t.string;
+export const created_by = t.string;
+export const file = t.object;
+
+export const id = NonEmptyString;
+export type Id = t.TypeOf<typeof id>;
+export const idOrUndefined = t.union([id, t.undefined]);
+export type IdOrUndefined = t.TypeOf<typeof idOrUndefined>;
+
+export const ip = t.string;
+export const ipOrUndefined = t.union([ip, t.undefined]);
+
+export const keyword = t.string;
+export const keywordOrUndefined = t.union([keyword, t.undefined]);
+
+export const value = t.string;
+export const valueOrUndefined = t.union([value, t.undefined]);
+
+export const tie_breaker_id = t.string; // TODO: Use UUID for this instead of a string for validation
+export const _index = t.string;
+
+export const type = t.keyof({ ip: null, keyword: null }); // TODO: Add the other data types here
+
+export const typeOrUndefined = t.union([type, t.undefined]);
+export type Type = t.TypeOf<typeof type>;
+
+export const meta = t.object;
+export type Meta = t.TypeOf<typeof meta>;
+export const metaOrUndefined = t.union([meta, t.undefined]);
+export type MetaOrUndefined = t.TypeOf<typeof metaOrUndefined>;
+
+export const esDataTypeUnion = t.union([t.type({ ip }), t.type({ keyword })]);
+export type EsDataTypeUnion = t.TypeOf<typeof esDataTypeUnion>;

x-pack/plugins/lists/common/schemas/elastic_query/create_es_bulk_type.ts

@@ -0,0 +1,17 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import * as t from 'io-ts';
+
+import { _index } from '../common/schemas';
+
+export const createEsBulkTypeSchema = t.exact(
+  t.type({
+    create: t.exact(t.type({ _index })),
+  })
+);
+
+export type CreateEsBulkTypeSchema = t.TypeOf<typeof createEsBulkTypeSchema>;

x-pack/plugins/lists/common/schemas/elastic_query/create_es_lists_items_schema.ts

@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+/* eslint-disable @typescript-eslint/camelcase */
+
+import * as t from 'io-ts';
+
+import {
+  created_at,
+  created_by,
+  esDataTypeUnion,
+  list_id,
+  metaOrUndefined,
+  tie_breaker_id,
+  updated_at,
+  updated_by,
+} from '../common/schemas';
+
+export const createEsListsItemsSchema = t.intersection([
+  t.exact(
+    t.type({
+      created_at,
+      created_by,
+      list_id,
+      meta: metaOrUndefined,
+      tie_breaker_id,
+      updated_at,
+      updated_by,
+    })
+  ),
+  esDataTypeUnion,
+]);
+
+export type CreateEsListsItemsSchema = t.TypeOf<typeof createEsListsItemsSchema>;

x-pack/plugins/lists/common/schemas/elastic_query/index.ts

@@ -0,0 +1,10 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+export * from './update_es_lists_schema';
+export * from './index_es_lists_schema';
+export * from './update_es_lists_items_schema';
+export * from './create_es_lists_items_schema';
+export * from './create_es_bulk_type';

x-pack/plugins/lists/common/schemas/elastic_query/index_es_lists_schema.ts

@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+/* eslint-disable @typescript-eslint/camelcase */
+
+import * as t from 'io-ts';
+
+import {
+  created_at,
+  created_by,
+  description,
+  metaOrUndefined,
+  name,
+  tie_breaker_id,
+  type,
+  updated_at,
+  updated_by,
+} from '../common/schemas';
+
+export const indexEsListsSchema = t.exact(
+  t.type({
+    created_at,
+    created_by,
+    description,
+    meta: metaOrUndefined,
+    name,
+    tie_breaker_id,
+    type,
+    updated_at,
+    updated_by,
+  })
+);
+
+export type IndexEsListsSchema = t.TypeOf<typeof indexEsListsSchema>;

x-pack/plugins/lists/common/schemas/elastic_query/update_es_lists_items_schema.ts

@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+/* eslint-disable @typescript-eslint/camelcase */
+
+import * as t from 'io-ts';
+
+import { esDataTypeUnion, metaOrUndefined, updated_at, updated_by } from '../common/schemas';
+
+export const updateEsListsItemsSchema = t.intersection([
+  t.exact(
+    t.type({
+      meta: metaOrUndefined,
+      updated_at,
+      updated_by,
+    })
+  ),
+  esDataTypeUnion,
+]);
+
+export type UpdateEsListsItemsSchema = t.TypeOf<typeof updateEsListsItemsSchema>;

x-pack/plugins/lists/common/schemas/elastic_query/update_es_lists_schema.ts

@@ -0,0 +1,29 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+/* eslint-disable @typescript-eslint/camelcase */
+
+import * as t from 'io-ts';
+
+import {
+  descriptionOrUndefined,
+  metaOrUndefined,
+  nameOrUndefined,
+  updated_at,
+  updated_by,
+} from '../common/schemas';
+
+export const updateEsListsSchema = t.exact(
+  t.type({
+    description: descriptionOrUndefined,
+    meta: metaOrUndefined,
+    name: nameOrUndefined,
+    updated_at,
+    updated_by,
+  })
+);
+
+export type UpdateEsListsSchema = t.TypeOf<typeof updateEsListsSchema>;

x-pack/plugins/lists/common/schemas/elastic_response/index.ts

@@ -0,0 +1,8 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export * from './search_es_lists_items_schema';
+export * from './search_es_lists_schema';