Merge branch 'master' into user-allowlist-artifacts-pt3

x-pack/plugins/lists/common/constants.mock.ts

@@ -41,6 +41,8 @@ export const OPERATOR = 'included';
 export const ENTRY_VALUE = 'some host name';
 export const MATCH = 'match';
 export const MATCH_ANY = 'match_any';
+export const MAX_IMPORT_PAYLOAD_BYTES = 40000000;
+export const IMPORT_BUFFER_SIZE = 1000;
 export const LIST = 'list';
 export const EXISTS = 'exists';
 export const NESTED = 'nested';

x-pack/plugins/lists/server/config.mock.ts

@@ -0,0 +1,27 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import {
+  IMPORT_BUFFER_SIZE,
+  LIST_INDEX,
+  LIST_ITEM_INDEX,
+  MAX_IMPORT_PAYLOAD_BYTES,
+} from '../common/constants.mock';
+
+import { ConfigType } from './config';
+
+export const getConfigMock = (): Partial<ConfigType> => ({
+  listIndex: LIST_INDEX,
+  listItemIndex: LIST_ITEM_INDEX,
+});
+
+export const getConfigMockDecoded = (): ConfigType => ({
+  enabled: true,
+  importBufferSize: IMPORT_BUFFER_SIZE,
+  listIndex: LIST_INDEX,
+  listItemIndex: LIST_ITEM_INDEX,
+  maxImportPayloadBytes: MAX_IMPORT_PAYLOAD_BYTES,
+});

x-pack/plugins/lists/server/config.test.ts

@@ -0,0 +1,64 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { ConfigSchema, ConfigType } from './config';
+import { getConfigMock, getConfigMockDecoded } from './config.mock';
+
+describe('config_schema', () => {
+  test('it works with expected basic mock data set and defaults', () => {
+    expect(ConfigSchema.validate(getConfigMock())).toEqual(getConfigMockDecoded());
+  });
+
+  test('it throws if given an invalid value', () => {
+    const mock: Partial<ConfigType> & { madeUpValue: string } = {
+      madeUpValue: 'something',
+      ...getConfigMock(),
+    };
+    expect(() => ConfigSchema.validate(mock)).toThrow(
+      '[madeUpValue]: definition for this key is missing'
+    );
+  });
+
+  test('it throws if the "maxImportPayloadBytes" value is 0', () => {
+    const mock: ConfigType = {
+      ...getConfigMockDecoded(),
+      maxImportPayloadBytes: 0,
+    };
+    expect(() => ConfigSchema.validate(mock)).toThrow(
+      '[maxImportPayloadBytes]: Value must be equal to or greater than [1].'
+    );
+  });
+
+  test('it throws if the "maxImportPayloadBytes" value is less than 0', () => {
+    const mock: ConfigType = {
+      ...getConfigMockDecoded(),
+      maxImportPayloadBytes: -1,
+    };
+    expect(() => ConfigSchema.validate(mock)).toThrow(
+      '[maxImportPayloadBytes]: Value must be equal to or greater than [1].'
+    );
+  });
+
+  test('it throws if the "importBufferSize" value is 0', () => {
+    const mock: ConfigType = {
+      ...getConfigMockDecoded(),
+      importBufferSize: 0,
+    };
+    expect(() => ConfigSchema.validate(mock)).toThrow(
+      '[importBufferSize]: Value must be equal to or greater than [1].'
+    );
+  });
+
+  test('it throws if the "importBufferSize" value is less than 0', () => {
+    const mock: ConfigType = {
+      ...getConfigMockDecoded(),
+      importBufferSize: -1,
+    };
+    expect(() => ConfigSchema.validate(mock)).toThrow(
+      '[importBufferSize]: Value must be equal to or greater than [1].'
+    );
+  });
+});

x-pack/plugins/lists/server/config.ts

@@ -8,8 +8,10 @@ import { TypeOf, schema } from '@kbn/config-schema';
 
 export const ConfigSchema = schema.object({
   enabled: schema.boolean({ defaultValue: true }),
+  importBufferSize: schema.number({ defaultValue: 1000, min: 1 }),
   listIndex: schema.string({ defaultValue: '.lists' }),
   listItemIndex: schema.string({ defaultValue: '.items' }),
+  maxImportPayloadBytes: schema.number({ defaultValue: 40000000, min: 1 }),
 });
 
 export type ConfigType = TypeOf<typeof ConfigSchema>;

x-pack/plugins/lists/server/create_config.ts

@@ -12,12 +12,6 @@ import { ConfigType } from './config';
 
 export const createConfig$ = (
   context: PluginInitializerContext
-): Observable<
-  Readonly<{
-    enabled: boolean;
-    listIndex: string;
-    listItemIndex: string;
-  }>
-> => {
+): Observable<Readonly<ConfigType>> => {
   return context.config.create<ConfigType>().pipe(map((config) => config));
 };

x-pack/plugins/lists/server/plugin.ts

@@ -48,7 +48,7 @@ export class ListPlugin
 
     core.http.registerRouteHandlerContext('lists', this.createRouteHandlerContext());
     const router = core.http.createRouter();
-    initRoutes(router);
+    initRoutes(router, config);
 
     return {
       getExceptionListClient: (savedObjectsClient, user): ExceptionListClient => {

x-pack/plugins/lists/server/routes/import_list_item_route.ts

@@ -4,50 +4,40 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { Readable } from 'stream';
-
 import { IRouter } from 'kibana/server';
+import { schema } from '@kbn/config-schema';
 
 import { LIST_ITEM_URL } from '../../common/constants';
 import { buildRouteValidation, buildSiemResponse, transformError } from '../siem_server_deps';
 import { validate } from '../../common/siem_common_deps';
-import { importListItemQuerySchema, importListItemSchema, listSchema } from '../../common/schemas';
-
-import { getListClient } from '.';
+import { importListItemQuerySchema, listSchema } from '../../common/schemas';
+import { ConfigType } from '../config';
 
-export interface HapiReadableStream extends Readable {
-  hapi: {
-    filename: string;
-  };
-}
+import { createStreamFromBuffer } from './utils/create_stream_from_buffer';
 
-/**
- * Special interface since we are streaming in a file through a reader
- */
-export interface ImportListItemHapiFileSchema {
-  file: HapiReadableStream;
-}
+import { getListClient } from '.';
 
-export const importListItemRoute = (router: IRouter): void => {
+export const importListItemRoute = (router: IRouter, config: ConfigType): void => {
   router.post(
     {
       options: {
         body: {
-          output: 'stream',
+          accepts: ['multipart/form-data'],
+          maxBytes: config.maxImportPayloadBytes,
+          parse: false,
         },
         tags: ['access:lists'],
       },
       path: `${LIST_ITEM_URL}/_import`,
       validate: {
-        body: buildRouteValidation<typeof importListItemSchema, ImportListItemHapiFileSchema>(
-          importListItemSchema
-        ),
+        body: schema.buffer(),
         query: buildRouteValidation(importListItemQuerySchema),
       },
     },
     async (context, request, response) => {
       const siemResponse = buildSiemResponse(response);
       try {
+        const stream = createStreamFromBuffer(request.body);
         const { deserializer, list_id: listId, serializer, type } = request.query;
         const lists = getListClient(context);
         if (listId != null) {
@@ -63,7 +53,7 @@ export const importListItemRoute = (router: IRouter): void => {
             listId,
             meta: undefined,
             serializer: list.serializer,
-            stream: request.body.file,
+            stream,
             type: list.type,
           });
 
@@ -74,26 +64,21 @@ export const importListItemRoute = (router: IRouter): void => {
             return response.ok({ body: validated ?? {} });
           }
         } else if (type != null) {
-          const { filename } = request.body.file.hapi;
-          // TODO: Should we prevent the same file from being uploaded multiple times?
-          const list = await lists.createListIfItDoesNotExist({
-            description: `File uploaded from file system of ${filename}`,
+          const importedList = await lists.importListItemsToStream({
             deserializer,
-            id: filename,
+            listId: undefined,
             meta: undefined,
-            name: filename,
             serializer,
+            stream,
             type,
           });
-          await lists.importListItemsToStream({
-            deserializer: list.deserializer,
-            listId: list.id,
-            meta: undefined,
-            serializer: list.serializer,
-            stream: request.body.file,
-            type: list.type,
-          });
-          const [validated, errors] = validate(list, listSchema);
+          if (importedList == null) {
+            return siemResponse.error({
+              body: 'Unable to parse a valid fileName during import',
+              statusCode: 400,
+            });
+          }
+          const [validated, errors] = validate(importedList, listSchema);
           if (errors != null) {
             return siemResponse.error({ body: errors, statusCode: 500 });
           } else {

x-pack/plugins/lists/server/routes/init_routes.ts

@@ -6,6 +6,8 @@
 
 import { IRouter } from 'kibana/server';
 
+import { ConfigType } from '../config';
+
 import {
   createExceptionListItemRoute,
   createExceptionListRoute,
@@ -36,7 +38,7 @@ import {
   updateListRoute,
 } from '.';
 
-export const initRoutes = (router: IRouter): void => {
+export const initRoutes = (router: IRouter, config: ConfigType): void => {
   // lists
   createListRoute(router);
   readListRoute(router);
@@ -52,7 +54,7 @@ export const initRoutes = (router: IRouter): void => {
   deleteListItemRoute(router);
   patchListItemRoute(router);
   exportListItemRoute(router);
-  importListItemRoute(router);
+  importListItemRoute(router, config);
   findListItemRoute(router);
 
   // indexes of lists

x-pack/plugins/lists/server/routes/utils/create_stream_from_buffer.ts

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+import { Readable } from 'stream';
+
+export const createStreamFromBuffer = (buffer: Buffer): Readable => {
+  const stream = new Readable();
+  stream.push(buffer);
+  stream.push(null);
+  return stream;
+};