From ee11235c0a564e3d6b471c87a273d1e1f9b5aee5 Mon Sep 17 00:00:00 2001 From: Angela Chuang Date: Wed, 1 Apr 2020 00:31:50 +0100 Subject: [PATCH 01/21] move away from joi --- .../create_timelines_stream_from_ndjson.ts | 20 +-- .../timeline/routes/export_timelines_route.ts | 26 ++- .../timeline/routes/import_timelines_route.ts | 52 +++--- .../routes/schemas/export_timelines_schema.ts | 17 +- .../routes/schemas/import_timelines_schema.ts | 73 +++----- .../lib/timeline/routes/schemas/schemas.ts | 158 +----------------- .../plugins/siem/server/lib/timeline/types.ts | 17 +- 7 files changed, 86 insertions(+), 277 deletions(-) diff --git a/x-pack/legacy/plugins/siem/server/lib/timeline/create_timelines_stream_from_ndjson.ts b/x-pack/legacy/plugins/siem/server/lib/timeline/create_timelines_stream_from_ndjson.ts index 5373570a4f8cc..83bf9d3088078 100644 --- a/x-pack/legacy/plugins/siem/server/lib/timeline/create_timelines_stream_from_ndjson.ts +++ b/x-pack/legacy/plugins/siem/server/lib/timeline/create_timelines_stream_from_ndjson.ts @@ -5,6 +5,10 @@ */ import { Transform } from 'stream'; +import { identity } from 'fp-ts/lib/function'; +import Boom from 'boom'; +import { pipe } from 'fp-ts/lib/pipeable'; +import { fold } from 'fp-ts/lib/Either'; import { createConcatStream, createSplitStream, @@ -15,22 +19,14 @@ import { filterExportedCounts, createLimitStream, } from '../detection_engine/rules/create_rules_stream_from_ndjson'; -import { importTimelinesSchema } from './routes/schemas/import_timelines_schema'; -import { BadRequestError } from '../detection_engine/errors/bad_request_error'; + import { ImportTimelineResponse } from './routes/utils/import_timelines'; +import { throwErrors } from '../../../../../../plugins/case/common/api'; +import { importTimelinesSchema } from './routes/schemas/import_timelines_schema'; export const validateTimelines = (): Transform => { return createMapStream((obj: ImportTimelineResponse) => { - if (!(obj instanceof Error)) { - const validated = importTimelinesSchema.validate(obj); - if (validated.error != null) { - return new BadRequestError(validated.error.message); - } else { - return validated.value; - } - } else { - return obj; - } + return pipe(importTimelinesSchema.decode(obj), fold(throwErrors(Boom.badRequest), identity)); }); }; diff --git a/x-pack/legacy/plugins/siem/server/lib/timeline/routes/export_timelines_route.ts b/x-pack/legacy/plugins/siem/server/lib/timeline/routes/export_timelines_route.ts index b8e7be13fff34..9d12f042800a2 100644 --- a/x-pack/legacy/plugins/siem/server/lib/timeline/routes/export_timelines_route.ts +++ b/x-pack/legacy/plugins/siem/server/lib/timeline/routes/export_timelines_route.ts @@ -4,23 +4,15 @@ * you may not use this file except in compliance with the Elastic License. */ +import { schema } from '@kbn/config-schema'; + import { set as _set } from 'lodash/fp'; import { IRouter } from '../../../../../../../../src/core/server'; import { LegacyServices } from '../../../types'; -import { ExportTimelineRequestParams } from '../types'; -import { - transformError, - buildRouteValidation, - buildSiemResponse, -} from '../../detection_engine/routes/utils'; +import { transformError, buildSiemResponse } from '../../detection_engine/routes/utils'; import { TIMELINE_EXPORT_URL } from '../../../../common/constants'; -import { - exportTimelinesSchema, - exportTimelinesQuerySchema, -} from './schemas/export_timelines_schema'; - import { getExportTimelineByObjectIds } from './utils/export_timelines'; export const exportTimelinesRoute = (router: IRouter, config: LegacyServices['config']) => { @@ -28,10 +20,13 @@ export const exportTimelinesRoute = (router: IRouter, config: LegacyServices['co { path: TIMELINE_EXPORT_URL, validate: { - query: buildRouteValidation( - exportTimelinesQuerySchema - ), - body: buildRouteValidation(exportTimelinesSchema), + query: schema.object({ + file_name: schema.string(), + exclude_export_details: schema.boolean(), + }), + body: schema.object({ + ids: schema.arrayOf(schema.string()), + }), }, options: { tags: ['access:siem'], @@ -42,6 +37,7 @@ export const exportTimelinesRoute = (router: IRouter, config: LegacyServices['co const siemResponse = buildSiemResponse(response); const savedObjectsClient = context.core.savedObjects.client; const exportSizeLimit = config().get('savedObjects.maxImportExportSize'); + if (request.body?.ids != null && request.body.ids.length > exportSizeLimit) { return siemResponse.error({ statusCode: 400, diff --git a/x-pack/legacy/plugins/siem/server/lib/timeline/routes/import_timelines_route.ts b/x-pack/legacy/plugins/siem/server/lib/timeline/routes/import_timelines_route.ts index 2b41b4e7843a7..21976feb0d204 100644 --- a/x-pack/legacy/plugins/siem/server/lib/timeline/routes/import_timelines_route.ts +++ b/x-pack/legacy/plugins/siem/server/lib/timeline/routes/import_timelines_route.ts @@ -6,8 +6,13 @@ import { extname } from 'path'; import { chunk, omit, set } from 'lodash/fp'; +import Boom from 'boom'; +import { pipe } from 'fp-ts/lib/pipeable'; +import { fold } from 'fp-ts/lib/Either'; +import { identity } from 'fp-ts/lib/function'; +import { schema } from '@kbn/config-schema'; +import { Readable } from 'stream'; import { - buildRouteValidation, buildSiemResponse, createBulkErrorObject, BulkError, @@ -23,7 +28,6 @@ import { isBulkError, isImportRegular, ImportTimelineResponse, - ImportTimelinesRequestParams, ImportTimelinesSchema, PromiseFromStreams, } from './utils/import_timelines'; @@ -38,6 +42,7 @@ import { LegacyServices } from '../../../types'; import { Timeline } from '../saved_object'; import { validate } from '../../detection_engine/routes/rules/validate'; import { FrameworkRequest } from '../../framework'; +import { throwErrors } from '../../../../../../../plugins/case/common/api'; const CHUNK_PARSED_OBJECT_SIZE = 10; @@ -52,9 +57,7 @@ export const importTimelinesRoute = ( { path: `${TIMELINE_IMPORT_URL}`, validate: { - body: buildRouteValidation( - importTimelinesPayloadSchema - ), + body: schema.object({}, { unknowns: 'allow' }), }, options: { tags: ['access:siem'], @@ -65,28 +68,34 @@ export const importTimelinesRoute = ( }, }, async (context, request, response) => { - const siemResponse = buildSiemResponse(response); - const savedObjectsClient = context.core.savedObjects.client; - if (!savedObjectsClient) { - return siemResponse.error({ statusCode: 404 }); - } - const { filename } = request.body.file.hapi; + try { + const siemResponse = buildSiemResponse(response); + const savedObjectsClient = context.core.savedObjects.client; + if (!savedObjectsClient) { + return siemResponse.error({ statusCode: 404 }); + } - const fileExtension = extname(filename).toLowerCase(); + const { file } = pipe( + importTimelinesPayloadSchema.decode(request.body), + fold(throwErrors(Boom.badRequest), identity) + ); - if (fileExtension !== '.ndjson') { - return siemResponse.error({ - statusCode: 400, - body: `Invalid file extension ${fileExtension}`, - }); - } + const { filename } = file.hapi; - const objectLimit = config().get('savedObjects.maxImportExportSize'); + const fileExtension = extname(filename).toLowerCase(); + + if (fileExtension !== '.ndjson') { + return siemResponse.error({ + statusCode: 400, + body: `Invalid file extension ${fileExtension}`, + }); + } + + const objectLimit = config().get('savedObjects.maxImportExportSize'); - try { const readStream = createTimelinesStreamFromNdJson(objectLimit); const parsedObjects = await createPromiseFromStreams([ - request.body.file, + (file as unknown) as Readable, ...readStream, ]); const [duplicateIdErrors, uniqueParsedObjects] = getTupleDuplicateErrorsAndUniqueTimeline( @@ -215,6 +224,7 @@ export const importTimelinesRoute = ( } } catch (err) { const error = transformError(err); + const siemResponse = buildSiemResponse(response); return siemResponse.error({ body: error.message, diff --git a/x-pack/legacy/plugins/siem/server/lib/timeline/routes/schemas/export_timelines_schema.ts b/x-pack/legacy/plugins/siem/server/lib/timeline/routes/schemas/export_timelines_schema.ts index 04edbbd7046c9..321810b872431 100644 --- a/x-pack/legacy/plugins/siem/server/lib/timeline/routes/schemas/export_timelines_schema.ts +++ b/x-pack/legacy/plugins/siem/server/lib/timeline/routes/schemas/export_timelines_schema.ts @@ -4,17 +4,12 @@ * you may not use this file except in compliance with the Elastic License. */ -import Joi from 'joi'; +import * as runtimeTypes from 'io-ts'; -/* eslint-disable @typescript-eslint/camelcase */ -import { ids, exclude_export_details, file_name } from './schemas'; -/* eslint-disable @typescript-eslint/camelcase */ +const ids = runtimeTypes.array(runtimeTypes.string); +export const exportTimelinesSchema = runtimeTypes.type({ ids }); -export const exportTimelinesSchema = Joi.object({ - ids, -}).min(1); - -export const exportTimelinesQuerySchema = Joi.object({ - file_name: file_name.default('export.ndjson'), - exclude_export_details: exclude_export_details.default(false), +export const exportTimelinesQuerySchema = runtimeTypes.type({ + file_name: runtimeTypes.string, + exclude_export_details: runtimeTypes.boolean, }); diff --git a/x-pack/legacy/plugins/siem/server/lib/timeline/routes/schemas/import_timelines_schema.ts b/x-pack/legacy/plugins/siem/server/lib/timeline/routes/schemas/import_timelines_schema.ts index 61ffa9681c53a..0ce50fdbe0201 100644 --- a/x-pack/legacy/plugins/siem/server/lib/timeline/routes/schemas/import_timelines_schema.ts +++ b/x-pack/legacy/plugins/siem/server/lib/timeline/routes/schemas/import_timelines_schema.ts @@ -3,55 +3,28 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -import Joi from 'joi'; -import { - columns, - created, - createdBy, - dataProviders, - dateRange, - description, - eventNotes, - eventType, - favorite, - filters, - globalNotes, - kqlMode, - kqlQuery, - savedObjectId, - savedQueryId, - sort, - title, - updated, - updatedBy, - version, - pinnedEventIds, -} from './schemas'; +import * as runtimeTypes from 'io-ts'; -export const importTimelinesPayloadSchema = Joi.object({ - file: Joi.object().required(), -}); +import { eventNotes, globalNotes, pinnedEventIds } from './schemas'; +import { SavedTimelineRuntimeType } from '../../types'; -export const importTimelinesSchema = Joi.object({ - columns, - created, - createdBy, - dataProviders, - dateRange, - description, - eventNotes, - eventType, - filters, - favorite, - globalNotes, - kqlMode, - kqlQuery, - savedObjectId, - savedQueryId, - sort, - title, - updated, - updatedBy, - version, - pinnedEventIds, -}); +const file = runtimeTypes.intersection([ + runtimeTypes.UnknownRecord, + runtimeTypes.type({ + hapi: runtimeTypes.type({ filename: runtimeTypes.string }), + }), +]); +export const importTimelinesPayloadSchema = runtimeTypes.type({ file }); + +export const importTimelinesSchema = runtimeTypes.intersection([ + SavedTimelineRuntimeType, + runtimeTypes.type({ + savedObjectId: runtimeTypes.string, + version: runtimeTypes.string, + }), + runtimeTypes.type({ + globalNotes, + eventNotes, + pinnedEventIds, + }), +]); diff --git a/x-pack/legacy/plugins/siem/server/lib/timeline/routes/schemas/schemas.ts b/x-pack/legacy/plugins/siem/server/lib/timeline/routes/schemas/schemas.ts index 6552f973a66fa..71627363ef0f8 100644 --- a/x-pack/legacy/plugins/siem/server/lib/timeline/routes/schemas/schemas.ts +++ b/x-pack/legacy/plugins/siem/server/lib/timeline/routes/schemas/schemas.ts @@ -3,156 +3,10 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -import Joi from 'joi'; +import * as runtimeTypes from 'io-ts'; +import { unionWithNullType } from '../../../framework'; +import { SavedNoteRuntimeType } from '../../../note/types'; -const allowEmptyString = Joi.string().allow([null, '']); -const columnHeaderType = allowEmptyString; -export const created = Joi.number().allow(null); -export const createdBy = allowEmptyString; - -export const description = allowEmptyString; -export const end = Joi.number(); -export const eventId = allowEmptyString; -export const eventType = allowEmptyString; - -export const filters = Joi.array() - .items( - Joi.object({ - meta: Joi.object({ - alias: allowEmptyString, - controlledBy: allowEmptyString, - disabled: Joi.boolean().allow(null), - field: allowEmptyString, - formattedValue: allowEmptyString, - index: allowEmptyString, - key: allowEmptyString, - negate: Joi.boolean().allow(null), - params: allowEmptyString, - type: allowEmptyString, - value: allowEmptyString, - }), - exists: allowEmptyString, - match_all: allowEmptyString, - missing: allowEmptyString, - query: allowEmptyString, - range: allowEmptyString, - script: allowEmptyString, - }) - ) - .allow(null); - -const name = allowEmptyString; - -export const noteId = allowEmptyString; -export const note = allowEmptyString; - -export const start = Joi.number(); -export const savedQueryId = allowEmptyString; -export const savedObjectId = allowEmptyString; - -export const timelineId = allowEmptyString; -export const title = allowEmptyString; - -export const updated = Joi.number().allow(null); -export const updatedBy = allowEmptyString; -export const version = allowEmptyString; - -export const columns = Joi.array().items( - Joi.object({ - aggregatable: Joi.boolean().allow(null), - category: allowEmptyString, - columnHeaderType, - description, - example: allowEmptyString, - indexes: allowEmptyString, - id: allowEmptyString, - name, - placeholder: allowEmptyString, - searchable: Joi.boolean().allow(null), - type: allowEmptyString, - }).required() -); -export const dataProviders = Joi.array() - .items( - Joi.object({ - id: allowEmptyString, - name: allowEmptyString, - enabled: Joi.boolean().allow(null), - excluded: Joi.boolean().allow(null), - kqlQuery: allowEmptyString, - queryMatch: Joi.object({ - field: allowEmptyString, - displayField: allowEmptyString, - value: allowEmptyString, - displayValue: allowEmptyString, - operator: allowEmptyString, - }), - and: Joi.array() - .items( - Joi.object({ - id: allowEmptyString, - name, - enabled: Joi.boolean().allow(null), - excluded: Joi.boolean().allow(null), - kqlQuery: allowEmptyString, - queryMatch: Joi.object({ - field: allowEmptyString, - displayField: allowEmptyString, - value: allowEmptyString, - displayValue: allowEmptyString, - operator: allowEmptyString, - }).allow(null), - }) - ) - .allow(null), - }) - ) - .allow(null); -export const dateRange = Joi.object({ - start, - end, -}); -export const favorite = Joi.array().items( - Joi.object({ - keySearch: allowEmptyString, - fullName: allowEmptyString, - userName: allowEmptyString, - favoriteDate: Joi.number(), - }).allow(null) -); -const noteItem = Joi.object({ - noteId, - version, - eventId, - note, - timelineId, - created, - createdBy, - updated, - updatedBy, -}); -export const eventNotes = Joi.array().items(noteItem); -export const globalNotes = Joi.array().items(noteItem); -export const kqlMode = allowEmptyString; -export const kqlQuery = Joi.object({ - filterQuery: Joi.object({ - kuery: Joi.object({ - kind: allowEmptyString, - expression: allowEmptyString, - }), - serializedQuery: allowEmptyString, - }), -}); -export const pinnedEventIds = Joi.array() - .items(allowEmptyString) - .allow(null); -export const sort = Joi.object({ - columnId: allowEmptyString, - sortDirection: allowEmptyString, -}); -/* eslint-disable @typescript-eslint/camelcase */ - -export const ids = Joi.array().items(allowEmptyString); - -export const exclude_export_details = Joi.boolean(); -export const file_name = allowEmptyString; +export const eventNotes = runtimeTypes.array(unionWithNullType(SavedNoteRuntimeType)); +export const globalNotes = runtimeTypes.array(unionWithNullType(SavedNoteRuntimeType)); +export const pinnedEventIds = runtimeTypes.array(unionWithNullType(runtimeTypes.string)); diff --git a/x-pack/legacy/plugins/siem/server/lib/timeline/types.ts b/x-pack/legacy/plugins/siem/server/lib/timeline/types.ts index 35bf86c17db7e..1c5ca1765a816 100644 --- a/x-pack/legacy/plugins/siem/server/lib/timeline/types.ts +++ b/x-pack/legacy/plugins/siem/server/lib/timeline/types.ts @@ -14,7 +14,7 @@ import { PinnedEventToReturnSavedObjectRuntimeType, PinnedEventSavedObject, } from '../pinned_event/types'; -import { SavedObjectsClient, KibanaRequest } from '../../../../../../../src/core/server'; +import { SavedObjectsClient } from '../../../../../../../src/core/server'; /* * ColumnHeader Types @@ -204,21 +204,6 @@ export const AllTimelineSavedObjectRuntimeType = runtimeTypes.type({ export interface AllTimelineSavedObject extends runtimeTypes.TypeOf {} -export interface ExportTimelineRequestParams { - body: { ids: string[] }; - query: { - file_name: string; - exclude_export_details: boolean; - }; -} - -export type ExportTimelineRequest = KibanaRequest< - unknown, - ExportTimelineRequestParams['query'], - ExportTimelineRequestParams['body'], - 'post' ->; - export type ExportTimelineSavedObjectsClient = Pick< SavedObjectsClient, | 'get' From 419b0838f4239bd1d62b6d553fdc406b3721ec5b Mon Sep 17 00:00:00 2001 From: Angela Chuang Date: Wed, 1 Apr 2020 12:29:48 +0100 Subject: [PATCH 02/21] update schema for filterQuery --- x-pack/legacy/plugins/siem/server/lib/timeline/types.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/x-pack/legacy/plugins/siem/server/lib/timeline/types.ts b/x-pack/legacy/plugins/siem/server/lib/timeline/types.ts index 1c5ca1765a816..83099c5897358 100644 --- a/x-pack/legacy/plugins/siem/server/lib/timeline/types.ts +++ b/x-pack/legacy/plugins/siem/server/lib/timeline/types.ts @@ -103,9 +103,11 @@ const SavedSerializedFilterQueryQueryRuntimeType = runtimeTypes.partial({ serializedQuery: unionWithNullType(runtimeTypes.string), }); -const SavedFilterQueryQueryRuntimeType = runtimeTypes.partial({ - filterQuery: unionWithNullType(SavedSerializedFilterQueryQueryRuntimeType), -}); +const SavedFilterQueryQueryRuntimeType = unionWithNullType( + runtimeTypes.partial({ + filterQuery: unionWithNullType(SavedSerializedFilterQueryQueryRuntimeType), + }) +); /* * DatePicker Range Types From 1097bb1a9027a8b8df20f294c26a814009c18f0a Mon Sep 17 00:00:00 2001 From: Angela Chuang Date: Thu, 2 Apr 2020 09:41:39 +0100 Subject: [PATCH 03/21] fix types --- .../create_timelines_stream_from_ndjson.ts | 4 +- .../timeline/routes/import_timelines_route.ts | 4 +- .../routes/schemas/import_timelines_schema.ts | 41 ++++++++++++------- .../plugins/siem/server/lib/timeline/types.ts | 8 ++-- 4 files changed, 34 insertions(+), 23 deletions(-) diff --git a/x-pack/legacy/plugins/siem/server/lib/timeline/create_timelines_stream_from_ndjson.ts b/x-pack/legacy/plugins/siem/server/lib/timeline/create_timelines_stream_from_ndjson.ts index 83bf9d3088078..888ff74ca7814 100644 --- a/x-pack/legacy/plugins/siem/server/lib/timeline/create_timelines_stream_from_ndjson.ts +++ b/x-pack/legacy/plugins/siem/server/lib/timeline/create_timelines_stream_from_ndjson.ts @@ -22,11 +22,11 @@ import { import { ImportTimelineResponse } from './routes/utils/import_timelines'; import { throwErrors } from '../../../../../../plugins/case/common/api'; -import { importTimelinesSchema } from './routes/schemas/import_timelines_schema'; +import { ImportTimelinesSchemaRt } from './routes/schemas/import_timelines_schema'; export const validateTimelines = (): Transform => { return createMapStream((obj: ImportTimelineResponse) => { - return pipe(importTimelinesSchema.decode(obj), fold(throwErrors(Boom.badRequest), identity)); + return pipe(ImportTimelinesSchemaRt.decode(obj), fold(throwErrors(Boom.badRequest), identity)); }); }; diff --git a/x-pack/legacy/plugins/siem/server/lib/timeline/routes/import_timelines_route.ts b/x-pack/legacy/plugins/siem/server/lib/timeline/routes/import_timelines_route.ts index 21976feb0d204..9afa0b1a44986 100644 --- a/x-pack/legacy/plugins/siem/server/lib/timeline/routes/import_timelines_route.ts +++ b/x-pack/legacy/plugins/siem/server/lib/timeline/routes/import_timelines_route.ts @@ -35,7 +35,7 @@ import { import { IRouter } from '../../../../../../../../src/core/server'; import { TIMELINE_IMPORT_URL } from '../../../../common/constants'; import { SetupPlugins } from '../../../plugin'; -import { importTimelinesPayloadSchema } from './schemas/import_timelines_schema'; +import { ImportTimelinesPayloadSchemaRt } from './schemas/import_timelines_schema'; import { importRulesSchema } from '../../detection_engine/routes/schemas/response/import_rules_schema'; import { LegacyServices } from '../../../types'; @@ -76,7 +76,7 @@ export const importTimelinesRoute = ( } const { file } = pipe( - importTimelinesPayloadSchema.decode(request.body), + ImportTimelinesPayloadSchemaRt.decode(request.body), fold(throwErrors(Boom.badRequest), identity) ); diff --git a/x-pack/legacy/plugins/siem/server/lib/timeline/routes/schemas/import_timelines_schema.ts b/x-pack/legacy/plugins/siem/server/lib/timeline/routes/schemas/import_timelines_schema.ts index 0ce50fdbe0201..056fdaf0d2515 100644 --- a/x-pack/legacy/plugins/siem/server/lib/timeline/routes/schemas/import_timelines_schema.ts +++ b/x-pack/legacy/plugins/siem/server/lib/timeline/routes/schemas/import_timelines_schema.ts @@ -3,28 +3,41 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -import * as runtimeTypes from 'io-ts'; +import * as rt from 'io-ts'; +import { Readable } from 'stream'; +import { either } from 'fp-ts/lib/Either'; import { eventNotes, globalNotes, pinnedEventIds } from './schemas'; import { SavedTimelineRuntimeType } from '../../types'; -const file = runtimeTypes.intersection([ - runtimeTypes.UnknownRecord, - runtimeTypes.type({ - hapi: runtimeTypes.type({ filename: runtimeTypes.string }), - }), -]); -export const importTimelinesPayloadSchema = runtimeTypes.type({ file }); - -export const importTimelinesSchema = runtimeTypes.intersection([ +export const ImportTimelinesSchemaRt = rt.intersection([ SavedTimelineRuntimeType, - runtimeTypes.type({ - savedObjectId: runtimeTypes.string, - version: runtimeTypes.string, + rt.type({ + savedObjectId: rt.string, + version: rt.string, }), - runtimeTypes.type({ + rt.type({ globalNotes, eventNotes, pinnedEventIds, }), ]); + +const ReadableRt = new rt.Type( + 'ReadableRt', + (u): u is Readable => u instanceof Readable, + (u, c) => + either.chain(rt.object.validate(u, c), s => { + const d = s as Readable; + return d.readable ? rt.success(d) : rt.failure(u, c); + }), + a => a +); +export const ImportTimelinesPayloadSchemaRt = rt.type({ + file: rt.intersection([ + ReadableRt, + rt.type({ + hapi: rt.type({ filename: rt.string }), + }), + ]), +}); diff --git a/x-pack/legacy/plugins/siem/server/lib/timeline/types.ts b/x-pack/legacy/plugins/siem/server/lib/timeline/types.ts index 83099c5897358..1c5ca1765a816 100644 --- a/x-pack/legacy/plugins/siem/server/lib/timeline/types.ts +++ b/x-pack/legacy/plugins/siem/server/lib/timeline/types.ts @@ -103,11 +103,9 @@ const SavedSerializedFilterQueryQueryRuntimeType = runtimeTypes.partial({ serializedQuery: unionWithNullType(runtimeTypes.string), }); -const SavedFilterQueryQueryRuntimeType = unionWithNullType( - runtimeTypes.partial({ - filterQuery: unionWithNullType(SavedSerializedFilterQueryQueryRuntimeType), - }) -); +const SavedFilterQueryQueryRuntimeType = runtimeTypes.partial({ + filterQuery: unionWithNullType(SavedSerializedFilterQueryQueryRuntimeType), +}); /* * DatePicker Range Types From 63ed91eddfa012e2f9378eb3a0104c5980d52f31 Mon Sep 17 00:00:00 2001 From: Angela Chuang Date: Thu, 2 Apr 2020 11:40:12 +0100 Subject: [PATCH 04/21] update schemas --- .../create_timelines_stream_from_ndjson.ts | 18 ++++++++++++++---- .../timeline/routes/import_timelines_route.ts | 4 ++-- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/x-pack/legacy/plugins/siem/server/lib/timeline/create_timelines_stream_from_ndjson.ts b/x-pack/legacy/plugins/siem/server/lib/timeline/create_timelines_stream_from_ndjson.ts index 888ff74ca7814..a80ca561724e8 100644 --- a/x-pack/legacy/plugins/siem/server/lib/timeline/create_timelines_stream_from_ndjson.ts +++ b/x-pack/legacy/plugins/siem/server/lib/timeline/create_timelines_stream_from_ndjson.ts @@ -21,21 +21,31 @@ import { } from '../detection_engine/rules/create_rules_stream_from_ndjson'; import { ImportTimelineResponse } from './routes/utils/import_timelines'; -import { throwErrors } from '../../../../../../plugins/case/common/api'; import { ImportTimelinesSchemaRt } from './routes/schemas/import_timelines_schema'; +import { KibanaResponseFactory } from '../../../../../../../src/core/server'; +import { excess, throwErrors } from '../../../../../../plugins/case/common/api'; -export const validateTimelines = (): Transform => { +export const validateTimelines = (response: KibanaResponseFactory): Transform => { return createMapStream((obj: ImportTimelineResponse) => { + // return pipe( + // ImportTimelinesSchemaRt.decode(obj), + // fold(e => { + // throw response.badRequest(); + // }, identity) + // ); return pipe(ImportTimelinesSchemaRt.decode(obj), fold(throwErrors(Boom.badRequest), identity)); }); }; -export const createTimelinesStreamFromNdJson = (ruleLimit: number) => { +export const createTimelinesStreamFromNdJson = ( + ruleLimit: number, + response: KibanaResponseFactory +) => { return [ createSplitStream('\n'), parseNdjsonStrings(), filterExportedCounts(), - validateTimelines(), + validateTimelines(response), createLimitStream(ruleLimit), createConcatStream([]), ]; diff --git a/x-pack/legacy/plugins/siem/server/lib/timeline/routes/import_timelines_route.ts b/x-pack/legacy/plugins/siem/server/lib/timeline/routes/import_timelines_route.ts index 9afa0b1a44986..97e079b04067f 100644 --- a/x-pack/legacy/plugins/siem/server/lib/timeline/routes/import_timelines_route.ts +++ b/x-pack/legacy/plugins/siem/server/lib/timeline/routes/import_timelines_route.ts @@ -93,9 +93,9 @@ export const importTimelinesRoute = ( const objectLimit = config().get('savedObjects.maxImportExportSize'); - const readStream = createTimelinesStreamFromNdJson(objectLimit); + const readStream = createTimelinesStreamFromNdJson(objectLimit, response); const parsedObjects = await createPromiseFromStreams([ - (file as unknown) as Readable, + file, ...readStream, ]); const [duplicateIdErrors, uniqueParsedObjects] = getTupleDuplicateErrorsAndUniqueTimeline( From 7314490125a8e774635972e0dc039539511dbc2f Mon Sep 17 00:00:00 2001 From: Angela Chuang Date: Thu, 2 Apr 2020 15:58:19 +0100 Subject: [PATCH 05/21] remove boom --- .../create_timelines_stream_from_ndjson.ts | 17 ++++++++--------- .../timeline/routes/import_timelines_route.ts | 1 - 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/x-pack/legacy/plugins/siem/server/lib/timeline/create_timelines_stream_from_ndjson.ts b/x-pack/legacy/plugins/siem/server/lib/timeline/create_timelines_stream_from_ndjson.ts index a80ca561724e8..a3531df3a7ace 100644 --- a/x-pack/legacy/plugins/siem/server/lib/timeline/create_timelines_stream_from_ndjson.ts +++ b/x-pack/legacy/plugins/siem/server/lib/timeline/create_timelines_stream_from_ndjson.ts @@ -6,7 +6,6 @@ import { Transform } from 'stream'; import { identity } from 'fp-ts/lib/function'; -import Boom from 'boom'; import { pipe } from 'fp-ts/lib/pipeable'; import { fold } from 'fp-ts/lib/Either'; import { @@ -23,17 +22,17 @@ import { import { ImportTimelineResponse } from './routes/utils/import_timelines'; import { ImportTimelinesSchemaRt } from './routes/schemas/import_timelines_schema'; import { KibanaResponseFactory } from '../../../../../../../src/core/server'; -import { excess, throwErrors } from '../../../../../../plugins/case/common/api'; +import { throwErrors } from '../../../../../../plugins/case/common/api'; export const validateTimelines = (response: KibanaResponseFactory): Transform => { return createMapStream((obj: ImportTimelineResponse) => { - // return pipe( - // ImportTimelinesSchemaRt.decode(obj), - // fold(e => { - // throw response.badRequest(); - // }, identity) - // ); - return pipe(ImportTimelinesSchemaRt.decode(obj), fold(throwErrors(Boom.badRequest), identity)); + return pipe( + ImportTimelinesSchemaRt.decode(obj), + fold( + throwErrors(msg => new Error(msg)), + identity + ) + ); }); }; diff --git a/x-pack/legacy/plugins/siem/server/lib/timeline/routes/import_timelines_route.ts b/x-pack/legacy/plugins/siem/server/lib/timeline/routes/import_timelines_route.ts index 97e079b04067f..07f16375aaa47 100644 --- a/x-pack/legacy/plugins/siem/server/lib/timeline/routes/import_timelines_route.ts +++ b/x-pack/legacy/plugins/siem/server/lib/timeline/routes/import_timelines_route.ts @@ -11,7 +11,6 @@ import { pipe } from 'fp-ts/lib/pipeable'; import { fold } from 'fp-ts/lib/Either'; import { identity } from 'fp-ts/lib/function'; import { schema } from '@kbn/config-schema'; -import { Readable } from 'stream'; import { buildSiemResponse, createBulkErrorObject, From b752c0df5e12694b124a4f20fc917bb168a82426 Mon Sep 17 00:00:00 2001 From: Angela Chuang Date: Thu, 2 Apr 2020 16:02:06 +0100 Subject: [PATCH 06/21] remove redundant params --- .../timeline/create_timelines_stream_from_ndjson.ts | 10 +++------- .../lib/timeline/routes/import_timelines_route.ts | 2 +- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/x-pack/legacy/plugins/siem/server/lib/timeline/create_timelines_stream_from_ndjson.ts b/x-pack/legacy/plugins/siem/server/lib/timeline/create_timelines_stream_from_ndjson.ts index a3531df3a7ace..c0c9a1efc6a51 100644 --- a/x-pack/legacy/plugins/siem/server/lib/timeline/create_timelines_stream_from_ndjson.ts +++ b/x-pack/legacy/plugins/siem/server/lib/timeline/create_timelines_stream_from_ndjson.ts @@ -21,10 +21,9 @@ import { import { ImportTimelineResponse } from './routes/utils/import_timelines'; import { ImportTimelinesSchemaRt } from './routes/schemas/import_timelines_schema'; -import { KibanaResponseFactory } from '../../../../../../../src/core/server'; import { throwErrors } from '../../../../../../plugins/case/common/api'; -export const validateTimelines = (response: KibanaResponseFactory): Transform => { +export const validateTimelines = (): Transform => { return createMapStream((obj: ImportTimelineResponse) => { return pipe( ImportTimelinesSchemaRt.decode(obj), @@ -36,15 +35,12 @@ export const validateTimelines = (response: KibanaResponseFactory): Transform => }); }; -export const createTimelinesStreamFromNdJson = ( - ruleLimit: number, - response: KibanaResponseFactory -) => { +export const createTimelinesStreamFromNdJson = (ruleLimit: number) => { return [ createSplitStream('\n'), parseNdjsonStrings(), filterExportedCounts(), - validateTimelines(response), + validateTimelines(), createLimitStream(ruleLimit), createConcatStream([]), ]; diff --git a/x-pack/legacy/plugins/siem/server/lib/timeline/routes/import_timelines_route.ts b/x-pack/legacy/plugins/siem/server/lib/timeline/routes/import_timelines_route.ts index 07f16375aaa47..0f21b8639792b 100644 --- a/x-pack/legacy/plugins/siem/server/lib/timeline/routes/import_timelines_route.ts +++ b/x-pack/legacy/plugins/siem/server/lib/timeline/routes/import_timelines_route.ts @@ -92,7 +92,7 @@ export const importTimelinesRoute = ( const objectLimit = config().get('savedObjects.maxImportExportSize'); - const readStream = createTimelinesStreamFromNdJson(objectLimit, response); + const readStream = createTimelinesStreamFromNdJson(objectLimit); const parsedObjects = await createPromiseFromStreams([ file, ...readStream, From 26235e535b47e6c6724eb6649c4a4928b21e01e6 Mon Sep 17 00:00:00 2001 From: Angela Chuang Date: Thu, 2 Apr 2020 17:15:18 +0100 Subject: [PATCH 07/21] reuse utils from case --- .../timeline/create_timelines_stream_from_ndjson.ts | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/x-pack/legacy/plugins/siem/server/lib/timeline/create_timelines_stream_from_ndjson.ts b/x-pack/legacy/plugins/siem/server/lib/timeline/create_timelines_stream_from_ndjson.ts index c0c9a1efc6a51..7ddcd28fc7f45 100644 --- a/x-pack/legacy/plugins/siem/server/lib/timeline/create_timelines_stream_from_ndjson.ts +++ b/x-pack/legacy/plugins/siem/server/lib/timeline/create_timelines_stream_from_ndjson.ts @@ -5,9 +5,6 @@ */ import { Transform } from 'stream'; -import { identity } from 'fp-ts/lib/function'; -import { pipe } from 'fp-ts/lib/pipeable'; -import { fold } from 'fp-ts/lib/Either'; import { createConcatStream, createSplitStream, @@ -21,17 +18,11 @@ import { import { ImportTimelineResponse } from './routes/utils/import_timelines'; import { ImportTimelinesSchemaRt } from './routes/schemas/import_timelines_schema'; -import { throwErrors } from '../../../../../../plugins/case/common/api'; +import { decodeOrThrow } from '../../../../../../plugins/case/common/api'; export const validateTimelines = (): Transform => { return createMapStream((obj: ImportTimelineResponse) => { - return pipe( - ImportTimelinesSchemaRt.decode(obj), - fold( - throwErrors(msg => new Error(msg)), - identity - ) - ); + return decodeOrThrow(ImportTimelinesSchemaRt)(obj); }); }; From 0fd8ec7b9cffc43b415c29bd8ec0f70c4f88316f Mon Sep 17 00:00:00 2001 From: Angela Chuang Date: Wed, 8 Apr 2020 02:45:40 +0100 Subject: [PATCH 08/21] update schemas for query params and body --- .../timeline/routes/export_timelines_route.ts | 30 +++++++---- .../routes/schemas/export_timelines_schema.ts | 50 ++++++++++++++++--- 2 files changed, 65 insertions(+), 15 deletions(-) diff --git a/x-pack/legacy/plugins/siem/server/lib/timeline/routes/export_timelines_route.ts b/x-pack/legacy/plugins/siem/server/lib/timeline/routes/export_timelines_route.ts index 9d12f042800a2..5398838e56b8b 100644 --- a/x-pack/legacy/plugins/siem/server/lib/timeline/routes/export_timelines_route.ts +++ b/x-pack/legacy/plugins/siem/server/lib/timeline/routes/export_timelines_route.ts @@ -4,8 +4,6 @@ * you may not use this file except in compliance with the Elastic License. */ -import { schema } from '@kbn/config-schema'; - import { set as _set } from 'lodash/fp'; import { IRouter } from '../../../../../../../../src/core/server'; import { LegacyServices } from '../../../types'; @@ -14,19 +12,33 @@ import { transformError, buildSiemResponse } from '../../detection_engine/routes import { TIMELINE_EXPORT_URL } from '../../../../common/constants'; import { getExportTimelineByObjectIds } from './utils/export_timelines'; +import { + exportTimelinesQuerySchema, + exportTimelinesRequestBodySchema, + buildRouteValidation, + decodeOrThrow, +} from './schemas/export_timelines_schema'; + +interface ExportTimelinesQuery { + file_name: string; + exclude_export_details: 'true' | 'false'; +} + +interface ExportTimelinesRequestBody { + ids: string[]; +} export const exportTimelinesRoute = (router: IRouter, config: LegacyServices['config']) => { router.post( { path: TIMELINE_EXPORT_URL, validate: { - query: schema.object({ - file_name: schema.string(), - exclude_export_details: schema.boolean(), - }), - body: schema.object({ - ids: schema.arrayOf(schema.string()), - }), + query: buildRouteValidation( + exportTimelinesQuerySchema + ), + body: buildRouteValidation( + exportTimelinesRequestBodySchema + ), }, options: { tags: ['access:siem'], diff --git a/x-pack/legacy/plugins/siem/server/lib/timeline/routes/schemas/export_timelines_schema.ts b/x-pack/legacy/plugins/siem/server/lib/timeline/routes/schemas/export_timelines_schema.ts index 321810b872431..00840efaf8a85 100644 --- a/x-pack/legacy/plugins/siem/server/lib/timeline/routes/schemas/export_timelines_schema.ts +++ b/x-pack/legacy/plugins/siem/server/lib/timeline/routes/schemas/export_timelines_schema.ts @@ -4,12 +4,50 @@ * you may not use this file except in compliance with the Elastic License. */ -import * as runtimeTypes from 'io-ts'; +import { either, fold } from 'fp-ts/lib/Either'; +import { identity } from 'fp-ts/lib/function'; +import { pipe } from 'fp-ts/lib/pipeable'; +import * as rt from 'io-ts'; +import { failure } from 'io-ts/lib/PathReporter'; +import { PathReporter } from 'io-ts/lib/PathReporter'; +import { ValidationError } from '@kbn/config-schema'; +import { + RouteValidationFunction, + RouteValidationResultFactory, + RouteValidationError, +} from '../../../../../../../../../src/core/server'; -const ids = runtimeTypes.array(runtimeTypes.string); -export const exportTimelinesSchema = runtimeTypes.type({ ids }); +const ids = rt.array(rt.string); +export const exportTimelinesSchema = rt.type({ ids }); -export const exportTimelinesQuerySchema = runtimeTypes.type({ - file_name: runtimeTypes.string, - exclude_export_details: runtimeTypes.boolean, +export const exportTimelinesQuerySchema = rt.type({ + file_name: rt.string, + exclude_export_details: rt.union([rt.literal('true'), rt.literal('false')]), }); + +export const exportTimelinesRequestBodySchema = rt.type({ + ids, +}); +type ErrorFactory = (message: string) => Error; + +export const createPlainError = (message: string) => new Error(message); + +export const throwErrors = (createError: ErrorFactory) => (errors: rt.Errors) => { + return createError(failure(errors).join('\n')); + // return { error: createError(failure(errors).join('\n')) }; + // return { + // error: new ValidationError( + // new RouteValidationError(`The validation rule provided in the handler is not valid`) + // ), + // }; +}; + +export const decodeOrThrow = ( + runtimeType: rt.Type, + createError: ErrorFactory = createPlainError +): RouteValidationFunction => (inputValue: I, validationResult: RouteValidationResultFactory) => + pipe(runtimeType.decode(inputValue), fold(throwErrors(createError), identity)); + +export const buildRouteValidation = ( + schema: rt.Type +): RouteValidationFunction => decodeOrThrow(schema); From a50e769e0230b631a264254590a7da243ea70263 Mon Sep 17 00:00:00 2001 From: Angela Chuang Date: Wed, 8 Apr 2020 05:20:27 +0100 Subject: [PATCH 09/21] fix types --- .../timeline/routes/export_timelines_route.ts | 11 +----- .../routes/schemas/export_timelines_schema.ts | 29 +++++++-------- .../plugins/siem/server/lib/timeline/types.ts | 36 ++++++++++++++++++- 3 files changed, 49 insertions(+), 27 deletions(-) diff --git a/x-pack/legacy/plugins/siem/server/lib/timeline/routes/export_timelines_route.ts b/x-pack/legacy/plugins/siem/server/lib/timeline/routes/export_timelines_route.ts index 5398838e56b8b..d09d669646dec 100644 --- a/x-pack/legacy/plugins/siem/server/lib/timeline/routes/export_timelines_route.ts +++ b/x-pack/legacy/plugins/siem/server/lib/timeline/routes/export_timelines_route.ts @@ -16,17 +16,8 @@ import { exportTimelinesQuerySchema, exportTimelinesRequestBodySchema, buildRouteValidation, - decodeOrThrow, } from './schemas/export_timelines_schema'; - -interface ExportTimelinesQuery { - file_name: string; - exclude_export_details: 'true' | 'false'; -} - -interface ExportTimelinesRequestBody { - ids: string[]; -} +import { ExportTimelinesQuery, ExportTimelinesRequestBody } from '../types'; export const exportTimelinesRoute = (router: IRouter, config: LegacyServices['config']) => { router.post( diff --git a/x-pack/legacy/plugins/siem/server/lib/timeline/routes/schemas/export_timelines_schema.ts b/x-pack/legacy/plugins/siem/server/lib/timeline/routes/schemas/export_timelines_schema.ts index 00840efaf8a85..41f7e042fb10c 100644 --- a/x-pack/legacy/plugins/siem/server/lib/timeline/routes/schemas/export_timelines_schema.ts +++ b/x-pack/legacy/plugins/siem/server/lib/timeline/routes/schemas/export_timelines_schema.ts @@ -4,18 +4,16 @@ * you may not use this file except in compliance with the Elastic License. */ -import { either, fold } from 'fp-ts/lib/Either'; -import { identity } from 'fp-ts/lib/function'; +import { fold } from 'fp-ts/lib/Either'; import { pipe } from 'fp-ts/lib/pipeable'; import * as rt from 'io-ts'; import { failure } from 'io-ts/lib/PathReporter'; -import { PathReporter } from 'io-ts/lib/PathReporter'; -import { ValidationError } from '@kbn/config-schema'; + import { RouteValidationFunction, RouteValidationResultFactory, - RouteValidationError, } from '../../../../../../../../../src/core/server'; +import { RequestValidationResult } from '../../types'; const ids = rt.array(rt.string); export const exportTimelinesSchema = rt.type({ ids }); @@ -32,21 +30,20 @@ type ErrorFactory = (message: string) => Error; export const createPlainError = (message: string) => new Error(message); -export const throwErrors = (createError: ErrorFactory) => (errors: rt.Errors) => { - return createError(failure(errors).join('\n')); - // return { error: createError(failure(errors).join('\n')) }; - // return { - // error: new ValidationError( - // new RouteValidationError(`The validation rule provided in the handler is not valid`) - // ), - // }; -}; +export const throwErrors = (createError: ErrorFactory) => (errors: rt.Errors) => + createError(failure(errors).join('\n')); export const decodeOrThrow = ( runtimeType: rt.Type, createError: ErrorFactory = createPlainError -): RouteValidationFunction => (inputValue: I, validationResult: RouteValidationResultFactory) => - pipe(runtimeType.decode(inputValue), fold(throwErrors(createError), identity)); +) => (inputValue: I, validationResult: RouteValidationResultFactory) => + pipe( + runtimeType.decode(inputValue), + fold>( + (errors: rt.Errors) => validationResult.badRequest(throwErrors(createError)(errors)), + (validatedInput: A) => validationResult.ok(validatedInput) + ) + ); export const buildRouteValidation = ( schema: rt.Type diff --git a/x-pack/legacy/plugins/siem/server/lib/timeline/types.ts b/x-pack/legacy/plugins/siem/server/lib/timeline/types.ts index 1c5ca1765a816..cf11f54a93323 100644 --- a/x-pack/legacy/plugins/siem/server/lib/timeline/types.ts +++ b/x-pack/legacy/plugins/siem/server/lib/timeline/types.ts @@ -14,7 +14,11 @@ import { PinnedEventToReturnSavedObjectRuntimeType, PinnedEventSavedObject, } from '../pinned_event/types'; -import { SavedObjectsClient } from '../../../../../../../src/core/server'; +import { + SavedObjectsClient, + KibanaRequest, + RouteValidationError, +} from '../../../../../../../src/core/server'; /* * ColumnHeader Types @@ -204,6 +208,36 @@ export const AllTimelineSavedObjectRuntimeType = runtimeTypes.type({ export interface AllTimelineSavedObject extends runtimeTypes.TypeOf {} +/** + * Import/export timelines + */ + +export interface ExportTimelinesQuery { + file_name: string; + exclude_export_details: 'true' | 'false'; +} + +export interface ExportTimelinesRequestBody { + ids: string[]; +} + +export type ExportTimelineRequest = KibanaRequest< + unknown, + ExportTimelinesQuery, + ExportTimelinesRequestBody, + 'post' +>; + +export type RequestValidationResult = + | { + value: T; + error?: undefined; + } + | { + value?: undefined; + error: RouteValidationError; + }; + export type ExportTimelineSavedObjectsClient = Pick< SavedObjectsClient, | 'get' From 58b757b8ed1e9bdaebe46ba1ebe3833e2cde4a35 Mon Sep 17 00:00:00 2001 From: Angela Chuang Date: Wed, 8 Apr 2020 16:38:36 +0100 Subject: [PATCH 10/21] update validation schema --- .../timeline/routes/import_timelines_route.ts | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/x-pack/legacy/plugins/siem/server/lib/timeline/routes/import_timelines_route.ts b/x-pack/legacy/plugins/siem/server/lib/timeline/routes/import_timelines_route.ts index 0f21b8639792b..a65a4fcb03454 100644 --- a/x-pack/legacy/plugins/siem/server/lib/timeline/routes/import_timelines_route.ts +++ b/x-pack/legacy/plugins/siem/server/lib/timeline/routes/import_timelines_route.ts @@ -10,7 +10,8 @@ import Boom from 'boom'; import { pipe } from 'fp-ts/lib/pipeable'; import { fold } from 'fp-ts/lib/Either'; import { identity } from 'fp-ts/lib/function'; -import { schema } from '@kbn/config-schema'; + +import { Readable } from 'stream'; import { buildSiemResponse, createBulkErrorObject, @@ -42,11 +43,17 @@ import { Timeline } from '../saved_object'; import { validate } from '../../detection_engine/routes/rules/validate'; import { FrameworkRequest } from '../../framework'; import { throwErrors } from '../../../../../../../plugins/case/common/api'; - +import { buildRouteValidation } from './schemas/export_timelines_schema'; const CHUNK_PARSED_OBJECT_SIZE = 10; const timelineLib = new Timeline(); +interface ImportTimelinesRequestBodySchema { + file: Readable & { + hapi: { filename: string }; + }; +} + export const importTimelinesRoute = ( router: IRouter, config: LegacyServices['config'], @@ -56,7 +63,11 @@ export const importTimelinesRoute = ( { path: `${TIMELINE_IMPORT_URL}`, validate: { - body: schema.object({}, { unknowns: 'allow' }), + body: buildRouteValidation< + ImportTimelinesRequestBodySchema, + ImportTimelinesRequestBodySchema, + unknown + >(ImportTimelinesPayloadSchemaRt), }, options: { tags: ['access:siem'], From 546badc05678c1a4ab521ccfdb830f01b4c0c1df Mon Sep 17 00:00:00 2001 From: Angela Chuang Date: Wed, 8 Apr 2020 17:46:09 +0100 Subject: [PATCH 11/21] fix unit test --- .../routes/__mocks__/request_responses.ts | 9 ++++++-- .../routes/export_timelines_route.test.ts | 23 +++++++++++++++++-- .../routes/import_timelines_route.test.ts | 5 +++- .../routes/schemas/export_timelines_schema.ts | 16 ++++--------- 4 files changed, 37 insertions(+), 16 deletions(-) diff --git a/x-pack/legacy/plugins/siem/server/lib/timeline/routes/__mocks__/request_responses.ts b/x-pack/legacy/plugins/siem/server/lib/timeline/routes/__mocks__/request_responses.ts index 0e73e4bdd6c97..a83c443773302 100644 --- a/x-pack/legacy/plugins/siem/server/lib/timeline/routes/__mocks__/request_responses.ts +++ b/x-pack/legacy/plugins/siem/server/lib/timeline/routes/__mocks__/request_responses.ts @@ -6,11 +6,16 @@ import { TIMELINE_EXPORT_URL, TIMELINE_IMPORT_URL } from '../../../../../common/constants'; import { requestMock } from '../../../detection_engine/routes/__mocks__'; - +import stream from 'stream'; +const readable = new stream.Readable(); export const getExportTimelinesRequest = () => requestMock.create({ method: 'get', path: TIMELINE_EXPORT_URL, + query: { + file_name: 'mock_export_timeline.ndjson', + exclude_export_details: 'false', + }, body: { ids: ['f0e58720-57b6-11ea-b88d-3f1a31716be8', '890b8ae0-57df-11ea-a7c9-3976b7f1cb37'], }, @@ -22,7 +27,7 @@ export const getImportTimelinesRequest = (filename?: string) => path: TIMELINE_IMPORT_URL, query: { overwrite: false }, body: { - file: { hapi: { filename: filename ?? 'filename.ndjson' } }, + file: { ...readable, hapi: { filename: filename ?? 'filename.ndjson' } }, }, }); diff --git a/x-pack/legacy/plugins/siem/server/lib/timeline/routes/export_timelines_route.test.ts b/x-pack/legacy/plugins/siem/server/lib/timeline/routes/export_timelines_route.test.ts index fe434b5399212..db4d66f6ef984 100644 --- a/x-pack/legacy/plugins/siem/server/lib/timeline/routes/export_timelines_route.test.ts +++ b/x-pack/legacy/plugins/siem/server/lib/timeline/routes/export_timelines_route.test.ts @@ -83,7 +83,7 @@ describe('export timelines', () => { }); describe('request validation', () => { - test('disallows singular id query param', async () => { + test('return validation error for query param', async () => { const request = requestMock.create({ method: 'get', path: TIMELINE_EXPORT_URL, @@ -91,7 +91,26 @@ describe('export timelines', () => { }); const result = server.validate(request); - expect(result.badRequest).toHaveBeenCalledWith('"id" is not allowed'); + expect(result.badRequest.mock.calls[0][0]).toEqual( + 'Invalid value undefined supplied to : { ids: Array }/ids: Array' + ); + }); + + test('return validation error for query body', async () => { + const request = requestMock.create({ + method: 'get', + path: TIMELINE_EXPORT_URL, + body: { id: 'someId' }, + }); + const result = server.validate(request); + + expect(result.badRequest.mock.calls[1][0]).toEqual( + [ + 'Invalid value undefined supplied to : { file_name: string, exclude_export_details: ("true" | "false") }/file_name: string', + 'Invalid value undefined supplied to : { file_name: string, exclude_export_details: ("true" | "false") }/exclude_export_details: ("true" | "false")/0: "true"', + 'Invalid value undefined supplied to : { file_name: string, exclude_export_details: ("true" | "false") }/exclude_export_details: ("true" | "false")/1: "false"', + ].join('\n') + ); }); }); }); diff --git a/x-pack/legacy/plugins/siem/server/lib/timeline/routes/import_timelines_route.test.ts b/x-pack/legacy/plugins/siem/server/lib/timeline/routes/import_timelines_route.test.ts index e89aef4c70ecb..352f8f0a355fc 100644 --- a/x-pack/legacy/plugins/siem/server/lib/timeline/routes/import_timelines_route.test.ts +++ b/x-pack/legacy/plugins/siem/server/lib/timeline/routes/import_timelines_route.test.ts @@ -334,7 +334,10 @@ describe('import timelines', () => { const result = server.validate(request); expect(result.badRequest).toHaveBeenCalledWith( - 'child "file" fails because ["file" is required]' + [ + 'Invalid value undefined supplied to : { file: (ReadableRt & { hapi: { filename: string } }) }/file: (ReadableRt & { hapi: { filename: string } })/0: ReadableRt', + 'Invalid value undefined supplied to : { file: (ReadableRt & { hapi: { filename: string } }) }/file: (ReadableRt & { hapi: { filename: string } })/1: { hapi: { filename: string } }', + ].join('\n') ); }); }); diff --git a/x-pack/legacy/plugins/siem/server/lib/timeline/routes/schemas/export_timelines_schema.ts b/x-pack/legacy/plugins/siem/server/lib/timeline/routes/schemas/export_timelines_schema.ts index 41f7e042fb10c..b562b634d7212 100644 --- a/x-pack/legacy/plugins/siem/server/lib/timeline/routes/schemas/export_timelines_schema.ts +++ b/x-pack/legacy/plugins/siem/server/lib/timeline/routes/schemas/export_timelines_schema.ts @@ -26,21 +26,15 @@ export const exportTimelinesQuerySchema = rt.type({ export const exportTimelinesRequestBodySchema = rt.type({ ids, }); -type ErrorFactory = (message: string) => Error; -export const createPlainError = (message: string) => new Error(message); - -export const throwErrors = (createError: ErrorFactory) => (errors: rt.Errors) => - createError(failure(errors).join('\n')); - -export const decodeOrThrow = ( - runtimeType: rt.Type, - createError: ErrorFactory = createPlainError -) => (inputValue: I, validationResult: RouteValidationResultFactory) => +export const decodeOrThrow = (runtimeType: rt.Type) => ( + inputValue: I, + validationResult: RouteValidationResultFactory +) => pipe( runtimeType.decode(inputValue), fold>( - (errors: rt.Errors) => validationResult.badRequest(throwErrors(createError)(errors)), + (errors: rt.Errors) => validationResult.badRequest(failure(errors).join('\n')), (validatedInput: A) => validationResult.ok(validatedInput) ) ); From f5913589b6b7251286005d08bab43a5d1b9035f0 Mon Sep 17 00:00:00 2001 From: Angela Chuang Date: Thu, 9 Apr 2020 10:31:34 +0100 Subject: [PATCH 12/21] update description for test cases --- .../server/lib/timeline/routes/export_timelines_route.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/legacy/plugins/siem/server/lib/timeline/routes/export_timelines_route.test.ts b/x-pack/legacy/plugins/siem/server/lib/timeline/routes/export_timelines_route.test.ts index db4d66f6ef984..4eadede40f5d9 100644 --- a/x-pack/legacy/plugins/siem/server/lib/timeline/routes/export_timelines_route.test.ts +++ b/x-pack/legacy/plugins/siem/server/lib/timeline/routes/export_timelines_route.test.ts @@ -83,7 +83,7 @@ describe('export timelines', () => { }); describe('request validation', () => { - test('return validation error for query param', async () => { + test('return validation error for request body', async () => { const request = requestMock.create({ method: 'get', path: TIMELINE_EXPORT_URL, @@ -96,7 +96,7 @@ describe('export timelines', () => { ); }); - test('return validation error for query body', async () => { + test('return validation error for request params', async () => { const request = requestMock.create({ method: 'get', path: TIMELINE_EXPORT_URL, From d34f645d2b499a7a18de78c35822ccc398eca2c5 Mon Sep 17 00:00:00 2001 From: Angela Chuang Date: Thu, 9 Apr 2020 11:11:17 +0100 Subject: [PATCH 13/21] remove import from case --- .../create_timelines_stream_from_ndjson.ts | 24 +++++++++++++++---- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/x-pack/legacy/plugins/siem/server/lib/timeline/create_timelines_stream_from_ndjson.ts b/x-pack/legacy/plugins/siem/server/lib/timeline/create_timelines_stream_from_ndjson.ts index 7ddcd28fc7f45..5f1d6cf61afcd 100644 --- a/x-pack/legacy/plugins/siem/server/lib/timeline/create_timelines_stream_from_ndjson.ts +++ b/x-pack/legacy/plugins/siem/server/lib/timeline/create_timelines_stream_from_ndjson.ts @@ -5,6 +5,10 @@ */ import { Transform } from 'stream'; +import { pipe } from 'fp-ts/lib/pipeable'; +import { fold } from 'fp-ts/lib/Either'; +import { failure } from 'io-ts/lib/PathReporter'; +import { identity } from 'fp-ts/lib/function'; import { createConcatStream, createSplitStream, @@ -18,14 +22,24 @@ import { import { ImportTimelineResponse } from './routes/utils/import_timelines'; import { ImportTimelinesSchemaRt } from './routes/schemas/import_timelines_schema'; -import { decodeOrThrow } from '../../../../../../plugins/case/common/api'; -export const validateTimelines = (): Transform => { - return createMapStream((obj: ImportTimelineResponse) => { - return decodeOrThrow(ImportTimelinesSchemaRt)(obj); - }); +type ErrorFactory = (message: string) => Error; + +export const createPlainError = (message: string) => new Error(message); + +export const throwErrors = (createError: ErrorFactory) => (errors: rt.Errors) => { + throw createError(failure(errors).join('\n')); }; +export const decodeOrThrow = ( + runtimeType: rt.Type, + createError: ErrorFactory = createPlainError +) => (inputValue: I) => + pipe(runtimeType.decode(inputValue), fold(throwErrors(createError), identity)); + +export const validateTimelines = (): Transform => + createMapStream((obj: ImportTimelineResponse) => decodeOrThrow(ImportTimelinesSchemaRt)(obj)); + export const createTimelinesStreamFromNdJson = (ruleLimit: number) => { return [ createSplitStream('\n'), From a9bc45db4bce1185894cfea3bf56af08517f3cf7 Mon Sep 17 00:00:00 2001 From: Angela Chuang Date: Thu, 9 Apr 2020 11:56:59 +0100 Subject: [PATCH 14/21] lifting common libs --- .../routes/rules/import_rules_route.test.ts | 2 +- .../routes/rules/import_rules_route.ts | 2 +- .../routes/rules/utils.test.ts | 45 +- .../detection_engine/routes/rules/utils.ts | 9 - .../rules/create_rules_stream_from_ndjson.ts | 45 +- .../create_timelines_stream_from_ndjson.ts | 4 +- .../timeline/routes/utils/export_timelines.ts | 2 +- .../create_rules_stream_from_ndjson.test.ts | 483 ++++++++++++++++++ .../create_rules_stream_from_ndjson.ts | 96 ++++ 9 files changed, 590 insertions(+), 98 deletions(-) create mode 100644 x-pack/legacy/plugins/siem/server/utils/read_stream/create_rules_stream_from_ndjson.test.ts create mode 100644 x-pack/legacy/plugins/siem/server/utils/read_stream/create_rules_stream_from_ndjson.ts diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/import_rules_route.test.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/import_rules_route.test.ts index 61f5e6faf1bdb..bc0b1e1330bc3 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/import_rules_route.test.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/import_rules_route.test.ts @@ -24,7 +24,7 @@ import { import { createMockConfig, requestContextMock, serverMock, requestMock } from '../__mocks__'; import { importRulesRoute } from './import_rules_route'; import { DEFAULT_SIGNALS_INDEX } from '../../../../../common/constants'; -import * as createRulesStreamFromNdJson from '../../rules/create_rules_stream_from_ndjson'; +import * as createRulesStreamFromNdJson from '../../../../utils/read_stream/create_rules_stream_from_ndjson'; import { setFeatureFlagsForTestsOnly, unSetFeatureFlagsForTestsOnly } from '../../feature_flags'; describe('import_rules_route', () => { diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/import_rules_route.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/import_rules_route.ts index 43e970702ba72..59621c64cadb3 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/import_rules_route.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/import_rules_route.ts @@ -26,7 +26,7 @@ import { buildSiemResponse, validateLicenseForRuleType, } from '../utils'; -import { createRulesStreamFromNdJson } from '../../rules/create_rules_stream_from_ndjson'; +import { createRulesStreamFromNdJson } from '../../../../utils/read_stream/create_rules_stream_from_ndjson'; import { ImportRuleAlertRest } from '../../types'; import { patchRules } from '../../rules/patch_rules'; import { importRulesQuerySchema, importRulesPayloadSchema } from '../schemas/import_rules_schema'; diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/utils.test.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/utils.test.ts index 31a0f37fe81c9..217f4822634c0 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/utils.test.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/utils.test.ts @@ -12,7 +12,6 @@ import { transformTags, getIdBulkError, transformOrBulkError, - transformDataToNdjson, transformAlertsToRules, transformOrImportError, getDuplicates, @@ -22,9 +21,8 @@ import { getResult } from '../__mocks__/request_responses'; import { INTERNAL_IDENTIFIER } from '../../../../../common/constants'; import { ImportRuleAlertRest, RuleAlertParamsRest, RuleTypeParams } from '../../types'; import { BulkError, ImportSuccessError } from '../utils'; -import { sampleRule } from '../../signals/__mocks__/es_results'; import { getSimpleRule, getOutputRuleAlertForRest } from '../__mocks__/utils'; -import { createRulesStreamFromNdJson } from '../../rules/create_rules_stream_from_ndjson'; +import { createRulesStreamFromNdJson } from '../../../../utils/read_stream/create_rules_stream_from_ndjson'; import { createPromiseFromStreams } from '../../../../../../../../../src/legacy/utils/streams'; import { PartialAlert } from '../../../../../../../../plugins/alerting/server'; import { SanitizedAlert } from '../../../../../../../../plugins/alerting/server/types'; @@ -396,47 +394,6 @@ describe('utils', () => { }); }); - describe('transformDataToNdjson', () => { - test('if rules are empty it returns an empty string', () => { - const ruleNdjson = transformDataToNdjson([]); - expect(ruleNdjson).toEqual(''); - }); - - test('single rule will transform with new line ending character for ndjson', () => { - const rule = sampleRule(); - const ruleNdjson = transformDataToNdjson([rule]); - expect(ruleNdjson.endsWith('\n')).toBe(true); - }); - - test('multiple rules will transform with two new line ending characters for ndjson', () => { - const result1 = sampleRule(); - const result2 = sampleRule(); - result2.id = 'some other id'; - result2.rule_id = 'some other id'; - result2.name = 'Some other rule'; - - const ruleNdjson = transformDataToNdjson([result1, result2]); - // this is how we count characters in JavaScript :-) - const count = ruleNdjson.split('\n').length - 1; - expect(count).toBe(2); - }); - - test('you can parse two rules back out without errors', () => { - const result1 = sampleRule(); - const result2 = sampleRule(); - result2.id = 'some other id'; - result2.rule_id = 'some other id'; - result2.name = 'Some other rule'; - - const ruleNdjson = transformDataToNdjson([result1, result2]); - const ruleStrings = ruleNdjson.split('\n'); - const reParsed1 = JSON.parse(ruleStrings[0]); - const reParsed2 = JSON.parse(ruleStrings[1]); - expect(reParsed1).toEqual(result1); - expect(reParsed2).toEqual(result2); - }); - }); - describe('transformAlertsToRules', () => { test('given an empty array returns an empty array', () => { expect(transformAlertsToRules([])).toEqual([]); diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/utils.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/utils.ts index 4d13fa1b6ae50..790603fa8cfc1 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/utils.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/utils.ts @@ -152,15 +152,6 @@ export const transformAlertToRule = ( }); }; -export const transformDataToNdjson = (data: unknown[]): string => { - if (data.length !== 0) { - const dataString = data.map(rule => JSON.stringify(rule)).join('\n'); - return `${dataString}\n`; - } else { - return ''; - } -}; - export const transformAlertsToRules = ( alerts: RuleAlertType[] ): Array> => { diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/create_rules_stream_from_ndjson.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/create_rules_stream_from_ndjson.ts index 3e22999528101..42c975c3bd3a8 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/create_rules_stream_from_ndjson.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/create_rules_stream_from_ndjson.ts @@ -4,39 +4,19 @@ * you may not use this file except in compliance with the Elastic License. */ import { Transform } from 'stream'; -import { has, isString } from 'lodash/fp'; import { ImportRuleAlertRest } from '../types'; import { createSplitStream, createMapStream, - createFilterStream, createConcatStream, } from '../../../../../../../../src/legacy/utils/streams'; import { importRulesSchema } from '../routes/schemas/import_rules_schema'; import { BadRequestError } from '../errors/bad_request_error'; - -export interface RulesObjectsExportResultDetails { - /** number of successfully exported objects */ - exportedCount: number; -} - -export const parseNdjsonStrings = (): Transform => { - return createMapStream((ndJsonStr: string) => { - if (isString(ndJsonStr) && ndJsonStr.trim() !== '') { - try { - return JSON.parse(ndJsonStr); - } catch (err) { - return err; - } - } - }); -}; - -export const filterExportedCounts = (): Transform => { - return createFilterStream( - obj => obj != null && !has('exported_count', obj) - ); -}; +import { + parseNdjsonStrings, + filterExportedCounts, + createLimitStream, +} from '../../../utils/read_stream/create_rules_stream_from_ndjson'; export const validateRules = (): Transform => { return createMapStream((obj: ImportRuleAlertRest) => { @@ -53,21 +33,6 @@ export const validateRules = (): Transform => { }); }; -// Adaptation from: saved_objects/import/create_limit_stream.ts -export const createLimitStream = (limit: number): Transform => { - let counter = 0; - return new Transform({ - objectMode: true, - async transform(obj, _, done) { - if (counter >= limit) { - return done(new Error(`Can't import more than ${limit} rules`)); - } - counter++; - done(undefined, obj); - }, - }); -}; - // TODO: Capture both the line number and the rule_id if you have that information for the error message // eventually and then pass it down so we can give error messages on the line number diff --git a/x-pack/legacy/plugins/siem/server/lib/timeline/create_timelines_stream_from_ndjson.ts b/x-pack/legacy/plugins/siem/server/lib/timeline/create_timelines_stream_from_ndjson.ts index 5f1d6cf61afcd..2be45c98b9692 100644 --- a/x-pack/legacy/plugins/siem/server/lib/timeline/create_timelines_stream_from_ndjson.ts +++ b/x-pack/legacy/plugins/siem/server/lib/timeline/create_timelines_stream_from_ndjson.ts @@ -3,7 +3,7 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ - +import * as rt from 'io-ts'; import { Transform } from 'stream'; import { pipe } from 'fp-ts/lib/pipeable'; import { fold } from 'fp-ts/lib/Either'; @@ -18,7 +18,7 @@ import { parseNdjsonStrings, filterExportedCounts, createLimitStream, -} from '../detection_engine/rules/create_rules_stream_from_ndjson'; +} from '../../utils/read_stream/create_rules_stream_from_ndjson'; import { ImportTimelineResponse } from './routes/utils/import_timelines'; import { ImportTimelinesSchemaRt } from './routes/schemas/import_timelines_schema'; diff --git a/x-pack/legacy/plugins/siem/server/lib/timeline/routes/utils/export_timelines.ts b/x-pack/legacy/plugins/siem/server/lib/timeline/routes/utils/export_timelines.ts index 8a28100fbae82..6649b892f6343 100644 --- a/x-pack/legacy/plugins/siem/server/lib/timeline/routes/utils/export_timelines.ts +++ b/x-pack/legacy/plugins/siem/server/lib/timeline/routes/utils/export_timelines.ts @@ -30,8 +30,8 @@ import { ExportedNotes, TimelineSavedObject, } from '../../types'; +import { transformDataToNdjson } from '../../../../utils/read_stream/create_rules_stream_from_ndjson'; -import { transformDataToNdjson } from '../../../detection_engine/routes/rules/utils'; export type TimelineSavedObjectsClient = Pick< SavedObjectsClient, | 'get' diff --git a/x-pack/legacy/plugins/siem/server/utils/read_stream/create_rules_stream_from_ndjson.test.ts b/x-pack/legacy/plugins/siem/server/utils/read_stream/create_rules_stream_from_ndjson.test.ts new file mode 100644 index 0000000000000..3204e6b50e0d9 --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/utils/read_stream/create_rules_stream_from_ndjson.test.ts @@ -0,0 +1,483 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { Readable } from 'stream'; +import { + createRulesStreamFromNdJson, + transformDataToNdjson, +} from './create_rules_stream_from_ndjson'; +import { createPromiseFromStreams } from '../../../../../../../src/legacy/utils/streams'; +import { ImportRuleAlertRest } from '../../lib/detection_engine/types'; +import { BadRequestError } from '../../lib/detection_engine/errors/bad_request_error'; +import { sampleRule } from '../../lib/detection_engine/signals/__mocks__/es_results'; + +type PromiseFromStreams = ImportRuleAlertRest | Error; + +export const getOutputSample = (): Partial => ({ + rule_id: 'rule-1', + output_index: '.siem-signals', + risk_score: 50, + description: 'some description', + from: 'now-5m', + to: 'now', + index: ['index-1'], + name: 'some-name', + severity: 'low', + interval: '5m', + type: 'query', +}); + +export const getSampleAsNdjson = (sample: Partial): string => { + return `${JSON.stringify(sample)}\n`; +}; + +describe('create_rules_stream_from_ndjson', () => { + describe('createRulesStreamFromNdJson', () => { + test('transforms an ndjson stream into a stream of rule objects', async () => { + const sample1 = getOutputSample(); + const sample2 = getOutputSample(); + sample2.rule_id = 'rule-2'; + const ndJsonStream = new Readable({ + read() { + this.push(getSampleAsNdjson(sample1)); + this.push(getSampleAsNdjson(sample2)); + this.push(null); + }, + }); + const rulesObjectsStream = createRulesStreamFromNdJson(1000); + const result = await createPromiseFromStreams([ + ndJsonStream, + ...rulesObjectsStream, + ]); + expect(result).toEqual([ + { + actions: [], + rule_id: 'rule-1', + output_index: '.siem-signals', + risk_score: 50, + description: 'some description', + from: 'now-5m', + to: 'now', + index: ['index-1'], + name: 'some-name', + severity: 'low', + interval: '5m', + type: 'query', + enabled: true, + false_positives: [], + immutable: false, + query: '', + language: 'kuery', + lists: [], + max_signals: 100, + tags: [], + threat: [], + throttle: null, + references: [], + version: 1, + }, + { + actions: [], + rule_id: 'rule-2', + output_index: '.siem-signals', + risk_score: 50, + description: 'some description', + from: 'now-5m', + to: 'now', + index: ['index-1'], + name: 'some-name', + severity: 'low', + interval: '5m', + type: 'query', + enabled: true, + false_positives: [], + immutable: false, + query: '', + language: 'kuery', + lists: [], + max_signals: 100, + tags: [], + threat: [], + throttle: null, + references: [], + version: 1, + }, + ]); + }); + + test('returns error when ndjson stream is larger than limit', async () => { + const sample1 = getOutputSample(); + const sample2 = getOutputSample(); + sample2.rule_id = 'rule-2'; + const ndJsonStream = new Readable({ + read() { + this.push(getSampleAsNdjson(sample1)); + this.push(getSampleAsNdjson(sample2)); + }, + }); + const rulesObjectsStream = createRulesStreamFromNdJson(1); + await expect( + createPromiseFromStreams([ndJsonStream, ...rulesObjectsStream]) + ).rejects.toThrowError("Can't import more than 1 rules"); + }); + + test('skips empty lines', async () => { + const sample1 = getOutputSample(); + const sample2 = getOutputSample(); + sample2.rule_id = 'rule-2'; + const ndJsonStream = new Readable({ + read() { + this.push(getSampleAsNdjson(sample1)); + this.push('\n'); + this.push(getSampleAsNdjson(sample2)); + this.push(''); + this.push(null); + }, + }); + const rulesObjectsStream = createRulesStreamFromNdJson(1000); + const result = await createPromiseFromStreams([ + ndJsonStream, + ...rulesObjectsStream, + ]); + expect(result).toEqual([ + { + actions: [], + rule_id: 'rule-1', + output_index: '.siem-signals', + risk_score: 50, + description: 'some description', + from: 'now-5m', + to: 'now', + index: ['index-1'], + name: 'some-name', + severity: 'low', + interval: '5m', + type: 'query', + enabled: true, + false_positives: [], + immutable: false, + query: '', + language: 'kuery', + max_signals: 100, + tags: [], + lists: [], + threat: [], + throttle: null, + references: [], + version: 1, + }, + { + actions: [], + rule_id: 'rule-2', + output_index: '.siem-signals', + risk_score: 50, + description: 'some description', + from: 'now-5m', + to: 'now', + index: ['index-1'], + name: 'some-name', + severity: 'low', + interval: '5m', + type: 'query', + enabled: true, + false_positives: [], + immutable: false, + query: '', + language: 'kuery', + max_signals: 100, + lists: [], + tags: [], + threat: [], + throttle: null, + references: [], + version: 1, + }, + ]); + }); + + test('filters the export details entry from the stream', async () => { + const sample1 = getOutputSample(); + const sample2 = getOutputSample(); + sample2.rule_id = 'rule-2'; + const ndJsonStream = new Readable({ + read() { + this.push(getSampleAsNdjson(sample1)); + this.push(getSampleAsNdjson(sample2)); + this.push('{"exported_count":1,"missing_rules":[],"missing_rules_count":0}\n'); + this.push(null); + }, + }); + const rulesObjectsStream = createRulesStreamFromNdJson(1000); + const result = await createPromiseFromStreams([ + ndJsonStream, + ...rulesObjectsStream, + ]); + expect(result).toEqual([ + { + actions: [], + rule_id: 'rule-1', + output_index: '.siem-signals', + risk_score: 50, + description: 'some description', + from: 'now-5m', + to: 'now', + index: ['index-1'], + name: 'some-name', + severity: 'low', + interval: '5m', + type: 'query', + enabled: true, + false_positives: [], + immutable: false, + query: '', + language: 'kuery', + max_signals: 100, + lists: [], + tags: [], + threat: [], + throttle: null, + references: [], + version: 1, + }, + { + actions: [], + rule_id: 'rule-2', + output_index: '.siem-signals', + risk_score: 50, + description: 'some description', + from: 'now-5m', + to: 'now', + index: ['index-1'], + name: 'some-name', + severity: 'low', + interval: '5m', + type: 'query', + enabled: true, + false_positives: [], + immutable: false, + query: '', + language: 'kuery', + max_signals: 100, + lists: [], + tags: [], + threat: [], + throttle: null, + references: [], + version: 1, + }, + ]); + }); + + test('handles non parsable JSON strings and inserts the error as part of the return array', async () => { + const sample1 = getOutputSample(); + const sample2 = getOutputSample(); + sample2.rule_id = 'rule-2'; + const ndJsonStream = new Readable({ + read() { + this.push(getSampleAsNdjson(sample1)); + this.push('{,,,,\n'); + this.push(getSampleAsNdjson(sample2)); + this.push(null); + }, + }); + const rulesObjectsStream = createRulesStreamFromNdJson(1000); + const result = await createPromiseFromStreams([ + ndJsonStream, + ...rulesObjectsStream, + ]); + const resultOrError = result as Error[]; + expect(resultOrError[0]).toEqual({ + actions: [], + rule_id: 'rule-1', + output_index: '.siem-signals', + risk_score: 50, + description: 'some description', + from: 'now-5m', + to: 'now', + index: ['index-1'], + name: 'some-name', + severity: 'low', + interval: '5m', + type: 'query', + enabled: true, + false_positives: [], + immutable: false, + query: '', + language: 'kuery', + max_signals: 100, + lists: [], + tags: [], + threat: [], + throttle: null, + references: [], + version: 1, + }); + expect(resultOrError[1].message).toEqual('Unexpected token , in JSON at position 1'); + expect(resultOrError[2]).toEqual({ + actions: [], + rule_id: 'rule-2', + output_index: '.siem-signals', + risk_score: 50, + description: 'some description', + from: 'now-5m', + to: 'now', + index: ['index-1'], + name: 'some-name', + severity: 'low', + interval: '5m', + type: 'query', + enabled: true, + false_positives: [], + immutable: false, + query: '', + language: 'kuery', + max_signals: 100, + lists: [], + tags: [], + threat: [], + throttle: null, + references: [], + version: 1, + }); + }); + + test('handles non-validated data', async () => { + const sample1 = getOutputSample(); + const sample2 = getOutputSample(); + sample2.rule_id = 'rule-2'; + const ndJsonStream = new Readable({ + read() { + this.push(getSampleAsNdjson(sample1)); + this.push(`{}\n`); + this.push(getSampleAsNdjson(sample2)); + this.push(null); + }, + }); + const rulesObjectsStream = createRulesStreamFromNdJson(1000); + const result = await createPromiseFromStreams([ + ndJsonStream, + ...rulesObjectsStream, + ]); + const resultOrError = result as BadRequestError[]; + expect(resultOrError[0]).toEqual({ + actions: [], + rule_id: 'rule-1', + output_index: '.siem-signals', + risk_score: 50, + description: 'some description', + from: 'now-5m', + to: 'now', + index: ['index-1'], + name: 'some-name', + severity: 'low', + interval: '5m', + type: 'query', + enabled: true, + false_positives: [], + immutable: false, + query: '', + language: 'kuery', + max_signals: 100, + lists: [], + tags: [], + threat: [], + throttle: null, + references: [], + version: 1, + }); + expect(resultOrError[1].message).toEqual( + 'child "description" fails because ["description" is required]' + ); + expect(resultOrError[2]).toEqual({ + actions: [], + rule_id: 'rule-2', + output_index: '.siem-signals', + risk_score: 50, + description: 'some description', + from: 'now-5m', + to: 'now', + index: ['index-1'], + name: 'some-name', + severity: 'low', + interval: '5m', + type: 'query', + enabled: true, + false_positives: [], + immutable: false, + query: '', + language: 'kuery', + max_signals: 100, + lists: [], + tags: [], + threat: [], + throttle: null, + references: [], + version: 1, + }); + }); + + test('non validated data is an instanceof BadRequestError', async () => { + const sample1 = getOutputSample(); + const sample2 = getOutputSample(); + sample2.rule_id = 'rule-2'; + const ndJsonStream = new Readable({ + read() { + this.push(getSampleAsNdjson(sample1)); + this.push(`{}\n`); + this.push(getSampleAsNdjson(sample2)); + this.push(null); + }, + }); + const rulesObjectsStream = createRulesStreamFromNdJson(1000); + const result = await createPromiseFromStreams([ + ndJsonStream, + ...rulesObjectsStream, + ]); + const resultOrError = result as BadRequestError[]; + expect(resultOrError[1] instanceof BadRequestError).toEqual(true); + }); + }); + + describe('transformDataToNdjson', () => { + test('if rules are empty it returns an empty string', () => { + const ruleNdjson = transformDataToNdjson([]); + expect(ruleNdjson).toEqual(''); + }); + + test('single rule will transform with new line ending character for ndjson', () => { + const rule = sampleRule(); + const ruleNdjson = transformDataToNdjson([rule]); + expect(ruleNdjson.endsWith('\n')).toBe(true); + }); + + test('multiple rules will transform with two new line ending characters for ndjson', () => { + const result1 = sampleRule(); + const result2 = sampleRule(); + result2.id = 'some other id'; + result2.rule_id = 'some other id'; + result2.name = 'Some other rule'; + + const ruleNdjson = transformDataToNdjson([result1, result2]); + // this is how we count characters in JavaScript :-) + const count = ruleNdjson.split('\n').length - 1; + expect(count).toBe(2); + }); + + test('you can parse two rules back out without errors', () => { + const result1 = sampleRule(); + const result2 = sampleRule(); + result2.id = 'some other id'; + result2.rule_id = 'some other id'; + result2.name = 'Some other rule'; + + const ruleNdjson = transformDataToNdjson([result1, result2]); + const ruleStrings = ruleNdjson.split('\n'); + const reParsed1 = JSON.parse(ruleStrings[0]); + const reParsed2 = JSON.parse(ruleStrings[1]); + expect(reParsed1).toEqual(result1); + expect(reParsed2).toEqual(result2); + }); + }); +}); diff --git a/x-pack/legacy/plugins/siem/server/utils/read_stream/create_rules_stream_from_ndjson.ts b/x-pack/legacy/plugins/siem/server/utils/read_stream/create_rules_stream_from_ndjson.ts new file mode 100644 index 0000000000000..9eeaf79475c56 --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/utils/read_stream/create_rules_stream_from_ndjson.ts @@ -0,0 +1,96 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { Transform } from 'stream'; +import { has, isString } from 'lodash/fp'; +import { ImportRuleAlertRest } from '../../lib/detection_engine/types'; +import { + createSplitStream, + createMapStream, + createFilterStream, + createConcatStream, +} from '../../../../../../../src/legacy/utils/streams'; +import { importRulesSchema } from '../../lib/detection_engine/routes/schemas/import_rules_schema'; +import { BadRequestError } from '../../lib/detection_engine/errors/bad_request_error'; + +export interface RulesObjectsExportResultDetails { + /** number of successfully exported objects */ + exportedCount: number; +} + +export const parseNdjsonStrings = (): Transform => { + return createMapStream((ndJsonStr: string) => { + if (isString(ndJsonStr) && ndJsonStr.trim() !== '') { + try { + return JSON.parse(ndJsonStr); + } catch (err) { + return err; + } + } + }); +}; + +export const filterExportedCounts = (): Transform => { + return createFilterStream( + obj => obj != null && !has('exported_count', obj) + ); +}; + +export const validateRules = (): Transform => { + return createMapStream((obj: ImportRuleAlertRest) => { + if (!(obj instanceof Error)) { + const validated = importRulesSchema.validate(obj); + if (validated.error != null) { + return new BadRequestError(validated.error.message); + } else { + return validated.value; + } + } else { + return obj; + } + }); +}; + +// Adaptation from: saved_objects/import/create_limit_stream.ts +export const createLimitStream = (limit: number): Transform => { + let counter = 0; + return new Transform({ + objectMode: true, + async transform(obj, _, done) { + if (counter >= limit) { + return done(new Error(`Can't import more than ${limit} rules`)); + } + counter++; + done(undefined, obj); + }, + }); +}; + +// TODO: Capture both the line number and the rule_id if you have that information for the error message +// eventually and then pass it down so we can give error messages on the line number + +/** + * Inspiration and the pattern of code followed is from: + * saved_objects/lib/create_saved_objects_stream_from_ndjson.ts + */ +export const createRulesStreamFromNdJson = (ruleLimit: number) => { + return [ + createSplitStream('\n'), + parseNdjsonStrings(), + filterExportedCounts(), + validateRules(), + createLimitStream(ruleLimit), + createConcatStream([]), + ]; +}; + +export const transformDataToNdjson = (data: unknown[]): string => { + if (data.length !== 0) { + const dataString = data.map(rule => JSON.stringify(rule)).join('\n'); + return `${dataString}\n`; + } else { + return ''; + } +}; From 9283eff1b4d419c71214d5f2ab3eb317d0716915 Mon Sep 17 00:00:00 2001 From: Angela Chuang Date: Thu, 9 Apr 2020 16:23:18 +0100 Subject: [PATCH 15/21] fix dependency --- .../siem/server/lib/detection_engine/rules/get_export_all.ts | 3 ++- .../lib/detection_engine/rules/get_export_by_object_ids.ts | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/get_export_all.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/get_export_all.ts index 6a27abb66ce85..8997ff39e1965 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/get_export_all.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/get_export_all.ts @@ -7,7 +7,8 @@ import { AlertsClient } from '../../../../../../../plugins/alerting/server'; import { getNonPackagedRules } from './get_existing_prepackaged_rules'; import { getExportDetailsNdjson } from './get_export_details_ndjson'; -import { transformAlertsToRules, transformDataToNdjson } from '../routes/rules/utils'; +import { transformAlertsToRules } from '../routes/rules/utils'; +import { transformDataToNdjson } from '../../../utils/read_stream/create_rules_stream_from_ndjson'; export const getExportAll = async ( alertsClient: AlertsClient diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/get_export_by_object_ids.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/get_export_by_object_ids.ts index 6f642231ebbaf..f927d9d2d3365 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/get_export_by_object_ids.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/get_export_by_object_ids.ts @@ -8,8 +8,9 @@ import { AlertsClient } from '../../../../../../../plugins/alerting/server'; import { getExportDetailsNdjson } from './get_export_details_ndjson'; import { isAlertType } from '../rules/types'; import { readRules } from './read_rules'; -import { transformDataToNdjson, transformAlertToRule } from '../routes/rules/utils'; +import { transformAlertToRule } from '../routes/rules/utils'; import { OutputRuleAlertRest } from '../types'; +import { transformDataToNdjson } from '../../../utils/read_stream/create_rules_stream_from_ndjson'; interface ExportSuccesRule { statusCode: 200; From f8b3b03991747f03ac66937a032adbecd1ba239b Mon Sep 17 00:00:00 2001 From: Angela Chuang Date: Tue, 14 Apr 2020 10:22:41 +0100 Subject: [PATCH 16/21] lifting validation builder function --- .../timeline/routes/export_timelines_route.ts | 11 ++----- .../timeline/routes/import_timelines_route.ts | 15 ++-------- .../routes/schemas/export_timelines_schema.ts | 25 ---------------- .../build_validation/route_validation.ts | 29 +++++++++++++++++++ 4 files changed, 34 insertions(+), 46 deletions(-) create mode 100644 x-pack/legacy/plugins/siem/server/utils/build_validation/route_validation.ts diff --git a/x-pack/legacy/plugins/siem/server/lib/timeline/routes/export_timelines_route.ts b/x-pack/legacy/plugins/siem/server/lib/timeline/routes/export_timelines_route.ts index d09d669646dec..4a188bd01c8a1 100644 --- a/x-pack/legacy/plugins/siem/server/lib/timeline/routes/export_timelines_route.ts +++ b/x-pack/legacy/plugins/siem/server/lib/timeline/routes/export_timelines_route.ts @@ -15,21 +15,16 @@ import { getExportTimelineByObjectIds } from './utils/export_timelines'; import { exportTimelinesQuerySchema, exportTimelinesRequestBodySchema, - buildRouteValidation, } from './schemas/export_timelines_schema'; -import { ExportTimelinesQuery, ExportTimelinesRequestBody } from '../types'; +import { buildRouteValidation } from '../../../utils/build_validation/route_validation'; export const exportTimelinesRoute = (router: IRouter, config: LegacyServices['config']) => { router.post( { path: TIMELINE_EXPORT_URL, validate: { - query: buildRouteValidation( - exportTimelinesQuerySchema - ), - body: buildRouteValidation( - exportTimelinesRequestBodySchema - ), + query: buildRouteValidation(exportTimelinesQuerySchema), + body: buildRouteValidation(exportTimelinesRequestBodySchema), }, options: { tags: ['access:siem'], diff --git a/x-pack/legacy/plugins/siem/server/lib/timeline/routes/import_timelines_route.ts b/x-pack/legacy/plugins/siem/server/lib/timeline/routes/import_timelines_route.ts index a65a4fcb03454..91c4699c0a739 100644 --- a/x-pack/legacy/plugins/siem/server/lib/timeline/routes/import_timelines_route.ts +++ b/x-pack/legacy/plugins/siem/server/lib/timeline/routes/import_timelines_route.ts @@ -11,7 +11,6 @@ import { pipe } from 'fp-ts/lib/pipeable'; import { fold } from 'fp-ts/lib/Either'; import { identity } from 'fp-ts/lib/function'; -import { Readable } from 'stream'; import { buildSiemResponse, createBulkErrorObject, @@ -43,17 +42,11 @@ import { Timeline } from '../saved_object'; import { validate } from '../../detection_engine/routes/rules/validate'; import { FrameworkRequest } from '../../framework'; import { throwErrors } from '../../../../../../../plugins/case/common/api'; -import { buildRouteValidation } from './schemas/export_timelines_schema'; +import { buildRouteValidation } from '../../../utils/build_validation/route_validation'; const CHUNK_PARSED_OBJECT_SIZE = 10; const timelineLib = new Timeline(); -interface ImportTimelinesRequestBodySchema { - file: Readable & { - hapi: { filename: string }; - }; -} - export const importTimelinesRoute = ( router: IRouter, config: LegacyServices['config'], @@ -63,11 +56,7 @@ export const importTimelinesRoute = ( { path: `${TIMELINE_IMPORT_URL}`, validate: { - body: buildRouteValidation< - ImportTimelinesRequestBodySchema, - ImportTimelinesRequestBodySchema, - unknown - >(ImportTimelinesPayloadSchemaRt), + body: buildRouteValidation(ImportTimelinesPayloadSchemaRt), }, options: { tags: ['access:siem'], diff --git a/x-pack/legacy/plugins/siem/server/lib/timeline/routes/schemas/export_timelines_schema.ts b/x-pack/legacy/plugins/siem/server/lib/timeline/routes/schemas/export_timelines_schema.ts index b562b634d7212..840aad389dc2b 100644 --- a/x-pack/legacy/plugins/siem/server/lib/timeline/routes/schemas/export_timelines_schema.ts +++ b/x-pack/legacy/plugins/siem/server/lib/timeline/routes/schemas/export_timelines_schema.ts @@ -4,16 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import { fold } from 'fp-ts/lib/Either'; -import { pipe } from 'fp-ts/lib/pipeable'; import * as rt from 'io-ts'; -import { failure } from 'io-ts/lib/PathReporter'; - -import { - RouteValidationFunction, - RouteValidationResultFactory, -} from '../../../../../../../../../src/core/server'; -import { RequestValidationResult } from '../../types'; const ids = rt.array(rt.string); export const exportTimelinesSchema = rt.type({ ids }); @@ -26,19 +17,3 @@ export const exportTimelinesQuerySchema = rt.type({ export const exportTimelinesRequestBodySchema = rt.type({ ids, }); - -export const decodeOrThrow = (runtimeType: rt.Type) => ( - inputValue: I, - validationResult: RouteValidationResultFactory -) => - pipe( - runtimeType.decode(inputValue), - fold>( - (errors: rt.Errors) => validationResult.badRequest(failure(errors).join('\n')), - (validatedInput: A) => validationResult.ok(validatedInput) - ) - ); - -export const buildRouteValidation = ( - schema: rt.Type -): RouteValidationFunction => decodeOrThrow(schema); diff --git a/x-pack/legacy/plugins/siem/server/utils/build_validation/route_validation.ts b/x-pack/legacy/plugins/siem/server/utils/build_validation/route_validation.ts new file mode 100644 index 0000000000000..3e3cc233295fb --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/utils/build_validation/route_validation.ts @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { fold } from 'fp-ts/lib/Either'; +import { pipe } from 'fp-ts/lib/pipeable'; +import * as rt from 'io-ts'; +import { failure } from 'io-ts/lib/PathReporter'; +import { + RouteValidationFunction, + RouteValidationResultFactory, +} from '../../../../../../../src/core/server'; +import { RequestValidationResult } from '../../lib/timeline/types'; + +export const buildRouteValidation = >( + schema: T +): RouteValidationFunction => ( + inputValue: unknown, + validationResult: RouteValidationResultFactory +) => + pipe( + schema.decode(inputValue), + fold>( + (errors: rt.Errors) => validationResult.badRequest(failure(errors).join('\n')), + (validatedInput: A) => validationResult.ok(validatedInput) + ) + ); From 81b98594b4b3d63971fa3a0a51923f9f6a921f36 Mon Sep 17 00:00:00 2001 From: Angela Chuang Date: Tue, 14 Apr 2020 21:35:25 +0100 Subject: [PATCH 17/21] add unit test --- .../build_validation/route_validation.test.ts | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 x-pack/legacy/plugins/siem/server/utils/build_validation/route_validation.test.ts diff --git a/x-pack/legacy/plugins/siem/server/utils/build_validation/route_validation.test.ts b/x-pack/legacy/plugins/siem/server/utils/build_validation/route_validation.test.ts new file mode 100644 index 0000000000000..888cd5dfe5390 --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/utils/build_validation/route_validation.test.ts @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { buildRouteValidation } from './route_validation'; +import * as rt from 'io-ts'; +import { RouteValidationResultFactory } from '../../../../../../../src/core/server/http'; + +describe('buildRouteValidation', () => { + const schema = rt.type({ + ids: rt.array(rt.string), + }); + const validationResult: RouteValidationResultFactory = { + ok: jest.fn().mockImplementation(validatedInput => validatedInput), + badRequest: jest.fn().mockImplementation(e => e), + }; + + beforeEach(() => { + jest.clearAllMocks(); + }); + + test('return validation error', () => { + const input = { id: 'someId' }; + const result = buildRouteValidation(schema)(input, validationResult); + + expect(result).toEqual( + 'Invalid value undefined supplied to : { ids: Array }/ids: Array' + ); + }); + + test('return validated input', () => { + const input = { ids: ['someId'] }; + const result = buildRouteValidation(schema)(input, validationResult); + + expect(result).toEqual(input); + }); +}); From cb05d5dd0f4973ccd4b92ac7b4872671dac8cc2a Mon Sep 17 00:00:00 2001 From: Angela Chuang Date: Wed, 15 Apr 2020 11:09:58 +0100 Subject: [PATCH 18/21] fix for code review --- .../timeline/routes/export_timelines_route.ts | 2 +- .../timeline/routes/import_timelines_route.ts | 11 +------ .../routes/schemas/export_timelines_schema.ts | 5 +-- .../timeline/routes/utils/export_timelines.ts | 23 +++++-------- .../plugins/siem/server/lib/timeline/types.ts | 32 +------------------ .../build_validation/route_validation.ts | 12 ++++++- 6 files changed, 23 insertions(+), 62 deletions(-) diff --git a/x-pack/legacy/plugins/siem/server/lib/timeline/routes/export_timelines_route.ts b/x-pack/legacy/plugins/siem/server/lib/timeline/routes/export_timelines_route.ts index 4a188bd01c8a1..fa849c1c325a9 100644 --- a/x-pack/legacy/plugins/siem/server/lib/timeline/routes/export_timelines_route.ts +++ b/x-pack/legacy/plugins/siem/server/lib/timeline/routes/export_timelines_route.ts @@ -45,7 +45,7 @@ export const exportTimelinesRoute = (router: IRouter, config: LegacyServices['co const responseBody = await getExportTimelineByObjectIds({ client: savedObjectsClient, - request, + ids: request.body.ids, }); return response.ok({ diff --git a/x-pack/legacy/plugins/siem/server/lib/timeline/routes/import_timelines_route.ts b/x-pack/legacy/plugins/siem/server/lib/timeline/routes/import_timelines_route.ts index 91c4699c0a739..ad7ee28d8ad51 100644 --- a/x-pack/legacy/plugins/siem/server/lib/timeline/routes/import_timelines_route.ts +++ b/x-pack/legacy/plugins/siem/server/lib/timeline/routes/import_timelines_route.ts @@ -6,10 +6,6 @@ import { extname } from 'path'; import { chunk, omit, set } from 'lodash/fp'; -import Boom from 'boom'; -import { pipe } from 'fp-ts/lib/pipeable'; -import { fold } from 'fp-ts/lib/Either'; -import { identity } from 'fp-ts/lib/function'; import { buildSiemResponse, @@ -41,7 +37,6 @@ import { LegacyServices } from '../../../types'; import { Timeline } from '../saved_object'; import { validate } from '../../detection_engine/routes/rules/validate'; import { FrameworkRequest } from '../../framework'; -import { throwErrors } from '../../../../../../../plugins/case/common/api'; import { buildRouteValidation } from '../../../utils/build_validation/route_validation'; const CHUNK_PARSED_OBJECT_SIZE = 10; @@ -74,11 +69,7 @@ export const importTimelinesRoute = ( return siemResponse.error({ statusCode: 404 }); } - const { file } = pipe( - ImportTimelinesPayloadSchemaRt.decode(request.body), - fold(throwErrors(Boom.badRequest), identity) - ); - + const { file } = request.body; const { filename } = file.hapi; const fileExtension = extname(filename).toLowerCase(); diff --git a/x-pack/legacy/plugins/siem/server/lib/timeline/routes/schemas/export_timelines_schema.ts b/x-pack/legacy/plugins/siem/server/lib/timeline/routes/schemas/export_timelines_schema.ts index 840aad389dc2b..6f8265903b2a7 100644 --- a/x-pack/legacy/plugins/siem/server/lib/timeline/routes/schemas/export_timelines_schema.ts +++ b/x-pack/legacy/plugins/siem/server/lib/timeline/routes/schemas/export_timelines_schema.ts @@ -6,14 +6,11 @@ import * as rt from 'io-ts'; -const ids = rt.array(rt.string); -export const exportTimelinesSchema = rt.type({ ids }); - export const exportTimelinesQuerySchema = rt.type({ file_name: rt.string, exclude_export_details: rt.union([rt.literal('true'), rt.literal('false')]), }); export const exportTimelinesRequestBodySchema = rt.type({ - ids, + ids: rt.array(rt.string), }); diff --git a/x-pack/legacy/plugins/siem/server/lib/timeline/routes/utils/export_timelines.ts b/x-pack/legacy/plugins/siem/server/lib/timeline/routes/utils/export_timelines.ts index 6649b892f6343..be191d76d3363 100644 --- a/x-pack/legacy/plugins/siem/server/lib/timeline/routes/utils/export_timelines.ts +++ b/x-pack/legacy/plugins/siem/server/lib/timeline/routes/utils/export_timelines.ts @@ -26,7 +26,6 @@ import { import { ExportedTimelines, ExportTimelineSavedObjectsClient, - ExportTimelineRequest, ExportedNotes, TimelineSavedObject, } from '../../types'; @@ -142,23 +141,17 @@ const getTimelines = async ( const getTimelinesFromObjects = async ( savedObjectsClient: ExportTimelineSavedObjectsClient, - request: ExportTimelineRequest + ids: string[] ): Promise => { - const timelines: TimelineSavedObject[] = await getTimelines(savedObjectsClient, request.body.ids); + const timelines: TimelineSavedObject[] = await getTimelines(savedObjectsClient, ids); // To Do for feature freeze // if (timelines.length !== request.body.ids.length) { // //figure out which is missing to tell user // } const [notes, pinnedEventIds] = await Promise.all([ - Promise.all( - request.body.ids.map(timelineId => getNotesByTimelineId(savedObjectsClient, timelineId)) - ), - Promise.all( - request.body.ids.map(timelineId => - getPinnedEventsByTimelineId(savedObjectsClient, timelineId) - ) - ), + Promise.all(ids.map(timelineId => getNotesByTimelineId(savedObjectsClient, timelineId))), + Promise.all(ids.map(timelineId => getPinnedEventsByTimelineId(savedObjectsClient, timelineId))), ]); const myNotes = notes.reduce( @@ -171,7 +164,7 @@ const getTimelinesFromObjects = async ( [] ); - const myResponse = request.body.ids.reduce((acc, timelineId) => { + const myResponse = ids.reduce((acc, timelineId) => { const myTimeline = timelines.find(t => t.savedObjectId === timelineId); if (myTimeline != null) { const timelineNotes = myNotes.filter(n => n.timelineId === timelineId); @@ -193,11 +186,11 @@ const getTimelinesFromObjects = async ( export const getExportTimelineByObjectIds = async ({ client, - request, + ids, }: { client: ExportTimelineSavedObjectsClient; - request: ExportTimelineRequest; + ids: string[]; }) => { - const timeline = await getTimelinesFromObjects(client, request); + const timeline = await getTimelinesFromObjects(client, ids); return transformDataToNdjson(timeline); }; diff --git a/x-pack/legacy/plugins/siem/server/lib/timeline/types.ts b/x-pack/legacy/plugins/siem/server/lib/timeline/types.ts index cf11f54a93323..523221192eca4 100644 --- a/x-pack/legacy/plugins/siem/server/lib/timeline/types.ts +++ b/x-pack/legacy/plugins/siem/server/lib/timeline/types.ts @@ -14,11 +14,7 @@ import { PinnedEventToReturnSavedObjectRuntimeType, PinnedEventSavedObject, } from '../pinned_event/types'; -import { - SavedObjectsClient, - KibanaRequest, - RouteValidationError, -} from '../../../../../../../src/core/server'; +import { SavedObjectsClient } from '../../../../../../../src/core/server'; /* * ColumnHeader Types @@ -212,32 +208,6 @@ export interface AllTimelineSavedObject * Import/export timelines */ -export interface ExportTimelinesQuery { - file_name: string; - exclude_export_details: 'true' | 'false'; -} - -export interface ExportTimelinesRequestBody { - ids: string[]; -} - -export type ExportTimelineRequest = KibanaRequest< - unknown, - ExportTimelinesQuery, - ExportTimelinesRequestBody, - 'post' ->; - -export type RequestValidationResult = - | { - value: T; - error?: undefined; - } - | { - value?: undefined; - error: RouteValidationError; - }; - export type ExportTimelineSavedObjectsClient = Pick< SavedObjectsClient, | 'get' diff --git a/x-pack/legacy/plugins/siem/server/utils/build_validation/route_validation.ts b/x-pack/legacy/plugins/siem/server/utils/build_validation/route_validation.ts index 3e3cc233295fb..1281c23cbc89a 100644 --- a/x-pack/legacy/plugins/siem/server/utils/build_validation/route_validation.ts +++ b/x-pack/legacy/plugins/siem/server/utils/build_validation/route_validation.ts @@ -11,8 +11,18 @@ import { failure } from 'io-ts/lib/PathReporter'; import { RouteValidationFunction, RouteValidationResultFactory, + RouteValidationError, } from '../../../../../../../src/core/server'; -import { RequestValidationResult } from '../../lib/timeline/types'; + +type RequestValidationResult = + | { + value: T; + error?: undefined; + } + | { + value?: undefined; + error: RouteValidationError; + }; export const buildRouteValidation = >( schema: T From 4f45e1739a0177fb48b3e5c3346441cf87d156f2 Mon Sep 17 00:00:00 2001 From: Angela Chuang Date: Wed, 15 Apr 2020 16:29:22 +0100 Subject: [PATCH 19/21] reve comments --- .../utils/read_stream/create_rules_stream_from_ndjson.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/x-pack/legacy/plugins/siem/server/utils/read_stream/create_rules_stream_from_ndjson.ts b/x-pack/legacy/plugins/siem/server/utils/read_stream/create_rules_stream_from_ndjson.ts index 9eeaf79475c56..5df238073a66d 100644 --- a/x-pack/legacy/plugins/siem/server/utils/read_stream/create_rules_stream_from_ndjson.ts +++ b/x-pack/legacy/plugins/siem/server/utils/read_stream/create_rules_stream_from_ndjson.ts @@ -68,9 +68,6 @@ export const createLimitStream = (limit: number): Transform => { }); }; -// TODO: Capture both the line number and the rule_id if you have that information for the error message -// eventually and then pass it down so we can give error messages on the line number - /** * Inspiration and the pattern of code followed is from: * saved_objects/lib/create_saved_objects_stream_from_ndjson.ts From ddf440961701d0ec0452ba48263788f919746a7a Mon Sep 17 00:00:00 2001 From: Angela Chuang Date: Wed, 15 Apr 2020 20:46:45 +0100 Subject: [PATCH 20/21] rename common utils --- .../routes/rules/import_rules_route.test.ts | 2 +- .../routes/rules/import_rules_route.ts | 2 +- .../routes/rules/utils.test.ts | 2 +- .../rules/create_rules_stream_from_ndjson.ts | 2 +- .../detection_engine/rules/get_export_all.ts | 2 +- .../rules/get_export_by_object_ids.ts | 2 +- .../create_timelines_stream_from_ndjson.ts | 2 +- .../timeline/routes/utils/export_timelines.ts | 2 +- .../create_rules_stream_from_ndjson.test.ts | 483 ------------------ .../create_stream_from_ndjson.test.ts | 74 +++ ...ndjson.ts => create_stream_from_ndjson.ts} | 15 - 11 files changed, 82 insertions(+), 506 deletions(-) delete mode 100644 x-pack/legacy/plugins/siem/server/utils/read_stream/create_rules_stream_from_ndjson.test.ts create mode 100644 x-pack/legacy/plugins/siem/server/utils/read_stream/create_stream_from_ndjson.test.ts rename x-pack/legacy/plugins/siem/server/utils/read_stream/{create_rules_stream_from_ndjson.ts => create_stream_from_ndjson.ts} (86%) diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/import_rules_route.test.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/import_rules_route.test.ts index bc0b1e1330bc3..61f5e6faf1bdb 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/import_rules_route.test.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/import_rules_route.test.ts @@ -24,7 +24,7 @@ import { import { createMockConfig, requestContextMock, serverMock, requestMock } from '../__mocks__'; import { importRulesRoute } from './import_rules_route'; import { DEFAULT_SIGNALS_INDEX } from '../../../../../common/constants'; -import * as createRulesStreamFromNdJson from '../../../../utils/read_stream/create_rules_stream_from_ndjson'; +import * as createRulesStreamFromNdJson from '../../rules/create_rules_stream_from_ndjson'; import { setFeatureFlagsForTestsOnly, unSetFeatureFlagsForTestsOnly } from '../../feature_flags'; describe('import_rules_route', () => { diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/import_rules_route.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/import_rules_route.ts index 9158fdb9cc33b..57ccc7a7806ac 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/import_rules_route.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/import_rules_route.ts @@ -26,13 +26,13 @@ import { buildSiemResponse, validateLicenseForRuleType, } from '../utils'; -import { createRulesStreamFromNdJson } from '../../../../utils/read_stream/create_rules_stream_from_ndjson'; import { ImportRuleAlertRest } from '../../types'; import { patchRules } from '../../rules/patch_rules'; import { importRulesQuerySchema, importRulesPayloadSchema } from '../schemas/import_rules_schema'; import { ImportRulesSchema, importRulesSchema } from '../schemas/response/import_rules_schema'; import { getTupleDuplicateErrorsAndUniqueRules } from './utils'; import { validate } from './validate'; +import { createRulesStreamFromNdJson } from '../../rules/create_rules_stream_from_ndjson'; type PromiseFromStreams = ImportRuleAlertRest | Error; diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/utils.test.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/utils.test.ts index 217f4822634c0..8b1b0cab3b2f2 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/utils.test.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/utils.test.ts @@ -22,12 +22,12 @@ import { INTERNAL_IDENTIFIER } from '../../../../../common/constants'; import { ImportRuleAlertRest, RuleAlertParamsRest, RuleTypeParams } from '../../types'; import { BulkError, ImportSuccessError } from '../utils'; import { getSimpleRule, getOutputRuleAlertForRest } from '../__mocks__/utils'; -import { createRulesStreamFromNdJson } from '../../../../utils/read_stream/create_rules_stream_from_ndjson'; import { createPromiseFromStreams } from '../../../../../../../../../src/legacy/utils/streams'; import { PartialAlert } from '../../../../../../../../plugins/alerting/server'; import { SanitizedAlert } from '../../../../../../../../plugins/alerting/server/types'; import { RuleAlertType } from '../../rules/types'; import { setFeatureFlagsForTestsOnly, unSetFeatureFlagsForTestsOnly } from '../../feature_flags'; +import { createRulesStreamFromNdJson } from '../../rules/create_rules_stream_from_ndjson'; type PromiseFromStreams = ImportRuleAlertRest | Error; diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/create_rules_stream_from_ndjson.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/create_rules_stream_from_ndjson.ts index 42c975c3bd3a8..27008d17d2192 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/create_rules_stream_from_ndjson.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/create_rules_stream_from_ndjson.ts @@ -16,7 +16,7 @@ import { parseNdjsonStrings, filterExportedCounts, createLimitStream, -} from '../../../utils/read_stream/create_rules_stream_from_ndjson'; +} from '../../../utils/read_stream/create_stream_from_ndjson'; export const validateRules = (): Transform => { return createMapStream((obj: ImportRuleAlertRest) => { diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/get_export_all.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/get_export_all.ts index 8997ff39e1965..40c07f28ea848 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/get_export_all.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/get_export_all.ts @@ -8,7 +8,7 @@ import { AlertsClient } from '../../../../../../../plugins/alerting/server'; import { getNonPackagedRules } from './get_existing_prepackaged_rules'; import { getExportDetailsNdjson } from './get_export_details_ndjson'; import { transformAlertsToRules } from '../routes/rules/utils'; -import { transformDataToNdjson } from '../../../utils/read_stream/create_rules_stream_from_ndjson'; +import { transformDataToNdjson } from '../../../utils/read_stream/create_stream_from_ndjson'; export const getExportAll = async ( alertsClient: AlertsClient diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/get_export_by_object_ids.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/get_export_by_object_ids.ts index f927d9d2d3365..048f09e95b062 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/get_export_by_object_ids.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/get_export_by_object_ids.ts @@ -10,7 +10,7 @@ import { isAlertType } from '../rules/types'; import { readRules } from './read_rules'; import { transformAlertToRule } from '../routes/rules/utils'; import { OutputRuleAlertRest } from '../types'; -import { transformDataToNdjson } from '../../../utils/read_stream/create_rules_stream_from_ndjson'; +import { transformDataToNdjson } from '../../../utils/read_stream/create_stream_from_ndjson'; interface ExportSuccesRule { statusCode: 200; diff --git a/x-pack/legacy/plugins/siem/server/lib/timeline/create_timelines_stream_from_ndjson.ts b/x-pack/legacy/plugins/siem/server/lib/timeline/create_timelines_stream_from_ndjson.ts index 2be45c98b9692..16654b2863ee5 100644 --- a/x-pack/legacy/plugins/siem/server/lib/timeline/create_timelines_stream_from_ndjson.ts +++ b/x-pack/legacy/plugins/siem/server/lib/timeline/create_timelines_stream_from_ndjson.ts @@ -18,7 +18,7 @@ import { parseNdjsonStrings, filterExportedCounts, createLimitStream, -} from '../../utils/read_stream/create_rules_stream_from_ndjson'; +} from '../../utils/read_stream/create_stream_from_ndjson'; import { ImportTimelineResponse } from './routes/utils/import_timelines'; import { ImportTimelinesSchemaRt } from './routes/schemas/import_timelines_schema'; diff --git a/x-pack/legacy/plugins/siem/server/lib/timeline/routes/utils/export_timelines.ts b/x-pack/legacy/plugins/siem/server/lib/timeline/routes/utils/export_timelines.ts index be191d76d3363..52ee2a891c9bb 100644 --- a/x-pack/legacy/plugins/siem/server/lib/timeline/routes/utils/export_timelines.ts +++ b/x-pack/legacy/plugins/siem/server/lib/timeline/routes/utils/export_timelines.ts @@ -29,7 +29,7 @@ import { ExportedNotes, TimelineSavedObject, } from '../../types'; -import { transformDataToNdjson } from '../../../../utils/read_stream/create_rules_stream_from_ndjson'; +import { transformDataToNdjson } from '../../../../utils/read_stream/create_stream_from_ndjson'; export type TimelineSavedObjectsClient = Pick< SavedObjectsClient, diff --git a/x-pack/legacy/plugins/siem/server/utils/read_stream/create_rules_stream_from_ndjson.test.ts b/x-pack/legacy/plugins/siem/server/utils/read_stream/create_rules_stream_from_ndjson.test.ts deleted file mode 100644 index 3204e6b50e0d9..0000000000000 --- a/x-pack/legacy/plugins/siem/server/utils/read_stream/create_rules_stream_from_ndjson.test.ts +++ /dev/null @@ -1,483 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -import { Readable } from 'stream'; -import { - createRulesStreamFromNdJson, - transformDataToNdjson, -} from './create_rules_stream_from_ndjson'; -import { createPromiseFromStreams } from '../../../../../../../src/legacy/utils/streams'; -import { ImportRuleAlertRest } from '../../lib/detection_engine/types'; -import { BadRequestError } from '../../lib/detection_engine/errors/bad_request_error'; -import { sampleRule } from '../../lib/detection_engine/signals/__mocks__/es_results'; - -type PromiseFromStreams = ImportRuleAlertRest | Error; - -export const getOutputSample = (): Partial => ({ - rule_id: 'rule-1', - output_index: '.siem-signals', - risk_score: 50, - description: 'some description', - from: 'now-5m', - to: 'now', - index: ['index-1'], - name: 'some-name', - severity: 'low', - interval: '5m', - type: 'query', -}); - -export const getSampleAsNdjson = (sample: Partial): string => { - return `${JSON.stringify(sample)}\n`; -}; - -describe('create_rules_stream_from_ndjson', () => { - describe('createRulesStreamFromNdJson', () => { - test('transforms an ndjson stream into a stream of rule objects', async () => { - const sample1 = getOutputSample(); - const sample2 = getOutputSample(); - sample2.rule_id = 'rule-2'; - const ndJsonStream = new Readable({ - read() { - this.push(getSampleAsNdjson(sample1)); - this.push(getSampleAsNdjson(sample2)); - this.push(null); - }, - }); - const rulesObjectsStream = createRulesStreamFromNdJson(1000); - const result = await createPromiseFromStreams([ - ndJsonStream, - ...rulesObjectsStream, - ]); - expect(result).toEqual([ - { - actions: [], - rule_id: 'rule-1', - output_index: '.siem-signals', - risk_score: 50, - description: 'some description', - from: 'now-5m', - to: 'now', - index: ['index-1'], - name: 'some-name', - severity: 'low', - interval: '5m', - type: 'query', - enabled: true, - false_positives: [], - immutable: false, - query: '', - language: 'kuery', - lists: [], - max_signals: 100, - tags: [], - threat: [], - throttle: null, - references: [], - version: 1, - }, - { - actions: [], - rule_id: 'rule-2', - output_index: '.siem-signals', - risk_score: 50, - description: 'some description', - from: 'now-5m', - to: 'now', - index: ['index-1'], - name: 'some-name', - severity: 'low', - interval: '5m', - type: 'query', - enabled: true, - false_positives: [], - immutable: false, - query: '', - language: 'kuery', - lists: [], - max_signals: 100, - tags: [], - threat: [], - throttle: null, - references: [], - version: 1, - }, - ]); - }); - - test('returns error when ndjson stream is larger than limit', async () => { - const sample1 = getOutputSample(); - const sample2 = getOutputSample(); - sample2.rule_id = 'rule-2'; - const ndJsonStream = new Readable({ - read() { - this.push(getSampleAsNdjson(sample1)); - this.push(getSampleAsNdjson(sample2)); - }, - }); - const rulesObjectsStream = createRulesStreamFromNdJson(1); - await expect( - createPromiseFromStreams([ndJsonStream, ...rulesObjectsStream]) - ).rejects.toThrowError("Can't import more than 1 rules"); - }); - - test('skips empty lines', async () => { - const sample1 = getOutputSample(); - const sample2 = getOutputSample(); - sample2.rule_id = 'rule-2'; - const ndJsonStream = new Readable({ - read() { - this.push(getSampleAsNdjson(sample1)); - this.push('\n'); - this.push(getSampleAsNdjson(sample2)); - this.push(''); - this.push(null); - }, - }); - const rulesObjectsStream = createRulesStreamFromNdJson(1000); - const result = await createPromiseFromStreams([ - ndJsonStream, - ...rulesObjectsStream, - ]); - expect(result).toEqual([ - { - actions: [], - rule_id: 'rule-1', - output_index: '.siem-signals', - risk_score: 50, - description: 'some description', - from: 'now-5m', - to: 'now', - index: ['index-1'], - name: 'some-name', - severity: 'low', - interval: '5m', - type: 'query', - enabled: true, - false_positives: [], - immutable: false, - query: '', - language: 'kuery', - max_signals: 100, - tags: [], - lists: [], - threat: [], - throttle: null, - references: [], - version: 1, - }, - { - actions: [], - rule_id: 'rule-2', - output_index: '.siem-signals', - risk_score: 50, - description: 'some description', - from: 'now-5m', - to: 'now', - index: ['index-1'], - name: 'some-name', - severity: 'low', - interval: '5m', - type: 'query', - enabled: true, - false_positives: [], - immutable: false, - query: '', - language: 'kuery', - max_signals: 100, - lists: [], - tags: [], - threat: [], - throttle: null, - references: [], - version: 1, - }, - ]); - }); - - test('filters the export details entry from the stream', async () => { - const sample1 = getOutputSample(); - const sample2 = getOutputSample(); - sample2.rule_id = 'rule-2'; - const ndJsonStream = new Readable({ - read() { - this.push(getSampleAsNdjson(sample1)); - this.push(getSampleAsNdjson(sample2)); - this.push('{"exported_count":1,"missing_rules":[],"missing_rules_count":0}\n'); - this.push(null); - }, - }); - const rulesObjectsStream = createRulesStreamFromNdJson(1000); - const result = await createPromiseFromStreams([ - ndJsonStream, - ...rulesObjectsStream, - ]); - expect(result).toEqual([ - { - actions: [], - rule_id: 'rule-1', - output_index: '.siem-signals', - risk_score: 50, - description: 'some description', - from: 'now-5m', - to: 'now', - index: ['index-1'], - name: 'some-name', - severity: 'low', - interval: '5m', - type: 'query', - enabled: true, - false_positives: [], - immutable: false, - query: '', - language: 'kuery', - max_signals: 100, - lists: [], - tags: [], - threat: [], - throttle: null, - references: [], - version: 1, - }, - { - actions: [], - rule_id: 'rule-2', - output_index: '.siem-signals', - risk_score: 50, - description: 'some description', - from: 'now-5m', - to: 'now', - index: ['index-1'], - name: 'some-name', - severity: 'low', - interval: '5m', - type: 'query', - enabled: true, - false_positives: [], - immutable: false, - query: '', - language: 'kuery', - max_signals: 100, - lists: [], - tags: [], - threat: [], - throttle: null, - references: [], - version: 1, - }, - ]); - }); - - test('handles non parsable JSON strings and inserts the error as part of the return array', async () => { - const sample1 = getOutputSample(); - const sample2 = getOutputSample(); - sample2.rule_id = 'rule-2'; - const ndJsonStream = new Readable({ - read() { - this.push(getSampleAsNdjson(sample1)); - this.push('{,,,,\n'); - this.push(getSampleAsNdjson(sample2)); - this.push(null); - }, - }); - const rulesObjectsStream = createRulesStreamFromNdJson(1000); - const result = await createPromiseFromStreams([ - ndJsonStream, - ...rulesObjectsStream, - ]); - const resultOrError = result as Error[]; - expect(resultOrError[0]).toEqual({ - actions: [], - rule_id: 'rule-1', - output_index: '.siem-signals', - risk_score: 50, - description: 'some description', - from: 'now-5m', - to: 'now', - index: ['index-1'], - name: 'some-name', - severity: 'low', - interval: '5m', - type: 'query', - enabled: true, - false_positives: [], - immutable: false, - query: '', - language: 'kuery', - max_signals: 100, - lists: [], - tags: [], - threat: [], - throttle: null, - references: [], - version: 1, - }); - expect(resultOrError[1].message).toEqual('Unexpected token , in JSON at position 1'); - expect(resultOrError[2]).toEqual({ - actions: [], - rule_id: 'rule-2', - output_index: '.siem-signals', - risk_score: 50, - description: 'some description', - from: 'now-5m', - to: 'now', - index: ['index-1'], - name: 'some-name', - severity: 'low', - interval: '5m', - type: 'query', - enabled: true, - false_positives: [], - immutable: false, - query: '', - language: 'kuery', - max_signals: 100, - lists: [], - tags: [], - threat: [], - throttle: null, - references: [], - version: 1, - }); - }); - - test('handles non-validated data', async () => { - const sample1 = getOutputSample(); - const sample2 = getOutputSample(); - sample2.rule_id = 'rule-2'; - const ndJsonStream = new Readable({ - read() { - this.push(getSampleAsNdjson(sample1)); - this.push(`{}\n`); - this.push(getSampleAsNdjson(sample2)); - this.push(null); - }, - }); - const rulesObjectsStream = createRulesStreamFromNdJson(1000); - const result = await createPromiseFromStreams([ - ndJsonStream, - ...rulesObjectsStream, - ]); - const resultOrError = result as BadRequestError[]; - expect(resultOrError[0]).toEqual({ - actions: [], - rule_id: 'rule-1', - output_index: '.siem-signals', - risk_score: 50, - description: 'some description', - from: 'now-5m', - to: 'now', - index: ['index-1'], - name: 'some-name', - severity: 'low', - interval: '5m', - type: 'query', - enabled: true, - false_positives: [], - immutable: false, - query: '', - language: 'kuery', - max_signals: 100, - lists: [], - tags: [], - threat: [], - throttle: null, - references: [], - version: 1, - }); - expect(resultOrError[1].message).toEqual( - 'child "description" fails because ["description" is required]' - ); - expect(resultOrError[2]).toEqual({ - actions: [], - rule_id: 'rule-2', - output_index: '.siem-signals', - risk_score: 50, - description: 'some description', - from: 'now-5m', - to: 'now', - index: ['index-1'], - name: 'some-name', - severity: 'low', - interval: '5m', - type: 'query', - enabled: true, - false_positives: [], - immutable: false, - query: '', - language: 'kuery', - max_signals: 100, - lists: [], - tags: [], - threat: [], - throttle: null, - references: [], - version: 1, - }); - }); - - test('non validated data is an instanceof BadRequestError', async () => { - const sample1 = getOutputSample(); - const sample2 = getOutputSample(); - sample2.rule_id = 'rule-2'; - const ndJsonStream = new Readable({ - read() { - this.push(getSampleAsNdjson(sample1)); - this.push(`{}\n`); - this.push(getSampleAsNdjson(sample2)); - this.push(null); - }, - }); - const rulesObjectsStream = createRulesStreamFromNdJson(1000); - const result = await createPromiseFromStreams([ - ndJsonStream, - ...rulesObjectsStream, - ]); - const resultOrError = result as BadRequestError[]; - expect(resultOrError[1] instanceof BadRequestError).toEqual(true); - }); - }); - - describe('transformDataToNdjson', () => { - test('if rules are empty it returns an empty string', () => { - const ruleNdjson = transformDataToNdjson([]); - expect(ruleNdjson).toEqual(''); - }); - - test('single rule will transform with new line ending character for ndjson', () => { - const rule = sampleRule(); - const ruleNdjson = transformDataToNdjson([rule]); - expect(ruleNdjson.endsWith('\n')).toBe(true); - }); - - test('multiple rules will transform with two new line ending characters for ndjson', () => { - const result1 = sampleRule(); - const result2 = sampleRule(); - result2.id = 'some other id'; - result2.rule_id = 'some other id'; - result2.name = 'Some other rule'; - - const ruleNdjson = transformDataToNdjson([result1, result2]); - // this is how we count characters in JavaScript :-) - const count = ruleNdjson.split('\n').length - 1; - expect(count).toBe(2); - }); - - test('you can parse two rules back out without errors', () => { - const result1 = sampleRule(); - const result2 = sampleRule(); - result2.id = 'some other id'; - result2.rule_id = 'some other id'; - result2.name = 'Some other rule'; - - const ruleNdjson = transformDataToNdjson([result1, result2]); - const ruleStrings = ruleNdjson.split('\n'); - const reParsed1 = JSON.parse(ruleStrings[0]); - const reParsed2 = JSON.parse(ruleStrings[1]); - expect(reParsed1).toEqual(result1); - expect(reParsed2).toEqual(result2); - }); - }); -}); diff --git a/x-pack/legacy/plugins/siem/server/utils/read_stream/create_stream_from_ndjson.test.ts b/x-pack/legacy/plugins/siem/server/utils/read_stream/create_stream_from_ndjson.test.ts new file mode 100644 index 0000000000000..1d92392e4df24 --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/utils/read_stream/create_stream_from_ndjson.test.ts @@ -0,0 +1,74 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { Readable } from 'stream'; +import { createRulesStreamFromNdJson, transformDataToNdjson } from './create_stream_from_ndjson'; +import { createPromiseFromStreams } from '../../../../../../../src/legacy/utils/streams'; +import { ImportRuleAlertRest } from '../../lib/detection_engine/types'; +import { BadRequestError } from '../../lib/detection_engine/errors/bad_request_error'; +import { sampleRule } from '../../lib/detection_engine/signals/__mocks__/es_results'; + +type PromiseFromStreams = ImportRuleAlertRest | Error; + +export const getOutputSample = (): Partial => ({ + rule_id: 'rule-1', + output_index: '.siem-signals', + risk_score: 50, + description: 'some description', + from: 'now-5m', + to: 'now', + index: ['index-1'], + name: 'some-name', + severity: 'low', + interval: '5m', + type: 'query', +}); + +export const getSampleAsNdjson = (sample: Partial): string => { + return `${JSON.stringify(sample)}\n`; +}; + +describe('create_rules_stream_from_ndjson', () => { + describe('transformDataToNdjson', () => { + test('if rules are empty it returns an empty string', () => { + const ruleNdjson = transformDataToNdjson([]); + expect(ruleNdjson).toEqual(''); + }); + + test('single rule will transform with new line ending character for ndjson', () => { + const rule = sampleRule(); + const ruleNdjson = transformDataToNdjson([rule]); + expect(ruleNdjson.endsWith('\n')).toBe(true); + }); + + test('multiple rules will transform with two new line ending characters for ndjson', () => { + const result1 = sampleRule(); + const result2 = sampleRule(); + result2.id = 'some other id'; + result2.rule_id = 'some other id'; + result2.name = 'Some other rule'; + + const ruleNdjson = transformDataToNdjson([result1, result2]); + // this is how we count characters in JavaScript :-) + const count = ruleNdjson.split('\n').length - 1; + expect(count).toBe(2); + }); + + test('you can parse two rules back out without errors', () => { + const result1 = sampleRule(); + const result2 = sampleRule(); + result2.id = 'some other id'; + result2.rule_id = 'some other id'; + result2.name = 'Some other rule'; + + const ruleNdjson = transformDataToNdjson([result1, result2]); + const ruleStrings = ruleNdjson.split('\n'); + const reParsed1 = JSON.parse(ruleStrings[0]); + const reParsed2 = JSON.parse(ruleStrings[1]); + expect(reParsed1).toEqual(result1); + expect(reParsed2).toEqual(result2); + }); + }); +}); diff --git a/x-pack/legacy/plugins/siem/server/utils/read_stream/create_rules_stream_from_ndjson.ts b/x-pack/legacy/plugins/siem/server/utils/read_stream/create_stream_from_ndjson.ts similarity index 86% rename from x-pack/legacy/plugins/siem/server/utils/read_stream/create_rules_stream_from_ndjson.ts rename to x-pack/legacy/plugins/siem/server/utils/read_stream/create_stream_from_ndjson.ts index 5df238073a66d..3b2ebd1551383 100644 --- a/x-pack/legacy/plugins/siem/server/utils/read_stream/create_rules_stream_from_ndjson.ts +++ b/x-pack/legacy/plugins/siem/server/utils/read_stream/create_stream_from_ndjson.ts @@ -68,21 +68,6 @@ export const createLimitStream = (limit: number): Transform => { }); }; -/** - * Inspiration and the pattern of code followed is from: - * saved_objects/lib/create_saved_objects_stream_from_ndjson.ts - */ -export const createRulesStreamFromNdJson = (ruleLimit: number) => { - return [ - createSplitStream('\n'), - parseNdjsonStrings(), - filterExportedCounts(), - validateRules(), - createLimitStream(ruleLimit), - createConcatStream([]), - ]; -}; - export const transformDataToNdjson = (data: unknown[]): string => { if (data.length !== 0) { const dataString = data.map(rule => JSON.stringify(rule)).join('\n'); From 2689f3baba0c7f2fa7ffa2345ed7b772021bef66 Mon Sep 17 00:00:00 2001 From: Angela Chuang Date: Wed, 15 Apr 2020 22:30:47 +0100 Subject: [PATCH 21/21] fix types --- .../utils/read_stream/create_stream_from_ndjson.test.ts | 7 +------ .../server/utils/read_stream/create_stream_from_ndjson.ts | 7 +------ 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/x-pack/legacy/plugins/siem/server/utils/read_stream/create_stream_from_ndjson.test.ts b/x-pack/legacy/plugins/siem/server/utils/read_stream/create_stream_from_ndjson.test.ts index 1d92392e4df24..2b5b34edca140 100644 --- a/x-pack/legacy/plugins/siem/server/utils/read_stream/create_stream_from_ndjson.test.ts +++ b/x-pack/legacy/plugins/siem/server/utils/read_stream/create_stream_from_ndjson.test.ts @@ -3,15 +3,10 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -import { Readable } from 'stream'; -import { createRulesStreamFromNdJson, transformDataToNdjson } from './create_stream_from_ndjson'; -import { createPromiseFromStreams } from '../../../../../../../src/legacy/utils/streams'; +import { transformDataToNdjson } from './create_stream_from_ndjson'; import { ImportRuleAlertRest } from '../../lib/detection_engine/types'; -import { BadRequestError } from '../../lib/detection_engine/errors/bad_request_error'; import { sampleRule } from '../../lib/detection_engine/signals/__mocks__/es_results'; -type PromiseFromStreams = ImportRuleAlertRest | Error; - export const getOutputSample = (): Partial => ({ rule_id: 'rule-1', output_index: '.siem-signals', diff --git a/x-pack/legacy/plugins/siem/server/utils/read_stream/create_stream_from_ndjson.ts b/x-pack/legacy/plugins/siem/server/utils/read_stream/create_stream_from_ndjson.ts index 3b2ebd1551383..0b7966926b5dd 100644 --- a/x-pack/legacy/plugins/siem/server/utils/read_stream/create_stream_from_ndjson.ts +++ b/x-pack/legacy/plugins/siem/server/utils/read_stream/create_stream_from_ndjson.ts @@ -6,12 +6,7 @@ import { Transform } from 'stream'; import { has, isString } from 'lodash/fp'; import { ImportRuleAlertRest } from '../../lib/detection_engine/types'; -import { - createSplitStream, - createMapStream, - createFilterStream, - createConcatStream, -} from '../../../../../../../src/legacy/utils/streams'; +import { createMapStream, createFilterStream } from '../../../../../../../src/legacy/utils/streams'; import { importRulesSchema } from '../../lib/detection_engine/routes/schemas/import_rules_schema'; import { BadRequestError } from '../../lib/detection_engine/errors/bad_request_error';