diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/import_rule_modal/__snapshots__/index.test.tsx.snap b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/import_rule_modal/__snapshots__/index.test.tsx.snap index 64b88912e53a3..6b5ea2c5390f1 100644 --- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/import_rule_modal/__snapshots__/index.test.tsx.snap +++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/import_rule_modal/__snapshots__/index.test.tsx.snap @@ -47,7 +47,7 @@ exports[`ImportRuleModal renders correctly against snapshot 1`] = ` Cancel diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/import_rule_modal/index.tsx b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/import_rule_modal/index.tsx index 91b2ee283609f..9a68797aea79b 100644 --- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/import_rule_modal/index.tsx +++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/import_rule_modal/index.tsx @@ -90,6 +90,11 @@ export const ImportRuleModalComponent = ({ } }, [selectedFiles, overwrite]); + const handleCloseModal = useCallback(() => { + setSelectedFiles(null); + closeModal(); + }, [closeModal]); + return ( <> {showModal && ( @@ -125,7 +130,7 @@ export const ImportRuleModalComponent = ({ - {i18n.CANCEL_BUTTON} + {i18n.CANCEL_BUTTON} ('savedObjects.maxImportExportSize'); - if (request.payload?.objects != null && request.payload.objects.length > exportSizeLimit) { - return Boom.badRequest(`Can't export more than ${exportSizeLimit} rules`); - } else { - const nonPackagedRulesCount = await getNonPackagedRulesCount({ alertsClient }); - if (nonPackagedRulesCount > exportSizeLimit) { + try { + const exportSizeLimit = server.config().get('savedObjects.maxImportExportSize'); + if (request.payload?.objects != null && request.payload.objects.length > exportSizeLimit) { return Boom.badRequest(`Can't export more than ${exportSizeLimit} rules`); + } else { + const nonPackagedRulesCount = await getNonPackagedRulesCount({ alertsClient }); + if (nonPackagedRulesCount > exportSizeLimit) { + return Boom.badRequest(`Can't export more than ${exportSizeLimit} rules`); + } } - } - const exported = - request.payload?.objects != null - ? await getExportByObjectIds(alertsClient, request.payload.objects) - : await getExportAll(alertsClient); + const exported = + request.payload?.objects != null + ? await getExportByObjectIds(alertsClient, request.payload.objects) + : await getExportAll(alertsClient); - const response = request.query.exclude_export_details - ? headers.response(exported.rulesNdjson) - : headers.response(`${exported.rulesNdjson}${exported.exportDetails}`); + const response = request.query.exclude_export_details + ? headers.response(exported.rulesNdjson) + : headers.response(`${exported.rulesNdjson}${exported.exportDetails}`); - return response - .header('Content-Disposition', `attachment; filename="${request.query.file_name}"`) - .header('Content-Type', 'application/ndjson'); + return response + .header('Content-Disposition', `attachment; filename="${request.query.file_name}"`) + .header('Content-Type', 'application/ndjson'); + } catch { + return Boom.badRequest(`Sorry, something went wrong to export rules`); + } }, }; }; diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/import_rules_route.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/import_rules_route.ts index 71fdef3623bc7..0d57f5739fc15 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/import_rules_route.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/import_rules_route.ts @@ -6,8 +6,9 @@ import Boom from 'boom'; import Hapi from 'hapi'; +import { chunk, isEmpty, isFunction } from 'lodash/fp'; import { extname } from 'path'; -import { isFunction } from 'lodash/fp'; +import uuid from 'uuid'; import { createPromiseFromStreams } from '../../../../../../../../../src/legacy/utils/streams'; import { DETECTION_ENGINE_RULES_URL } from '../../../../../common/constants'; import { createRules } from '../../rules/create_rules'; @@ -18,17 +19,25 @@ import { getIndexExists } from '../../index/get_index_exists'; import { callWithRequestFactory, getIndex, - createImportErrorObject, - transformImportError, - ImportSuccessError, + createBulkErrorObject, + ImportRuleResponse, } from '../utils'; import { createRulesStreamFromNdJson } from '../../rules/create_rules_stream_from_ndjson'; import { ImportRuleAlertRest } from '../../types'; -import { transformOrImportError } from './utils'; import { updateRules } from '../../rules/update_rules'; import { importRulesQuerySchema, importRulesPayloadSchema } from '../schemas/import_rules_schema'; import { KibanaRequest } from '../../../../../../../../../src/core/server'; +type PromiseFromStreams = ImportRuleAlertRest | Error; + +/* + * We were getting some error like that possible EventEmitter memory leak detected + * So we decide to batch the update by 10 to avoid any complication in the node side + * https://nodejs.org/docs/latest/api/events.html#events_emitter_setmaxlisteners_n + * + */ +const CHUNK_PARSED_OBJECT_SIZE = 10; + export const createImportRulesRoute = (server: ServerFacade): Hapi.ServerRoute => { return { method: 'POST', @@ -67,145 +76,189 @@ export const createImportRulesRoute = (server: ServerFacade): Hapi.ServerRoute = const objectLimit = server.config().get('savedObjects.maxImportExportSize'); const readStream = createRulesStreamFromNdJson(request.payload.file, objectLimit); - const parsedObjects = await createPromiseFromStreams<[ImportRuleAlertRest | Error]>([ - readStream, - ]); - - const reduced = await parsedObjects.reduce>( - async (accum, parsedRule) => { - const existingImportSuccessError = await accum; - if (parsedRule instanceof Error) { - // If the JSON object had a validation or parse error then we return - // early with the error and an (unknown) for the ruleId - return createImportErrorObject({ - ruleId: '(unknown)', // TODO: Better handling where we know which ruleId is having issues with imports - statusCode: 400, - message: parsedRule.message, - existingImportSuccessError, - }); - } + const parsedObjects = await createPromiseFromStreams([readStream]); - const { - description, - enabled, - false_positives: falsePositives, - from, - immutable, - query, - language, - output_index: outputIndex, - saved_id: savedId, - meta, - filters, - rule_id: ruleId, - index, - interval, - max_signals: maxSignals, - risk_score: riskScore, - name, - severity, - tags, - threat, - to, - type, - references, - timeline_id: timelineId, - timeline_title: timelineTitle, - version, - } = parsedRule; - try { - const finalIndex = outputIndex != null ? outputIndex : getIndex(request, server); - const callWithRequest = callWithRequestFactory(request, server); - const indexExists = await getIndexExists(callWithRequest, finalIndex); - if (!indexExists) { - return createImportErrorObject({ - ruleId, - statusCode: 409, - message: `To create a rule, the index must exist first. Index ${finalIndex} does not exist`, - existingImportSuccessError, - }); - } - const rule = await readRules({ alertsClient, ruleId }); - if (rule == null) { - const createdRule = await createRules({ - alertsClient, - actionsClient, - description, - enabled, - falsePositives, - from, - immutable, - query, - language, - outputIndex: finalIndex, - savedId, - timelineId, - timelineTitle, - meta, - filters, - ruleId, - index, - interval, - maxSignals, - riskScore, - name, - severity, - tags, - to, - type, - threat, - references, - version, - }); - return transformOrImportError(ruleId, createdRule, existingImportSuccessError); - } else if (rule != null && request.query.overwrite) { - const updatedRule = await updateRules({ - alertsClient, - actionsClient, - savedObjectsClient, - description, - enabled, - falsePositives, - from, - immutable, - query, - language, - outputIndex, - savedId, - timelineId, - timelineTitle, - meta, - filters, - id: undefined, - ruleId, - index, - interval, - maxSignals, - riskScore, - name, - severity, - tags, - to, - type, - threat, - references, - version, - }); - return transformOrImportError(ruleId, updatedRule, existingImportSuccessError); - } else { - return existingImportSuccessError; - } - } catch (err) { - return transformImportError(ruleId, err, existingImportSuccessError); - } - }, - Promise.resolve({ - success: true, - success_count: 0, - errors: [], - }) + const uniqueParsedObjects = Array.from( + parsedObjects + .reduce( + (acc, parsedRule) => { + if (parsedRule instanceof Error) { + acc.set(uuid.v4(), parsedRule); + } else { + const { rule_id: ruleId } = parsedRule; + if (ruleId != null) { + acc.set(ruleId, parsedRule); + } else { + acc.set(uuid.v4(), parsedRule); + } + } + return acc; + }, // using map (preserves ordering) + new Map() + ) + .values() ); - return reduced; + + const chunkParseObjects = chunk(CHUNK_PARSED_OBJECT_SIZE, uniqueParsedObjects); + let importRuleResponse: ImportRuleResponse[] = []; + + while (chunkParseObjects.length) { + const batchParseObjects = chunkParseObjects.shift() ?? []; + const newImportRuleResponse = await Promise.all( + batchParseObjects.reduce>>((accum, parsedRule) => { + const importsWorkerPromise = new Promise( + async (resolve, reject) => { + if (parsedRule instanceof Error) { + // If the JSON object had a validation or parse error then we return + // early with the error and an (unknown) for the ruleId + resolve( + createBulkErrorObject({ + ruleId: '(unknown)', + statusCode: 400, + message: parsedRule.message, + }) + ); + return null; + } + const { + description, + false_positives: falsePositives, + from, + immutable, + query, + language, + output_index: outputIndex, + saved_id: savedId, + meta, + filters, + rule_id: ruleId, + index, + interval, + max_signals: maxSignals, + risk_score: riskScore, + name, + severity, + tags, + threat, + to, + type, + references, + timeline_id: timelineId, + timeline_title: timelineTitle, + version, + } = parsedRule; + try { + const finalIndex = getIndex(request, server); + const callWithRequest = callWithRequestFactory(request, server); + const indexExists = await getIndexExists(callWithRequest, finalIndex); + if (!indexExists) { + resolve( + createBulkErrorObject({ + ruleId, + statusCode: 409, + message: `To create a rule, the index must exist first. Index ${finalIndex} does not exist`, + }) + ); + } + const rule = await readRules({ alertsClient, ruleId }); + if (rule == null) { + await createRules({ + alertsClient, + actionsClient, + description, + enabled: false, + falsePositives, + from, + immutable, + query, + language, + outputIndex: finalIndex, + savedId, + timelineId, + timelineTitle, + meta, + filters, + ruleId, + index, + interval, + maxSignals, + riskScore, + name, + severity, + tags, + to, + type, + threat, + references, + version, + }); + resolve({ rule_id: ruleId, status_code: 200 }); + } else if (rule != null && request.query.overwrite) { + await updateRules({ + alertsClient, + actionsClient, + savedObjectsClient, + description, + enabled: false, + falsePositives, + from, + immutable, + query, + language, + outputIndex, + savedId, + timelineId, + timelineTitle, + meta, + filters, + id: undefined, + ruleId, + index, + interval, + maxSignals, + riskScore, + name, + severity, + tags, + to, + type, + threat, + references, + version, + }); + resolve({ rule_id: ruleId, status_code: 200 }); + } else if (rule != null) { + resolve( + createBulkErrorObject({ + ruleId, + statusCode: 409, + message: `This Rule "${rule.name}" already exists`, + }) + ); + } + } catch (err) { + resolve( + createBulkErrorObject({ + ruleId, + statusCode: 400, + message: err.message, + }) + ); + } + } + ); + return [...accum, importsWorkerPromise]; + }, []) + ); + importRuleResponse = [...importRuleResponse, ...newImportRuleResponse]; + } + + const errorsResp = importRuleResponse.filter(resp => !isEmpty(resp.error)); + return { + success: errorsResp.length === 0, + success_count: importRuleResponse.filter(resp => resp.status_code === 200).length, + errors: errorsResp, + }; }, }; }; diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/utils.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/utils.ts index 19cd972b60e1a..416c76b5d4eb5 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/utils.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/utils.ts @@ -52,6 +52,16 @@ export const createBulkErrorObject = ({ }; }; +export interface ImportRuleResponse { + rule_id: string; + status_code?: number; + message?: string; + error?: { + status_code: number; + message: string; + }; +} + export interface ImportSuccessError { success: boolean; success_count: number; diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/get_export_by_object_ids.test.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/get_export_by_object_ids.test.ts index 05e455efb3f22..236d04acc782b 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/get_export_by_object_ids.test.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/get_export_by_object_ids.test.ts @@ -66,6 +66,7 @@ describe('get_export_by_object_ids', () => { const objects = [{ rule_id: 'rule-1' }]; const exports = await getRulesFromObjects(unsafeCast, objects); const expected: RulesErrors = { + exportedCount: 1, missingRules: [], rules: [ { @@ -141,6 +142,7 @@ describe('get_export_by_object_ids', () => { const objects = [{ rule_id: 'rule-1' }]; const exports = await getRulesFromObjects(unsafeCast, objects); const expected: RulesErrors = { + exportedCount: 0, missingRules: [{ rule_id: 'rule-1' }], rules: [], }; @@ -164,6 +166,7 @@ describe('get_export_by_object_ids', () => { const objects = [{ rule_id: 'rule-1' }]; const exports = await getRulesFromObjects(unsafeCast, objects); const expected: RulesErrors = { + exportedCount: 0, missingRules: [{ rule_id: 'rule-1' }], rules: [], }; diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/get_export_by_object_ids.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/get_export_by_object_ids.ts index a5cf1bbfb7858..7e0d61d040617 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/get_export_by_object_ids.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/get_export_by_object_ids.ts @@ -11,7 +11,20 @@ import { readRules } from './read_rules'; import { transformRulesToNdjson, transformAlertToRule } from '../routes/rules/utils'; import { OutputRuleAlertRest } from '../types'; +interface ExportSuccesRule { + statusCode: 200; + rule: Partial; +} + +interface ExportFailedRule { + statusCode: 404; + missingRuleId: { rule_id: string }; +} + +type ExportRules = ExportSuccesRule | ExportFailedRule; + export interface RulesErrors { + exportedCount: number; missingRules: Array<{ rule_id: string }>; rules: Array>; } @@ -33,28 +46,44 @@ export const getRulesFromObjects = async ( alertsClient: AlertsClient, objects: Array<{ rule_id: string }> ): Promise => { - const alertsAndErrors = await objects.reduce>( - async (accumPromise, object) => { - const accum = await accumPromise; - const rule = await readRules({ alertsClient, ruleId: object.rule_id }); - if (rule != null && isAlertType(rule) && rule.params.immutable !== true) { - const transformedRule = transformAlertToRule(rule); - return { - missingRules: accum.missingRules, - rules: [...accum.rules, transformedRule], - }; - } else { - return { - missingRules: [...accum.missingRules, { rule_id: object.rule_id }], - rules: accum.rules, - }; - } - }, - Promise.resolve({ - exportedCount: 0, - missingRules: [], - rules: [], - }) + const alertsAndErrors = await Promise.all( + objects.reduce>>((accumPromise, object) => { + const exportWorkerPromise = new Promise(async resolve => { + try { + const rule = await readRules({ alertsClient, ruleId: object.rule_id }); + if (rule != null && isAlertType(rule) && rule.params.immutable !== true) { + const transformedRule = transformAlertToRule(rule); + resolve({ + statusCode: 200, + rule: transformedRule, + }); + } else { + resolve({ + statusCode: 404, + missingRuleId: { rule_id: object.rule_id }, + }); + } + } catch { + resolve({ + statusCode: 404, + missingRuleId: { rule_id: object.rule_id }, + }); + } + }); + return [...accumPromise, exportWorkerPromise]; + }, []) ); - return alertsAndErrors; + + const missingRules = alertsAndErrors.filter( + resp => resp.statusCode === 404 + ) as ExportFailedRule[]; + const exportedRules = alertsAndErrors.filter( + resp => resp.statusCode === 200 + ) as ExportSuccesRule[]; + + return { + exportedCount: exportedRules.length, + missingRules: missingRules.map(mr => mr.missingRuleId), + rules: exportedRules.map(er => er.rule), + }; };