Skip to content

Commit

Permalink
Fixing exceptions export format (#114920) (#114949)
Browse files Browse the repository at this point in the history
### Summary

Fixing exceptions export format and adding integration tests for it.

Co-authored-by: Yara Tercero <[email protected]>
  • Loading branch information
kibanamachine and yctercero authored Oct 14, 2021
1 parent eee9b07 commit 184592e
Show file tree
Hide file tree
Showing 17 changed files with 292 additions and 113 deletions.
2 changes: 1 addition & 1 deletion packages/kbn-securitysolution-list-api/src/api/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -558,7 +558,7 @@ export const exportExceptionList = async ({
signal,
}: ExportExceptionListProps): Promise<Blob> =>
http.fetch<Blob>(`${EXCEPTION_LIST_URL}/_export`, {
method: 'GET',
method: 'POST',
query: { id, list_id: listId, namespace_type: namespaceType },
signal,
});
1 change: 1 addition & 0 deletions packages/kbn-securitysolution-utils/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,3 +7,4 @@
*/

export * from './add_remove_id_to_item';
export * from './transform_data_to_ndjson';
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/

import { transformDataToNdjson } from './';

export const ANCHOR_DATE = '2020-02-20T03:57:54.037Z';

const getRulesSchemaMock = (anchorDate: string = ANCHOR_DATE) => ({
author: [],
id: '7a7065d7-6e8b-4aae-8d20-c93613dec9f9',
created_at: new Date(anchorDate).toISOString(),
updated_at: new Date(anchorDate).toISOString(),
created_by: 'elastic',
description: 'some description',
enabled: true,
false_positives: ['false positive 1', 'false positive 2'],
from: 'now-6m',
immutable: false,
name: 'Query with a rule id',
query: 'user.name: root or user.name: admin',
references: ['test 1', 'test 2'],
severity: 'high',
severity_mapping: [],
updated_by: 'elastic_kibana',
tags: ['some fake tag 1', 'some fake tag 2'],
to: 'now',
type: 'query',
threat: [],
version: 1,
status: 'succeeded',
status_date: '2020-02-22T16:47:50.047Z',
last_success_at: '2020-02-22T16:47:50.047Z',
last_success_message: 'succeeded',
output_index: '.siem-signals-default',
max_signals: 100,
risk_score: 55,
risk_score_mapping: [],
language: 'kuery',
rule_id: 'query-rule-id',
interval: '5m',
exceptions_list: [],
});

describe('transformDataToNdjson', () => {
test('if rules are empty it returns an empty string', () => {
const ruleNdjson = transformDataToNdjson([]);
expect(ruleNdjson).toEqual('');
});

test('single rule will transform with new line ending character for ndjson', () => {
const rule = getRulesSchemaMock();
const ruleNdjson = transformDataToNdjson([rule]);
expect(ruleNdjson.endsWith('\n')).toBe(true);
});

test('multiple rules will transform with two new line ending characters for ndjson', () => {
const result1 = getRulesSchemaMock();
const result2 = getRulesSchemaMock();
result2.id = 'some other id';
result2.rule_id = 'some other id';
result2.name = 'Some other rule';

const ruleNdjson = transformDataToNdjson([result1, result2]);
// this is how we count characters in JavaScript :-)
const count = ruleNdjson.split('\n').length - 1;
expect(count).toBe(2);
});

test('you can parse two rules back out without errors', () => {
const result1 = getRulesSchemaMock();
const result2 = getRulesSchemaMock();
result2.id = 'some other id';
result2.rule_id = 'some other id';
result2.name = 'Some other rule';

const ruleNdjson = transformDataToNdjson([result1, result2]);
const ruleStrings = ruleNdjson.split('\n');
const reParsed1 = JSON.parse(ruleStrings[0]);
const reParsed2 = JSON.parse(ruleStrings[1]);
expect(reParsed1).toEqual(result1);
expect(reParsed2).toEqual(result2);
});
});
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/

export const transformDataToNdjson = (data: unknown[]): string => {
if (data.length !== 0) {
const dataString = data.map((item) => JSON.stringify(item)).join('\n');
return `${dataString}\n`;
} else {
return '';
}
};
2 changes: 1 addition & 1 deletion x-pack/plugins/lists/public/exceptions/api.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -759,7 +759,7 @@ describe('Exceptions Lists API', () => {
});

expect(httpMock.fetch).toHaveBeenCalledWith('/api/exception_lists/_export', {
method: 'GET',
method: 'POST',
query: {
id: 'some-id',
list_id: 'list-id',
Expand Down
42 changes: 19 additions & 23 deletions x-pack/plugins/lists/server/routes/export_exception_list_route.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
*/

import { transformError } from '@kbn/securitysolution-es-utils';
import { transformDataToNdjson } from '@kbn/securitysolution-utils';
import { exportExceptionListQuerySchema } from '@kbn/securitysolution-io-ts-list-types';
import { EXCEPTION_LIST_URL } from '@kbn/securitysolution-list-constants';

Expand All @@ -14,7 +15,7 @@ import type { ListsPluginRouter } from '../types';
import { buildRouteValidation, buildSiemResponse, getExceptionListClient } from './utils';

export const exportExceptionListRoute = (router: ListsPluginRouter): void => {
router.get(
router.post(
{
options: {
tags: ['access:lists-read'],
Expand All @@ -26,6 +27,7 @@ export const exportExceptionListRoute = (router: ListsPluginRouter): void => {
},
async (context, request, response) => {
const siemResponse = buildSiemResponse(response);

try {
const { id, list_id: listId, namespace_type: namespaceType } = request.query;
const exceptionLists = getExceptionListClient(context);
Expand All @@ -37,11 +39,10 @@ export const exportExceptionListRoute = (router: ListsPluginRouter): void => {

if (exceptionList == null) {
return siemResponse.error({
body: `list_id: ${listId} does not exist`,
body: `exception list with list_id: ${listId} does not exist`,
statusCode: 400,
});
} else {
const { exportData: exportList } = getExport([exceptionList]);
const listItems = await exceptionLists.findExceptionListItem({
filter: undefined,
listId,
Expand All @@ -51,19 +52,15 @@ export const exportExceptionListRoute = (router: ListsPluginRouter): void => {
sortField: 'exception-list.created_at',
sortOrder: 'desc',
});
const exceptionItems = listItems?.data ?? [];

const { exportData: exportListItems, exportDetails } = getExport(listItems?.data ?? []);

const responseBody = [
exportList,
exportListItems,
{ exception_list_items_details: exportDetails },
];
const { exportData } = getExport([exceptionList, ...exceptionItems]);
const { exportDetails } = getExportDetails(exceptionItems);

// TODO: Allow the API to override the name of the file to export
const fileName = exceptionList.list_id;
return response.ok({
body: transformDataToNdjson(responseBody),
body: `${exportData}${exportDetails}`,
headers: {
'Content-Disposition': `attachment; filename="${fileName}"`,
'Content-Type': 'application/ndjson',
Expand All @@ -81,24 +78,23 @@ export const exportExceptionListRoute = (router: ListsPluginRouter): void => {
);
};

const transformDataToNdjson = (data: unknown[]): string => {
if (data.length !== 0) {
const dataString = data.map((dataItem) => JSON.stringify(dataItem)).join('\n');
return `${dataString}\n`;
} else {
return '';
}
};

export const getExport = (
data: unknown[]
): {
exportData: string;
exportDetails: string;
} => {
const ndjson = transformDataToNdjson(data);

return { exportData: ndjson };
};

export const getExportDetails = (
items: unknown[]
): {
exportDetails: string;
} => {
const exportDetails = JSON.stringify({
exported_count: data.length,
exported_list_items_count: items.length,
});
return { exportData: ndjson, exportDetails: `${exportDetails}\n` };
return { exportDetails: `${exportDetails}\n` };
};
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
{"_version":"WzQyNjA0LDFd","created_at":"2021-10-14T01:30:22.034Z","created_by":"elastic","description":"Test exception list description","id":"4c65a230-2c8e-11ec-be1c-2bbdec602f88","immutable":false,"list_id":"test_exception_list","name":"Test exception list","namespace_type":"single","os_types":[],"tags":[],"tie_breaker_id":"b04983b4-1617-441c-bb6c-c729281fa2e9","type":"detection","updated_at":"2021-10-14T01:30:22.036Z","updated_by":"elastic","version":1}
{"exported_list_items_count":0}
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ describe('Exceptions Table', () => {

cy.wait('@export').then(({ response }) =>
cy
.wrap(response?.body!)
.wrap(response?.body)
.should('eql', expectedExportedExceptionList(this.exceptionListResponse))
);
});
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,5 @@ export const expectedExportedExceptionList = (
exceptionListResponse: Cypress.Response<ExceptionListItemSchema>
): string => {
const jsonrule = exceptionListResponse.body;

return `"{\\"_version\\":\\"${jsonrule._version}\\",\\"created_at\\":\\"${jsonrule.created_at}\\",\\"created_by\\":\\"elastic\\",\\"description\\":\\"${jsonrule.description}\\",\\"id\\":\\"${jsonrule.id}\\",\\"immutable\\":false,\\"list_id\\":\\"test_exception_list\\",\\"name\\":\\"Test exception list\\",\\"namespace_type\\":\\"single\\",\\"os_types\\":[],\\"tags\\":[],\\"tie_breaker_id\\":\\"${jsonrule.tie_breaker_id}\\",\\"type\\":\\"detection\\",\\"updated_at\\":\\"${jsonrule.updated_at}\\",\\"updated_by\\":\\"elastic\\",\\"version\\":1}\\n"\n""\n{"exception_list_items_details":"{\\"exported_count\\":0}\\n"}\n`;
return `{"_version":"${jsonrule._version}","created_at":"${jsonrule.created_at}","created_by":"elastic","description":"${jsonrule.description}","id":"${jsonrule.id}","immutable":false,"list_id":"test_exception_list","name":"Test exception list","namespace_type":"single","os_types":[],"tags":[],"tie_breaker_id":"${jsonrule.tie_breaker_id}","type":"detection","updated_at":"${jsonrule.updated_at}","updated_by":"elastic","version":1}\n{"exported_list_items_count":0}\n`;
};
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,12 @@
* 2.0.
*/

import { transformDataToNdjson } from '@kbn/securitysolution-utils';

import { RulesClient } from '../../../../../alerting/server';
import { getNonPackagedRules } from './get_existing_prepackaged_rules';
import { getExportDetailsNdjson } from './get_export_details_ndjson';
import { transformAlertsToRules } from '../routes/rules/utils';
import { transformDataToNdjson } from '../../../utils/read_stream/create_stream_from_ndjson';

export const getExportAll = async (
rulesClient: RulesClient,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,13 @@
*/

import { chunk } from 'lodash';
import { transformDataToNdjson } from '@kbn/securitysolution-utils';

import { RulesSchema } from '../../../../common/detection_engine/schemas/response/rules_schema';
import { RulesClient } from '../../../../../alerting/server';
import { getExportDetailsNdjson } from './get_export_details_ndjson';
import { isAlertType } from '../rules/types';
import { transformAlertToRule } from '../routes/rules/utils';
import { transformDataToNdjson } from '../../../utils/read_stream/create_stream_from_ndjson';
import { INTERNAL_RULE_ID_KEY } from '../../../../common/constants';
import { findRules } from './find_rules';

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,11 @@
import { cloneDeep } from 'lodash';
import axios from 'axios';
import { URL } from 'url';
import { transformDataToNdjson } from '@kbn/securitysolution-utils';

import { Logger } from 'src/core/server';
import { TelemetryPluginStart, TelemetryPluginSetup } from 'src/plugins/telemetry/server';
import { UsageCounter } from 'src/plugins/usage_collection/server';
import { transformDataToNdjson } from '../../utils/read_stream/create_stream_from_ndjson';
import {
TaskManagerSetupContract,
TaskManagerStartContract,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
*/

import { omit } from 'lodash/fp';
import { transformDataToNdjson } from '@kbn/securitysolution-utils';

import {
ExportedTimelines,
Expand All @@ -15,8 +16,6 @@ import {
import { NoteSavedObject } from '../../../../../../common/types/timeline/note';
import { PinnedEventSavedObject } from '../../../../../../common/types/timeline/pinned_event';

import { transformDataToNdjson } from '../../../../../utils/read_stream/create_stream_from_ndjson';

import { FrameworkRequest } from '../../../../framework';
import * as noteLib from '../../../saved_object/notes';
import * as pinnedEventLib from '../../../saved_object/pinned_events';
Expand Down

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -48,12 +48,3 @@ export const createLimitStream = (limit: number): Transform => {
},
});
};

export const transformDataToNdjson = (data: unknown[]): string => {
if (data.length !== 0) {
const dataString = data.map((rule) => JSON.stringify(rule)).join('\n');
return `${dataString}\n`;
} else {
return '';
}
};
Loading

0 comments on commit 184592e

Please sign in to comment.