From de6d3c51c4cc87dbfa818d6bde9659dc6a94e409 Mon Sep 17 00:00:00 2001 From: Christos Nasikas Date: Mon, 8 Apr 2024 19:11:51 +0300 Subject: [PATCH] Attach ungrouped alerts --- .../cases/server/connectors/cases/README.md | 8 +- .../cases/cases_connector_executor.test.ts | 129 ++++++++++++++++-- .../cases/cases_connector_executor.ts | 25 +++- .../server/connectors/cases/index.mock.ts | 7 +- .../trial/connectors/cases/cases_connector.ts | 96 +++++++++++++ 5 files changed, 247 insertions(+), 18 deletions(-) diff --git a/x-pack/plugins/cases/server/connectors/cases/README.md b/x-pack/plugins/cases/server/connectors/cases/README.md index aeba38d563d9d..703ea76627b2e 100644 --- a/x-pack/plugins/cases/server/connectors/cases/README.md +++ b/x-pack/plugins/cases/server/connectors/cases/README.md @@ -29,7 +29,7 @@ The case action groups all alerts based on the grouping field defined by the use 2. Check if the case is older than the defined time window. If yes it will create a new case as described in Step 1. 3. Check if the case is closed. If it is, it will check if the case should be reopened. If yes the case action will reopen the case and attach the alerts to it. If not it will create a new case and attach the alerts to the new case. -If an alert does not belong to a group it will not be attached to any case. Also, if no grouping field is configured by the user, all alerts will be attached to the same case. +If an alert does not belong to a group it will be attached to a case representing the `unknown` value. Also, if no grouping field is configured by the user, all alerts will be attached to the same case. ```mermaid flowchart TB @@ -132,7 +132,7 @@ flowchart LR ## Case creation -For each rule and each group produced by the grouping step, a case will be created and the alerts of that group will be attached to the new case. In future executions of the rule, new alerts that belong to the same group will be attached to the same case. If an alert cannot be grouped, because the grouping field does not exist in its data, it will not be attached to any case. +For each rule and each group produced by the grouping step, a case will be created and the alerts of that group will be attached to the new case. In future executions of the rule, new alerts that belong to the same group will be attached to the same case. If an alert cannot be grouped, because the grouping field does not exist in its data, it will be attached to a case that represents the `unknown` value. To support this, the case action constructs a deterministic deduplication ID which will be set as the case ID. The ID can be constructed on each execution of the case action without the need to persist it and can correctly map alerts of the same group to the case that represents that group. A deduplication ID has two main advantages: @@ -178,7 +178,7 @@ flowchart LR alert23[Alert 3] end - subgraph no_grouping ["No value"] + subgraph no_grouping ["IP: unknown"] alert34[Alert 4] end @@ -188,6 +188,7 @@ flowchart LR top --> caseOne bottom --> caseTwo + no_grouping --> caseThree alert1[Alert 1] alert2[Alert 2] @@ -196,6 +197,7 @@ flowchart LR groupingBy[Grouping by IP] caseOne[Case 1] caseTwo[Case 2] + caseThree[Case 3] ``` ## Time window diff --git a/x-pack/plugins/cases/server/connectors/cases/cases_connector_executor.test.ts b/x-pack/plugins/cases/server/connectors/cases/cases_connector_executor.test.ts index 33e34da6d891b..e09cf40487dc6 100644 --- a/x-pack/plugins/cases/server/connectors/cases/cases_connector_executor.test.ts +++ b/x-pack/plugins/cases/server/connectors/cases/cases_connector_executor.test.ts @@ -33,6 +33,7 @@ import { reopenClosedCases, updatedCounterOracleRecord, alertsNested, + alertsWithNoGrouping, } from './index.mock'; import { expectCasesToHaveTheCorrectAlertsAttachedWithGrouping, @@ -1409,22 +1410,128 @@ describe('CasesConnectorExecutor', () => { }); }); - describe('Skipping execution', () => { - it('skips execution if alerts cannot be grouped', async () => { + describe('Non grouped alerts', () => { + it('attaches the non grouped alerts to a case correctly when no alerts have the fields set in groupingBy', async () => { await connectorExecutor.execute({ ...params, groupingBy: ['does.not.exists'], }); - expect(mockGetRecordId).not.toHaveBeenCalled(); - expect(mockBulkGetRecords).not.toHaveBeenCalled(); - expect(mockBulkCreateRecords).not.toHaveBeenCalled(); - expect(mockBulkUpdateRecord).not.toHaveBeenCalled(); - expect(mockGetCaseId).not.toHaveBeenCalled(); - expect(casesClientMock.cases.bulkGet).not.toHaveBeenCalled(); - expect(casesClientMock.cases.bulkCreate).not.toHaveBeenCalled(); - expect(casesClientMock.cases.bulkUpdate).not.toHaveBeenCalled(); - expect(casesClientMock.configure.get).not.toHaveBeenCalled(); + expect(mockGetRecordId).toHaveBeenCalledWith({ + ruleId: rule.id, + grouping: { 'does.not.exists': 'unknown' }, + owner, + spaceId: 'default', + }); + + expect(mockBulkGetRecords).toHaveBeenCalledWith(['so-oracle-record-0']); + + expect(mockGetCaseId).toHaveBeenCalledWith({ + ruleId: rule.id, + grouping: { 'does.not.exists': 'unknown' }, + owner, + spaceId: 'default', + counter: 1, + }); + + expect(casesClientMock.cases.bulkGet).toHaveBeenCalledWith({ + ids: ['mock-id-1'], + }); + + expect(casesClientMock.attachments.bulkCreate).toHaveBeenCalledTimes(1); + + expect(casesClientMock.attachments.bulkCreate).nthCalledWith(1, { + caseId: 'mock-id-1', + attachments: [ + { + alertId: alerts.map((alert) => alert._id), + index: alerts.map((alert) => alert._index), + owner: 'securitySolution', + rule: { + id: 'rule-test-id', + name: 'Test rule', + }, + type: 'alert', + }, + ], + }); + }); + + it('attaches the non grouped alerts to a case correctly when some alerts do not have the fields set in groupingBy', async () => { + mockBulkGetRecords.mockResolvedValue([ + ...oracleRecords, + { + id: 'so-oracle-record-3', + version: 'so-version-1', + counter: 1, + cases: [], + rules: [], + grouping: {}, + createdAt: '2023-10-12T10:23:42.769Z', + updatedAt: '2023-10-12T10:23:42.769Z', + }, + ]); + + casesClientMock.cases.bulkGet.mockResolvedValue({ + cases: [...cases, { ...cases[2], id: 'mock-id-4' }], + errors: [], + }); + + await connectorExecutor.execute({ + ...params, + alerts: alertsWithNoGrouping, + }); + + expect(mockGetRecordId).toHaveBeenCalledTimes(4); + expect(mockGetRecordId).nthCalledWith(4, { + ruleId: rule.id, + grouping: { + 'dest.ip': 'unknown', + 'host.name': 'unknown', + }, + owner, + spaceId: 'default', + }); + + expect(mockBulkGetRecords).toHaveBeenCalledWith([ + 'so-oracle-record-0', + 'so-oracle-record-1', + 'so-oracle-record-2', + 'so-oracle-record-3', + ]); + + expect(mockGetCaseId).toHaveBeenCalledTimes(4); + expect(mockGetCaseId).nthCalledWith(3, { + ruleId: rule.id, + grouping: { + 'dest.ip': 'unknown', + 'host.name': 'unknown', + }, + owner, + spaceId: 'default', + counter: 1, + }); + + expect(casesClientMock.cases.bulkGet).toHaveBeenCalledWith({ + ids: ['mock-id-1', 'mock-id-2', 'mock-id-3', 'mock-id-4'], + }); + + expect(casesClientMock.attachments.bulkCreate).toHaveBeenCalledTimes(4); + expect(casesClientMock.attachments.bulkCreate).nthCalledWith(3, { + caseId: 'mock-id-3', + attachments: [ + { + type: 'alert', + rule: { + id: 'rule-test-id', + name: 'Test rule', + }, + alertId: ['alert-id-4', 'alert-id-5'], + index: ['alert-index-4', 'alert-index-5'], + owner: 'securitySolution', + }, + ], + }); }); }); }); diff --git a/x-pack/plugins/cases/server/connectors/cases/cases_connector_executor.ts b/x-pack/plugins/cases/server/connectors/cases/cases_connector_executor.ts index 1c81032e534ea..a5f07a9b65fc7 100644 --- a/x-pack/plugins/cases/server/connectors/cases/cases_connector_executor.ts +++ b/x-pack/plugins/cases/server/connectors/cases/cases_connector_executor.ts @@ -180,16 +180,16 @@ export class CasesConnectorExecutor { * of the groupingBy fields defined by the users. All other * alerts will not be attached to any case. */ - const filteredAlerts = alerts.filter((alert) => + const [alertsWithAllGroupingFields, noGroupedAlerts] = partition(alerts, (alert) => uniqueGroupingByFields.every((groupingByField) => Boolean(get(alert, groupingByField, null))) ); this.logger.debug( - `[CasesConnector][CasesConnectorExecutor][groupAlerts] Total alerts to be grouped: ${filteredAlerts.length} out of ${alerts.length}`, + `[CasesConnector][CasesConnectorExecutor][groupAlerts] Total alerts to be grouped: ${alertsWithAllGroupingFields.length} out of ${alerts.length}`, this.getLogMetadata(params, { tags: ['case-connector:groupAlerts'] }) ); - for (const alert of filteredAlerts) { + for (const alert of alertsWithAllGroupingFields) { const alertWithOnlyTheGroupingFields = pick(alert, uniqueGroupingByFields); const groupingKey = stringify(alertWithOnlyTheGroupingFields); @@ -205,9 +205,28 @@ export class CasesConnectorExecutor { } } + if (noGroupedAlerts.length > 0) { + const noGroupedGrouping = this.generateNoGroupAlertGrouping(params.groupingBy); + + groupingMap.set(stringify(noGroupedGrouping), { + alerts: noGroupedAlerts, + grouping: noGroupedGrouping, + }); + } + return Array.from(groupingMap.values()); } + private generateNoGroupAlertGrouping = (groupingBy: string[]) => { + const noGroupedGrouping = groupingBy.reduce((acc, field) => { + acc[field] = 'unknown'; + + return acc; + }, {} as Record); + + return noGroupedGrouping; + }; + private applyCircuitBreakers( params: CasesConnectorRunParams, groupedAlerts: GroupedAlerts[] diff --git a/x-pack/plugins/cases/server/connectors/cases/index.mock.ts b/x-pack/plugins/cases/server/connectors/cases/index.mock.ts index b97def9c97be6..ef474d5d22aa2 100644 --- a/x-pack/plugins/cases/server/connectors/cases/index.mock.ts +++ b/x-pack/plugins/cases/server/connectors/cases/index.mock.ts @@ -44,7 +44,6 @@ export const alerts = [ }, { _id: 'alert-id-2', _index: 'alert-index-2', 'host.name': 'A', 'dest.ip': '0.0.0.1' }, { _id: 'alert-id-3', _index: 'alert-index-3', 'host.name': 'B', 'dest.ip': '0.0.0.3' }, - { _id: 'alert-id-4', _index: 'alert-index-4', 'host.name': 'A', 'source.ip': '0.0.0.5' }, ]; export const alertsNested = [ @@ -82,6 +81,12 @@ export const alertsNested = [ }, ]; +export const alertsWithNoGrouping = [ + ...alerts, + { _id: 'alert-id-4', _index: 'alert-index-4', 'host.name': 'A', 'source.ip': '0.0.0.5' }, + { _id: 'alert-id-5', _index: 'alert-index-5' }, +]; + export const groupingBy = ['host.name', 'dest.ip']; export const rule = { id: 'rule-test-id', diff --git a/x-pack/test/cases_api_integration/security_and_spaces/tests/trial/connectors/cases/cases_connector.ts b/x-pack/test/cases_api_integration/security_and_spaces/tests/trial/connectors/cases/cases_connector.ts index 6e28ef558cba7..59ec2e7fb981c 100644 --- a/x-pack/test/cases_api_integration/security_and_spaces/tests/trial/connectors/cases/cases_connector.ts +++ b/x-pack/test/cases_api_integration/security_and_spaces/tests/trial/connectors/cases/cases_connector.ts @@ -954,6 +954,102 @@ export default ({ getService }: FtrProviderContext): void => { }); }); }); + + describe('Non grouped alerts', () => { + it('should attach non grouped alerts correctly', async () => { + await executeConnectorAndVerifyCorrectness({ + supertest, + connectorId, + req: getRequest({ + alerts: [ + { _id: 'alert-id-0', _index: 'alert-index-0', 'host.name': 'A' }, + { _id: 'alert-id-1', _index: 'alert-index-1', 'dest.ip': '0.0.0.1' }, + ], + groupingBy: ['host.name'], + }), + }); + + const cases = await findCases({ supertest }); + + expect(cases.total).to.be(2); + + const firstOracleId = generateOracleId({ + ruleId: req.params.subActionParams.rule.id, + grouping: { 'host.name': 'A' }, + }); + + const secondOracleId = generateOracleId({ + ruleId: req.params.subActionParams.rule.id, + grouping: { 'host.name': 'unknown' }, + }); + + const firstOracleRecord = await getOracleRecord({ + kibanaServer, + oracleId: firstOracleId, + }); + + const secondOracleRecord = await getOracleRecord({ + kibanaServer, + oracleId: secondOracleId, + }); + + expect(firstOracleRecord.grouping).to.eql({ 'host.name': 'A' }); + expect(secondOracleRecord.grouping).to.eql({ 'host.name': 'unknown' }); + + const firstCaseId = generateCaseId({ + ruleId: req.params.subActionParams.rule.id, + grouping: { 'host.name': 'A' }, + }); + + const secondCaseId = generateCaseId({ + ruleId: req.params.subActionParams.rule.id, + grouping: { 'host.name': 'unknown' }, + }); + + const firstCase = removeServerGeneratedData( + cases.cases.find((theCase) => theCase.id === firstCaseId)! + ); + + const secondCase = removeServerGeneratedData( + cases.cases.find((theCase) => theCase.id === secondCaseId)! + ); + + expect(firstCase.description).to.be( + 'This case is auto-created by [Test rule](https://example.com/rules/rule-test-id). \n\n Grouping: `host.name` equals `A`' + ); + expect(secondCase.description).to.be( + 'This case is auto-created by [Test rule](https://example.com/rules/rule-test-id). \n\n Grouping: `host.name` equals `unknown`' + ); + + const firstCaseAttachments = await getAllComments({ + supertest, + caseId: firstCase.id, + }); + + const secondCaseAttachments = await getAllComments({ + supertest, + caseId: secondCase.id, + }); + + verifyAlertsAttachedToCase({ + caseAttachments: firstCaseAttachments, + expectedAlertIdsToBeAttachedToCase: new Set(['alert-id-0']), + rule: { + id: req.params.subActionParams.rule.id, + name: req.params.subActionParams.rule.name, + }, + }); + + verifyAlertsAttachedToCase({ + caseAttachments: secondCaseAttachments, + expectedAlertIdsToBeAttachedToCase: new Set(['alert-id-1']), + rule: { + id: req.params.subActionParams.rule.id, + name: req.params.subActionParams.rule.name, + }, + }); + }); + }); }); });