Skip to content

Commit

Permalink
[RAC][Observability] preserve lifecycle alert changes for active aler…
Browse files Browse the repository at this point in the history
…ts (elastic#110124)

* preserve lifecycle changes for active alerts

* fix failing tests

* fix failing lifecycle executor tests

Co-authored-by: Kibana Machine <[email protected]>
  • Loading branch information
mgiota and kibanamachine committed Aug 31, 2021
1 parent 33a8c30 commit 0d7c5e3
Show file tree
Hide file tree
Showing 3 changed files with 62 additions and 12 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import {
ALERT_STATUS,
ALERT_STATUS_ACTIVE,
ALERT_STATUS_RECOVERED,
ALERT_WORKFLOW_STATUS,
ALERT_UUID,
EVENT_ACTION,
EVENT_KIND,
Expand Down Expand Up @@ -118,25 +119,43 @@ describe('createLifecycleExecutor', () => {
);
});

it('overwrites existing documents for repeatedly firing alerts', async () => {
it('updates existing documents for repeatedly firing alerts', async () => {
const logger = loggerMock.create();
const ruleDataClientMock = createRuleDataClientMock();
ruleDataClientMock.getReader().search.mockResolvedValue({
hits: {
hits: [
{
fields: {
'@timestamp': '',
[ALERT_ID]: 'TEST_ALERT_0',
[ALERT_UUID]: 'ALERT_0_UUID',
[ALERT_RULE_CATEGORY]: 'RULE_TYPE_NAME',
[ALERT_RULE_CONSUMER]: 'CONSUMER',
[ALERT_RULE_NAME]: 'NAME',
[ALERT_RULE_PRODUCER]: 'PRODUCER',
[ALERT_RULE_TYPE_ID]: 'RULE_TYPE_ID',
labels: { LABEL_0_KEY: 'LABEL_0_VALUE' }, // this must not show up in the written doc
[ALERT_RULE_UUID]: 'RULE_UUID',
[ALERT_STATUS]: ALERT_STATUS_ACTIVE,
[ALERT_WORKFLOW_STATUS]: 'closed',
[SPACE_IDS]: ['fake-space-id'],
labels: { LABEL_0_KEY: 'LABEL_0_VALUE' }, // this must show up in the written doc
},
},
{
fields: {
'@timestamp': '',
[ALERT_ID]: 'TEST_ALERT_1',
[ALERT_UUID]: 'ALERT_1_UUID',
[ALERT_RULE_CATEGORY]: 'RULE_TYPE_NAME',
[ALERT_RULE_CONSUMER]: 'CONSUMER',
[ALERT_RULE_NAME]: 'NAME',
[ALERT_RULE_PRODUCER]: 'PRODUCER',
[ALERT_RULE_TYPE_ID]: 'RULE_TYPE_ID',
[ALERT_RULE_UUID]: 'RULE_UUID',
[ALERT_STATUS]: ALERT_STATUS_ACTIVE,
[ALERT_WORKFLOW_STATUS]: 'open',
[SPACE_IDS]: ['fake-space-id'],
labels: { LABEL_0_KEY: 'LABEL_0_VALUE' }, // this must not show up in the written doc
},
},
Expand Down Expand Up @@ -188,14 +207,19 @@ describe('createLifecycleExecutor', () => {
{ index: { _id: 'TEST_ALERT_0_UUID' } },
expect.objectContaining({
[ALERT_ID]: 'TEST_ALERT_0',
[ALERT_WORKFLOW_STATUS]: 'closed',
[ALERT_STATUS]: ALERT_STATUS_ACTIVE,
labels: { LABEL_0_KEY: 'LABEL_0_VALUE' },

[EVENT_ACTION]: 'active',
[EVENT_KIND]: 'signal',
}),
{ index: { _id: 'TEST_ALERT_1_UUID' } },
expect.objectContaining({
[ALERT_ID]: 'TEST_ALERT_1',
[ALERT_WORKFLOW_STATUS]: 'open',
[ALERT_STATUS]: ALERT_STATUS_ACTIVE,

[EVENT_ACTION]: 'active',
[EVENT_KIND]: 'signal',
}),
Expand All @@ -216,8 +240,6 @@ describe('createLifecycleExecutor', () => {
});

it('updates existing documents for recovered alerts', async () => {
// NOTE: the documents should actually also be updated for recurring,
// active alerts (see elastic/kibana#108670)
const logger = loggerMock.create();
const ruleDataClientMock = createRuleDataClientMock();
ruleDataClientMock.getReader().search.mockResolvedValue({
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -182,19 +182,17 @@ export const createLifecycleExecutor = (

const allAlertIds = [...new Set(currentAlertIds.concat(trackedAlertIds))];

const trackedAlertStatesOfRecovered = Object.values(state.trackedAlerts).filter(
(trackedAlertState) => !currentAlerts[trackedAlertState.alertId]
);
const trackedAlertStates = Object.values(state.trackedAlerts);

logger.debug(
`Tracking ${allAlertIds.length} alerts (${newAlertIds.length} new, ${trackedAlertStatesOfRecovered.length} recovered)`
`Tracking ${allAlertIds.length} alerts (${newAlertIds.length} new, ${trackedAlertStates.length} previous)`
);

const alertsDataMap: Record<string, Partial<ParsedTechnicalFields>> = {
...currentAlerts,
};

if (trackedAlertStatesOfRecovered.length) {
if (trackedAlertStates.length) {
const { hits } = await ruleDataClient.getReader().search({
body: {
query: {
Expand All @@ -207,15 +205,15 @@ export const createLifecycleExecutor = (
},
{
terms: {
[ALERT_UUID]: trackedAlertStatesOfRecovered.map(
[ALERT_UUID]: trackedAlertStates.map(
(trackedAlertState) => trackedAlertState.alertUuid
),
},
},
],
},
},
size: trackedAlertStatesOfRecovered.length,
size: trackedAlertStates.length,
collapse: {
field: ALERT_UUID,
},
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -263,6 +263,36 @@ describe('createLifecycleRuleTypeFactory', () => {
},
]);

// TODO mock the resolved value before calling alertWithLifecycle again
const lastOpbeansNodeDoc = helpers.ruleDataClientMock
.getWriter()
.bulk.mock.calls[0][0].body?.concat()
.reverse()
.find(
(doc: any) => !('index' in doc) && doc['service.name'] === 'opbeans-node'
) as Record<string, any>;

const stored = mapValues(lastOpbeansNodeDoc, (val) => {
return castArray(val);
});

helpers.ruleDataClientMock.getReader().search.mockResolvedValueOnce({
hits: {
hits: [{ fields: stored } as any],
total: {
value: 1,
relation: 'eq',
},
},
took: 0,
timed_out: false,
_shards: {
failed: 0,
successful: 1,
total: 1,
},
});

await helpers.alertWithLifecycle([
{
id: 'opbeans-java',
Expand All @@ -274,14 +304,14 @@ describe('createLifecycleRuleTypeFactory', () => {
id: 'opbeans-node',
fields: {
'service.name': 'opbeans-node',
'kibana.alert.workflow_status': 'closed',
},
},
]);
});

it('writes the correct alerts', () => {
expect(helpers.ruleDataClientMock.getWriter().bulk).toHaveBeenCalledTimes(2);

const body = helpers.ruleDataClientMock.getWriter().bulk.mock.calls[1][0].body!;

const documents = body.filter((op: any) => !('index' in op)) as any[];
Expand Down

0 comments on commit 0d7c5e3

Please sign in to comment.