Skip to content

Commit

Permalink
Adding autoRecoverAlerts flag to rule type to determine whether recov…
Browse files Browse the repository at this point in the history
…ery alerts should be calculated and alerts stored in task state. Hacking 1h throttle to 5m for testing
  • Loading branch information
ymao1 committed Dec 14, 2022
1 parent 81b1a67 commit 526f376
Show file tree
Hide file tree
Showing 8 changed files with 41 additions and 11 deletions.
13 changes: 11 additions & 2 deletions x-pack/plugins/alerting/server/lib/process_alerts.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ interface ProcessAlertsOpts<
previouslyRecoveredAlerts: Record<string, Alert<State, Context>>;
hasReachedAlertLimit: boolean;
alertLimit: number;
autoRecoverAlerts: boolean;
// flag used to determine whether or not we want to push the flapping state on to the flappingHistory array
setFlapping: boolean;
}
Expand Down Expand Up @@ -47,6 +48,7 @@ export function processAlerts<
previouslyRecoveredAlerts,
hasReachedAlertLimit,
alertLimit,
autoRecoverAlerts,
setFlapping,
}: ProcessAlertsOpts<State, Context>): ProcessAlertsResult<
State,
Expand All @@ -62,7 +64,13 @@ export function processAlerts<
alertLimit,
setFlapping
)
: processAlertsHelper(alerts, existingAlerts, previouslyRecoveredAlerts, setFlapping);
: processAlertsHelper(
alerts,
existingAlerts,
previouslyRecoveredAlerts,
autoRecoverAlerts,
setFlapping
);
}

function processAlertsHelper<
Expand All @@ -74,6 +82,7 @@ function processAlertsHelper<
alerts: Record<string, Alert<State, Context>>,
existingAlerts: Record<string, Alert<State, Context>>,
previouslyRecoveredAlerts: Record<string, Alert<State, Context>>,
autoRecoverAlerts: boolean,
setFlapping: boolean
): ProcessAlertsResult<State, Context, ActionGroupIds, RecoveryActionGroupId> {
const existingAlertIds = new Set(Object.keys(existingAlerts));
Expand Down Expand Up @@ -123,7 +132,7 @@ function processAlertsHelper<
updateAlertFlappingHistory(activeAlerts[id], false);
}
}
} else if (existingAlertIds.has(id)) {
} else if (existingAlertIds.has(id) && autoRecoverAlerts) {
recoveredAlerts[id] = alerts[id];
currentRecoveredAlerts[id] = alerts[id];

Expand Down
2 changes: 2 additions & 0 deletions x-pack/plugins/alerting/server/plugin.ts
Original file line number Diff line number Diff line change
Expand Up @@ -335,6 +335,8 @@ export class AlertingPlugin {
ruleType.cancelAlertsOnRuleTimeout =
ruleType.cancelAlertsOnRuleTimeout ?? this.config.cancelAlertsOnRuleTimeout;
ruleType.doesSetRecoveryContext = ruleType.doesSetRecoveryContext ?? false;
ruleType.autoRecoverAlerts =
ruleType.autoRecoverAlerts === undefined ? true : ruleType.autoRecoverAlerts;
ruleTypeRegistry.register(ruleType);
},
getSecurityHealth: async () => {
Expand Down
23 changes: 17 additions & 6 deletions x-pack/plugins/alerting/server/task_runner/task_runner.ts
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ import {
RuleTypeState,
parseDuration,
WithoutReservedActionGroups,
RawAlertInstance,
} from '../../common';
import { NormalizedRuleType, UntypedNormalizedRuleType } from '../rule_type_registry';
import { getEsErrorMessage } from '../lib/errors';
Expand Down Expand Up @@ -415,6 +416,8 @@ export class TaskRunner<
previouslyRecoveredAlerts: originalRecoveredAlerts,
hasReachedAlertLimit,
alertLimit: this.maxAlerts,
autoRecoverAlerts:
this.ruleType.autoRecoverAlerts !== undefined ? this.ruleType.autoRecoverAlerts : true,
setFlapping: true,
});

Expand Down Expand Up @@ -479,12 +482,20 @@ export class TaskRunner<
}
});

const { alertsToReturn, recoveredAlertsToReturn } = determineAlertsToReturn<
State,
Context,
ActionGroupIds,
RecoveryActionGroupId
>(activeAlerts, recoveredAlerts);
let alertsToReturn: Record<string, RawAlertInstance> = {};
let recoveredAlertsToReturn: Record<string, RawAlertInstance> = {};

// Only serialize alerts into task state if we're auto-recovering, otherwise
// we don't need to keep this information around.
if (this.ruleType.autoRecoverAlerts) {
const { alertsToReturn: alerts, recoveredAlertsToReturn: recovered } =
determineAlertsToReturn<State, Context, ActionGroupIds, RecoveryActionGroupId>(
activeAlerts,
recoveredAlerts
);
alertsToReturn = alerts;
recoveredAlertsToReturn = recovered;
}

return {
metrics: ruleRunMetricsStore.getMetrics(),
Expand Down
6 changes: 6 additions & 0 deletions x-pack/plugins/alerting/server/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -196,6 +196,12 @@ export interface RuleType<
cancelAlertsOnRuleTimeout?: boolean;
doesSetRecoveryContext?: boolean;
getSummarizedAlerts?: GetSummarizedAlertsFn;

/**
* Determines whether framework should
* automatically make recovery determination. Defaults to true.
*/
autoRecoverAlerts?: boolean;
}
export type UntypedRuleType = RuleType<
RuleTypeParams,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,7 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper
};
})
.filter((_, idx) => response.body.items[idx].create?.status === 201);
console.log(`persistence alerts ${createdAlerts.length}`);

createdAlerts.forEach((alert) =>
options.services.alertFactory.create(alert._id).scheduleActions('default', {})
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ export const updateRules = async ({
frequency: {
summary: throttle !== null,
notifyWhen,
throttle,
throttle: throttle === '1h' ? '5m' : throttle,
},
};
})
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -476,7 +476,7 @@ export const convertPatchAPIToInternalSchema = (
frequency: {
summary: throttle !== null,
notifyWhen,
throttle,
throttle: throttle === '1h' ? '5m' : throttle,
},
};
})
Expand Down Expand Up @@ -546,7 +546,7 @@ export const convertCreateAPIToInternalSchema = (
frequency: {
summary: throttle !== null,
notifyWhen,
throttle,
throttle: throttle === '1h' ? '5m' : throttle,
},
};
}) ?? [],
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ export const createSecurityRuleTypeWrapper: CreateSecurityRuleTypeWrapper =
injectReferences: (params, savedObjectReferences) =>
injectReferences({ logger, params, savedObjectReferences }),
},
autoRecoverAlerts: false,
async executor(options) {
agent.setTransactionName(`${options.rule.ruleTypeId} execution`);
return withSecuritySpan('securityRuleTypeExecutor', async () => {
Expand Down

0 comments on commit 526f376

Please sign in to comment.