Skip to content

Commit

Permalink
[Synthetics] fix recovery message (#188461)
Browse files Browse the repository at this point in the history 
## Summary

Fixes #188387

We previously relied on state to populate values in alert recovery
messages. However, values were removed from state when onboarding the
rule with FAAD. Fortunately, onboarding with FAAD means we now have
access to the alert document within recovery. This PR uses values from
that document, instead of state, to populate the alert recovery message.

### Testing
1. Create an oblt cluster with `/create-ccs-cluster` on slack. Choose
`dev-oblt`.
2. Add the configuration values from the oblt command to your kibana.yml
3. Navigate to `app/synthetics/settings/alerting` and add a default
connector. The easiest connector to add would be a Server log
4. Create an intentionally down HTTP monitor
5. Wait to receive an alert to your alert connector
6. Update the monitor so that the monitor comes up
7. Wait to receive an alert recovery message to your alert connector.
The recovery message data should be completely populated
  • Loading branch information
@dominiqueclarke
dominiqueclarke authored Jul 18, 2024
1 parent ac5beb6 commit b562c8e
Show file tree
Hide file tree
Showing 2 changed files with 48 additions and 39 deletions.
66 changes: 36 additions & 30 deletions x-pack/plugins/observability_solution/synthetics/server/alert_rules/common.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -186,7 +186,7 @@ describe('updateState', () => {

describe('setRecoveredAlertsContext', () => {
const alertUuid = 'alert-id';
const location = 'US Central';
const location = 'us_west';
const configId = '12345';
const idWithLocation = `${configId}-${location}`;
const basePath = {
Expand Down Expand Up @@ -222,13 +222,15 @@ describe('setRecoveredAlertsContext', () => {
getRecoveredAlerts: jest.fn().mockReturnValue([
{
alert: {
getId: () => alertUuid,
getState: () => ({
idWithLocation,
monitorName: 'test-monitor',
}),
setContext: jest.fn(),
getUuid: () => alertUuid,
getId: () => idWithLocation,
getState: () => ({}),
setContext: jest.fn(),
},
hit: {
'kibana.alert.instance.id': idWithLocation,
'location.id': location,
configId,
},
},
]),
Expand Down Expand Up @@ -264,11 +266,10 @@ describe('setRecoveredAlertsContext', () => {
tz: 'UTC',
});
expect(alertsClientMock.setAlertData).toBeCalledWith({
id: 'alert-id',
id: idWithLocation,
context: {
checkedAt: 'Feb 26, 2023 @ 00:00:00.000',
configId: '12345',
idWithLocation,
linkMessage: '',
alertDetailsUrl: 'https://localhost:5601/app/observability/alerts/alert-id',
monitorName: 'test-monitor',
Expand All @@ -280,6 +281,8 @@ describe('setRecoveredAlertsContext', () => {
'Monitor "test-monitor" from Unnamed-location is recovered. Checked at February 25, 2023 7:00 PM.',
stateId: '123456',
status: 'recovered',
locationId: location,
idWithLocation,
},
});
});
Expand All @@ -292,13 +295,15 @@ describe('setRecoveredAlertsContext', () => {
getRecoveredAlerts: jest.fn().mockReturnValue([
{
alert: {
getId: () => alertUuid,
getState: () => ({
idWithLocation,
monitorName: 'test-monitor',
}),
setContext: jest.fn(),
getUuid: () => alertUuid,
getId: () => idWithLocation,
getState: () => ({}),
setContext: jest.fn(),
},
hit: {
'kibana.alert.instance.id': idWithLocation,
'location.id': location,
configId,
},
},
]),
Expand Down Expand Up @@ -334,14 +339,13 @@ describe('setRecoveredAlertsContext', () => {
tz: 'UTC',
});
expect(alertsClientMock.setAlertData).toBeCalledWith({
id: 'alert-id',
id: idWithLocation,
context: {
configId: '12345',
checkedAt: 'Feb 26, 2023 @ 00:00:00.000',
monitorUrl: '(unavailable)',
reason:
'Monitor "test-monitor" from Unnamed-location is recovered. Checked at February 25, 2023 7:00 PM.',
idWithLocation,
linkMessage: '',
alertDetailsUrl: 'https://localhost:5601/app/observability/alerts/alert-id',
monitorName: 'test-monitor',
Expand All @@ -350,6 +354,8 @@ describe('setRecoveredAlertsContext', () => {
stateId: '123456',
status: 'recovered',
monitorUrlLabel: 'URL',
idWithLocation,
locationId: location,
},
});
});
Expand All @@ -362,15 +368,15 @@ describe('setRecoveredAlertsContext', () => {
getRecoveredAlerts: jest.fn().mockReturnValue([
{
alert: {
getId: () => alertUuid,
getState: () => ({
idWithLocation,
monitorName: 'test-monitor',
locationId: 'us_west',
configId: '12345-67891',
}),
setContext: jest.fn(),
getId: () => idWithLocation,
getUuid: () => alertUuid,
getState: () => ({}),
setContext: jest.fn(),
},
hit: {
'kibana.alert.instance.id': idWithLocation,
'location.id': location,
configId,
},
},
]),
Expand All @@ -382,7 +388,7 @@ describe('setRecoveredAlertsContext', () => {
configId,
monitorQueryId: 'stale-config',
status: 'down',
locationId: 'location',
locationId: location,
ping: {
state: {
id: '123456',
Expand All @@ -406,20 +412,20 @@ describe('setRecoveredAlertsContext', () => {
tz: 'UTC',
});
expect(alertsClientMock.setAlertData).toBeCalledWith({
id: 'alert-id',
id: idWithLocation,
context: {
configId: '12345-67891',
configId,
idWithLocation,
alertDetailsUrl: 'https://localhost:5601/app/observability/alerts/alert-id',
monitorName: 'test-monitor',
status: 'up',
recoveryReason:
'the monitor is now up again. It ran successfully at Feb 26, 2023 @ 00:00:00.000',
recoveryStatus: 'is now up',
locationId: 'us_west',
locationId: location,
checkedAt: 'Feb 26, 2023 @ 00:00:00.000',
linkMessage:
'- Link: https://localhost:5601/app/synthetics/monitor/12345-67891/errors/123456?locationId=us_west',
'- Link: https://localhost:5601/app/synthetics/monitor/12345/errors/123456?locationId=us_west',
monitorUrl: '(unavailable)',
monitorUrlLabel: 'URL',
reason:
Expand Down
21 changes: 12 additions & 9 deletions x-pack/plugins/observability_solution/synthetics/server/alert_rules/common.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -186,6 +186,9 @@ export const setRecoveredAlertsContext = ({
const alertUuid = recoveredAlert.alert.getUuid();

const state = recoveredAlert.alert.getState();
const alertHit = recoveredAlert.hit;
const locationId = alertHit?.['location.id'];
const configId = alertHit?.configId;

let recoveryReason = '';
let recoveryStatus = i18n.translate(
Expand All @@ -199,15 +202,14 @@ export const setRecoveredAlertsContext = ({
let monitorSummary: MonitorSummaryStatusRule | null = null;
let lastErrorMessage;

if (state?.idWithLocation && staleDownConfigs[state.idWithLocation]) {
const { idWithLocation, locationId } = state;
const downConfig = staleDownConfigs[idWithLocation];
const { ping, configId } = downConfig;
if (recoveredAlertId && locationId && staleDownConfigs[recoveredAlertId]) {
const downConfig = staleDownConfigs[recoveredAlertId];
const { ping } = downConfig;
monitorSummary = getMonitorSummary(
ping,
RECOVERED_LABEL,
locationId,
configId,
downConfig.configId,
dateFormat,
tz
);
Expand Down Expand Up @@ -242,12 +244,11 @@ export const setRecoveredAlertsContext = ({
}
}

if (state?.idWithLocation && upConfigs[state.idWithLocation]) {
const { idWithLocation, configId, locationId } = state;
if (configId && recoveredAlertId && locationId && upConfigs[recoveredAlertId]) {
// pull the last error from state, since it is not available on the up ping
lastErrorMessage = state.lastErrorMessage;
lastErrorMessage = alertHit?.['error.message'];

const upConfig = upConfigs[idWithLocation];
const upConfig = upConfigs[recoveredAlertId];
isUp = Boolean(upConfig) || false;
const ping = upConfig.ping;

Expand Down Expand Up @@ -290,6 +291,8 @@ export const setRecoveredAlertsContext = ({
const context = {
...state,
...(monitorSummary ? monitorSummary : {}),
locationId,
idWithLocation: recoveredAlertId,
lastErrorMessage,
recoveryStatus,
linkMessage,
Expand Down

0 comments on commit b562c8e

Please sign in to comment.