From c6b33613e89e7aaaccbca184b65194b31a40e6ef Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Wed, 25 Sep 2019 08:47:50 +0200 Subject: [PATCH] [APM]: Replace error occurrence watchers with Kibana Alerting --- .../plugins/apm/common/alerting/constants.ts | 8 + x-pack/legacy/plugins/apm/index.ts | 9 +- .../ServiceIntegrations/WatcherFlyout.tsx | 145 ++++------------ .../createErrorOccurrenceAlert.ts | 48 ++++++ .../ServiceIntegrations/index.tsx | 3 + .../components/app/ServiceDetails/index.tsx | 7 +- .../alerting/error_occurrence/create_alert.ts | 159 ++++++++++++++++++ .../register_error_occurrence_alert_type.ts | 140 +++++++++++++++ .../lib/alerting/register_alert_types.ts | 24 +++ .../plugins/apm/server/new-platform/plugin.ts | 2 + .../apm/server/routes/create_apm_api.ts | 6 +- .../server/routes/error_occurrence_alert.ts | 40 +++++ 12 files changed, 472 insertions(+), 119 deletions(-) create mode 100644 x-pack/legacy/plugins/apm/common/alerting/constants.ts create mode 100644 x-pack/legacy/plugins/apm/public/components/app/ServiceDetails/ServiceIntegrations/createErrorOccurrenceAlert.ts create mode 100644 x-pack/legacy/plugins/apm/server/lib/alerting/error_occurrence/create_alert.ts create mode 100644 x-pack/legacy/plugins/apm/server/lib/alerting/error_occurrence/register_error_occurrence_alert_type.ts create mode 100644 x-pack/legacy/plugins/apm/server/lib/alerting/register_alert_types.ts create mode 100644 x-pack/legacy/plugins/apm/server/routes/error_occurrence_alert.ts diff --git a/x-pack/legacy/plugins/apm/common/alerting/constants.ts b/x-pack/legacy/plugins/apm/common/alerting/constants.ts new file mode 100644 index 0000000000000..c43985bc22584 --- /dev/null +++ b/x-pack/legacy/plugins/apm/common/alerting/constants.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export const ERROR_OCCURRENCE_ALERT_TYPE_ID = 'apm.error_occurrence'; +export const NOTIFICATION_EMAIL_ACTION_ID = 'apm.notificationEmail'; diff --git a/x-pack/legacy/plugins/apm/index.ts b/x-pack/legacy/plugins/apm/index.ts index 556bce9d37bb5..2409858696a02 100644 --- a/x-pack/legacy/plugins/apm/index.ts +++ b/x-pack/legacy/plugins/apm/index.ts @@ -17,7 +17,14 @@ import { plugin } from './server/new-platform/index'; export const apm: LegacyPluginInitializer = kibana => { return new kibana.Plugin({ - require: ['kibana', 'elasticsearch', 'xpack_main', 'apm_oss'], + require: [ + 'kibana', + 'elasticsearch', + 'xpack_main', + 'apm_oss', + 'alerting', + 'actions' + ], id: 'apm', configPrefix: 'xpack.apm', publicDir: resolve(__dirname, 'public'), diff --git a/x-pack/legacy/plugins/apm/public/components/app/ServiceDetails/ServiceIntegrations/WatcherFlyout.tsx b/x-pack/legacy/plugins/apm/public/components/app/ServiceDetails/ServiceIntegrations/WatcherFlyout.tsx index 291208b2d9032..1ec8d9a619b62 100644 --- a/x-pack/legacy/plugins/apm/public/components/app/ServiceDetails/ServiceIntegrations/WatcherFlyout.tsx +++ b/x-pack/legacy/plugins/apm/public/components/app/ServiceDetails/ServiceIntegrations/WatcherFlyout.tsx @@ -17,7 +17,6 @@ import { EuiForm, EuiFormRow, EuiLink, - EuiRadio, EuiSelect, EuiSpacer, EuiSwitch, @@ -26,15 +25,17 @@ import { } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n/react'; -import { padLeft, range } from 'lodash'; -import moment from 'moment-timezone'; import React, { Component } from 'react'; import styled from 'styled-components'; +import { toastNotifications } from 'ui/notify'; +import { NOT_AVAILABLE_LABEL } from '../../../../../common/i18n'; import { KibanaCoreContext } from '../../../../../../observability/public'; import { IUrlParams } from '../../../../context/UrlParamsContext/types'; import { KibanaLink } from '../../../shared/Links/KibanaLink'; -import { createErrorGroupWatch, Schedule } from './createErrorGroupWatch'; +import { Schedule } from './createErrorGroupWatch'; import { ElasticDocsLink } from '../../../shared/Links/ElasticDocsLink'; +import { createErrorOccurrenceAlert } from './createErrorOccurrenceAlert'; +import { APMClient } from '../../../../services/rest/createCallApmApi'; type ScheduleKey = keyof Schedule; @@ -49,6 +50,7 @@ const SmallInput = styled.div` interface WatcherFlyoutProps { urlParams: IUrlParams; + callApmApi: APMClient; onClose: () => void; isOpen: boolean; } @@ -149,57 +151,32 @@ export class WatcherFlyout extends Component< }; public createWatch = () => { - const core = this.context; const { serviceName } = this.props.urlParams; if (!serviceName) { return; } - const emails = this.state.actions.email - ? this.state.emails - .split(',') - .map(email => email.trim()) - .filter(email => !!email) - : []; + const email = this.state.actions.email ? this.state.emails : ''; const slackUrl = this.state.actions.slack ? this.state.slackUrl : ''; - const schedule = - this.state.schedule === 'interval' - ? { - interval: `${this.state.interval.value}${this.state.interval.unit}` - } - : { - daily: { at: `${this.state.daily}` } - }; + const timeRange = { + value: this.state.interval.value, + unit: this.state.interval.unit + }; - const timeRange = - this.state.schedule === 'interval' - ? { - value: this.state.interval.value, - unit: this.state.interval.unit - } - : { - value: 24, - unit: 'h' - }; - - const apmIndexPatternTitle = core.injectedMetadata.getInjectedVar( - 'apmIndexPatternTitle' - ) as string; - - return createErrorGroupWatch({ - emails, - schedule, + return createErrorOccurrenceAlert({ + callApmApi: this.props.callApmApi, + email, serviceName, slackUrl, threshold: this.state.threshold, - timeRange, - apmIndexPatternTitle + timeRange }) - .then((id: string) => { + .then(savedObject => { this.props.onClose(); + const id = 'id' in savedObject ? savedObject.id : NOT_AVAILABLE_LABEL; this.addSuccessToast(id); }) .catch(e => { @@ -210,9 +187,7 @@ export class WatcherFlyout extends Component< }; public addErrorToast = () => { - const core = this.context; - - core.notifications.toasts.addWarning({ + toastNotifications.addWarning({ title: i18n.translate( 'xpack.apm.serviceDetails.enableErrorReportsPanel.watchCreationFailedNotificationTitle', { @@ -234,9 +209,7 @@ export class WatcherFlyout extends Component< }; public addSuccessToast = (id: string) => { - const core = this.context; - - core.notifications.toasts.addSuccess({ + toastNotifications.addSuccess({ title: i18n.translate( 'xpack.apm.serviceDetails.enableErrorReportsPanel.watchCreatedNotificationTitle', { @@ -255,18 +228,16 @@ export class WatcherFlyout extends Component< } } )}{' '} - - - {i18n.translate( - 'xpack.apm.serviceDetails.enableErrorReportsPanel.watchCreatedNotificationText.viewWatchLinkText', - { - defaultMessage: 'View watch' - } - )} - - + + {i18n.translate( + 'xpack.apm.serviceDetails.enableErrorReportsPanel.watchCreatedNotificationText.viewWatchLinkText', + { + defaultMessage: 'View watch' + } + )} +

) }); @@ -277,20 +248,6 @@ export class WatcherFlyout extends Component< return null; } - const dailyTime = this.state.daily; - const inputTime = `${dailyTime}Z`; // Add tz to make into UTC - const inputFormat = 'HH:mmZ'; // Parse as 24 hour w. tz - const dailyTimeFormatted = moment(inputTime, inputFormat).format('HH:mm'); // Format as 24h - const dailyTime12HourFormatted = moment(inputTime, inputFormat).format( - 'hh:mm A (z)' - ); // Format as 12h w. tz - - // Generate UTC hours for Daily Report select field - const intervalHours = range(24).map(i => { - const hour = padLeft(i.toString(), 2, '0'); - return { value: `${hour}:00`, text: `${hour}:00 UTC` }; - }); - const flyoutBody = (

@@ -367,50 +324,6 @@ export class WatcherFlyout extends Component< } )} - - this.onChangeSchedule('daily')} - checked={this.state.schedule === 'daily'} - /> - - - - - - this.onChangeSchedule('interval')} - checked={this.state.schedule === 'interval'} - /> - diff --git a/x-pack/legacy/plugins/apm/public/components/app/ServiceDetails/ServiceIntegrations/createErrorOccurrenceAlert.ts b/x-pack/legacy/plugins/apm/public/components/app/ServiceDetails/ServiceIntegrations/createErrorOccurrenceAlert.ts new file mode 100644 index 0000000000000..36ebb9ca789c5 --- /dev/null +++ b/x-pack/legacy/plugins/apm/public/components/app/ServiceDetails/ServiceIntegrations/createErrorOccurrenceAlert.ts @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { APMClient } from '../../../../services/rest/createCallApmApi'; + +export interface Schedule { + interval: string; +} + +interface Arguments { + callApmApi: APMClient; + email: string; + serviceName: string; + slackUrl: string; + threshold: number; + timeRange: { + value: number; + unit: string; + }; +} + +export async function createErrorOccurrenceAlert({ + callApmApi, + email, + serviceName, + slackUrl, + threshold, + timeRange +}: Arguments) { + return callApmApi({ + pathname: '/api/apm/alerts/error_occurrence', + method: 'POST', + params: { + body: { + serviceName, + threshold, + interval: `${timeRange.value}${timeRange.unit}`, + actions: { + email, + slack: slackUrl + } + } + } + }); +} diff --git a/x-pack/legacy/plugins/apm/public/components/app/ServiceDetails/ServiceIntegrations/index.tsx b/x-pack/legacy/plugins/apm/public/components/app/ServiceDetails/ServiceIntegrations/index.tsx index 8a5d7ad10f22b..a81bda0e64a29 100644 --- a/x-pack/legacy/plugins/apm/public/components/app/ServiceDetails/ServiceIntegrations/index.tsx +++ b/x-pack/legacy/plugins/apm/public/components/app/ServiceDetails/ServiceIntegrations/index.tsx @@ -19,9 +19,11 @@ import { IUrlParams } from '../../../../context/UrlParamsContext/types'; import { LicenseContext } from '../../../../context/LicenseContext'; import { MachineLearningFlyout } from './MachineLearningFlyout'; import { WatcherFlyout } from './WatcherFlyout'; +import { APMClient } from '../../../../services/rest/createCallApmApi'; interface Props { urlParams: IUrlParams; + callApmApi: APMClient; } interface State { isPopoverOpen: boolean; @@ -162,6 +164,7 @@ export class ServiceIntegrations extends React.Component { urlParams={this.props.urlParams} /> ['tab']; @@ -18,6 +19,7 @@ interface Props { export function ServiceDetails({ tab }: Props) { const { urlParams } = useUrlParams(); const { serviceName } = urlParams; + const callApmApi = useCallApmApi(); return (

@@ -29,7 +31,10 @@ export function ServiceDetails({ tab }: Props) { - + diff --git a/x-pack/legacy/plugins/apm/server/lib/alerting/error_occurrence/create_alert.ts b/x-pack/legacy/plugins/apm/server/lib/alerting/error_occurrence/create_alert.ts new file mode 100644 index 0000000000000..6e6ebd79cbb4b --- /dev/null +++ b/x-pack/legacy/plugins/apm/server/lib/alerting/error_occurrence/create_alert.ts @@ -0,0 +1,159 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { i18n } from '@kbn/i18n'; +import { ERROR_OCCURRENCE_ALERT_TYPE_ID } from '../../../../common/alerting/constants'; +import { ActionsClient } from '../../../../../actions'; +import { AlertsClient } from '../../../../../alerting'; + +interface Properties { + serviceName: string; + threshold: number; + interval: string; + actions: { + email: string; + slack: string; + }; +} + +interface Clients { + alertsClient: AlertsClient; + actionsClient: ActionsClient; +} + +interface Action { + actionTypeId: string; + description: string; + config: Record; + secrets: Record; +} +const createAlert = async ( + { alertsClient, actionsClient }: Clients, + { actions: { slack, email }, serviceName, threshold, interval }: Properties +) => { + const actions: Array<{ params: Record; action: Action }> = []; + + const values = { + errorLogMessage: '{{{context.errorLogMessage}}}', + errorCulprit: '{{{context.errorCulprit}}}', + docCount: '{{context.docCount}}', + serviceName, + threshold, + interval + }; + + if (email) { + actions.push({ + action: { + actionTypeId: '.email', + description: 'Email notifications for error occurrences', + config: { + from: '' + }, + secrets: { + user: '', + password: '' + } + }, + params: { + to: email, + subject: i18n.translate( + 'xpack.apm.serviceDetails.enableErrorReportsPanel.emailSubjectText', + { + defaultMessage: + '{serviceName} has error groups which exceeds the threshold', + values: { serviceName } + } + ), + body: i18n.translate( + 'xpack.apm.serviceDetails.enableErrorReportsPanel.emailTemplateText', + { + defaultMessage: + `Your service {serviceName} has error groups which exceeds {threshold} occurrences within {interval}
` + + '
' + + '{errorLogMessage}
' + + '{errorCulprit}
' + + '{docCount} occurrences
', + values + } + ) + } + }); + } + + if (slack) { + actions.push({ + action: { + actionTypeId: '.slack', + description: 'Slack notifications for error occurrences', + config: {}, + secrets: { + webhookUrl: slack + } + }, + params: { + message: i18n.translate( + 'xpack.apm.serviceDetails.enableErrorReportsPanel.slackTemplateText', + { + defaultMessage: `Your service *{serviceName}* has error groups which exceeds {threshold} occurrences within {interval} +>*{errorLogMessage}* +>\`{errorCulprit}\` +>{docCount} occurrences`, + values + } + ) + } + }); + } + + if (!actions.length) { + throw new Error( + 'No actions were defined. Need at least one of email, slack' + ); + } + + const actionResults = await Promise.all( + actions.map(({ action, params }) => { + return actionsClient + .create({ + action: { + actionTypeId: action.actionTypeId, + description: action.description, + config: action.config, + secrets: action.secrets + } + }) + .then(result => ({ + result, + params + })); + }) + ); + + return alertsClient.create({ + data: { + name: `Error threshold for ${serviceName} exceeded`, + alertTypeId: ERROR_OCCURRENCE_ALERT_TYPE_ID, + alertTypeParams: { + serviceName, + threshold, + interval + }, + interval, + enabled: true, + actions: actionResults.map(({ result, params }) => { + return { + group: 'default', + id: result.id, + params + }; + }), + throttle: null + } + }); +}; + +export { createAlert }; diff --git a/x-pack/legacy/plugins/apm/server/lib/alerting/error_occurrence/register_error_occurrence_alert_type.ts b/x-pack/legacy/plugins/apm/server/lib/alerting/error_occurrence/register_error_occurrence_alert_type.ts new file mode 100644 index 0000000000000..70b72c55dae95 --- /dev/null +++ b/x-pack/legacy/plugins/apm/server/lib/alerting/error_occurrence/register_error_occurrence_alert_type.ts @@ -0,0 +1,140 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { schema, TypeOf } from '@kbn/config-schema'; +import { InternalCoreSetup } from 'src/core/server'; +import { idx } from '@kbn/elastic-idx'; +import { ESSearchResponse } from '../../../../typings/elasticsearch'; +import { NOT_AVAILABLE_LABEL } from '../../../../common/i18n'; +import { APMError } from '../../../../typings/es_schemas/ui/APMError'; +import { ERROR_OCCURRENCE_ALERT_TYPE_ID } from '../../../../common/alerting/constants'; +import { + SERVICE_NAME, + PROCESSOR_EVENT, + ERROR_LOG_MESSAGE, + ERROR_EXC_MESSAGE, + ERROR_EXC_HANDLED, + ERROR_CULPRIT, + ERROR_GROUP_ID +} from '../../../../common/elasticsearch_fieldnames'; + +const paramsSchema = schema.object({ + threshold: schema.number({ min: 1 }), + interval: schema.string(), + serviceName: schema.string() +}); + +const registerErrorOccurrenceAlertType = async (core: InternalCoreSetup) => { + const { alerting, elasticsearch } = core.http.server.plugins; + + if (!alerting) { + throw new Error( + 'Cannot register error occurrence alert type. Both the actions and alerting plugins need to be enabled' + ); + } + + alerting.setup.registerType({ + id: ERROR_OCCURRENCE_ALERT_TYPE_ID, + name: 'Error occurrences', + actionGroups: ['default'], + validate: { + params: paramsSchema + }, + async executor({ params, services }) { + const { threshold, interval, serviceName } = params as TypeOf< + typeof paramsSchema + >; + + const request = { + body: { + query: { + bool: { + filter: [ + { term: { [SERVICE_NAME]: serviceName } }, + { term: { [PROCESSOR_EVENT]: 'error' } }, + { + range: { + '@timestamp': { + gte: `now-${interval}` + } + } + } + ] + } + }, + aggs: { + error_groups: { + terms: { + min_doc_count: threshold, + field: ERROR_GROUP_ID, + size: 10, + order: { + _count: 'desc' as const + } + }, + aggs: { + sample: { + top_hits: { + _source: [ + ERROR_LOG_MESSAGE, + ERROR_EXC_MESSAGE, + ERROR_EXC_HANDLED, + ERROR_CULPRIT, + ERROR_GROUP_ID, + '@timestamp' + ], + sort: [ + { + '@timestamp': 'desc' as const + } + ], + size: 1 + } + } + } + } + } + } + }; + + const { callWithInternalUser } = elasticsearch.getCluster('admin'); + + const response: ESSearchResponse< + unknown, + typeof request + > = await callWithInternalUser('search', request); + const { aggregations } = response; + + if (!aggregations) { + throw new Error( + 'No aggregations were returned for search. This happens when no matching indices are found.' + ); + } + + const infringingErrorGroups = aggregations.error_groups.buckets; + + const shouldAlert = infringingErrorGroups.length > 0; + + if (shouldAlert) { + const alertInstance = services.alertInstanceFactory(''); + + const sampleError = infringingErrorGroups[0].sample.hits.hits[0] + ._source as APMError; + + alertInstance.scheduleActions('default', { + errorGroupsBuckets: infringingErrorGroups, + errorLogMessage: + idx(sampleError, _ => _.error.log.message) || + idx(sampleError, _ => _.error.exception[0].message), + errorCulprit: + idx(sampleError, _ => _.error.culprit) || NOT_AVAILABLE_LABEL, + docCount: infringingErrorGroups[0].doc_count + }); + } + } + }); +}; + +export { registerErrorOccurrenceAlertType }; diff --git a/x-pack/legacy/plugins/apm/server/lib/alerting/register_alert_types.ts b/x-pack/legacy/plugins/apm/server/lib/alerting/register_alert_types.ts new file mode 100644 index 0000000000000..2654ee40bfa8b --- /dev/null +++ b/x-pack/legacy/plugins/apm/server/lib/alerting/register_alert_types.ts @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { InternalCoreSetup } from 'src/core/server'; +import { registerErrorOccurrenceAlertType } from './error_occurrence/register_error_occurrence_alert_type'; + +const registerAlertTypes = (core: InternalCoreSetup) => { + const { server } = core.http; + + const { alerting } = server.plugins; + + if (alerting) { + const registerFns = [registerErrorOccurrenceAlertType]; + + registerFns.forEach(fn => { + fn(core); + }); + } +}; + +export { registerAlertTypes }; diff --git a/x-pack/legacy/plugins/apm/server/new-platform/plugin.ts b/x-pack/legacy/plugins/apm/server/new-platform/plugin.ts index 0458c8e4fedf0..e2caabee91aeb 100644 --- a/x-pack/legacy/plugins/apm/server/new-platform/plugin.ts +++ b/x-pack/legacy/plugins/apm/server/new-platform/plugin.ts @@ -9,11 +9,13 @@ import { makeApmUsageCollector } from '../lib/apm_telemetry'; import { CoreSetupWithUsageCollector } from '../lib/apm_telemetry/make_apm_usage_collector'; import { createApmAgentConfigurationIndex } from '../lib/settings/agent_configuration/create_agent_config_index'; import { createApmApi } from '../routes/create_apm_api'; +import { registerAlertTypes } from '../lib/alerting/register_alert_types'; export class Plugin { public setup(core: InternalCoreSetup) { createApmApi().init(core); createApmAgentConfigurationIndex(core); makeApmUsageCollector(core as CoreSetupWithUsageCollector); + registerAlertTypes(core); } } diff --git a/x-pack/legacy/plugins/apm/server/routes/create_apm_api.ts b/x-pack/legacy/plugins/apm/server/routes/create_apm_api.ts index 7d22d9488718f..820477085f70b 100644 --- a/x-pack/legacy/plugins/apm/server/routes/create_apm_api.ts +++ b/x-pack/legacy/plugins/apm/server/routes/create_apm_api.ts @@ -51,6 +51,7 @@ import { serviceNodesLocalFiltersRoute, uiFiltersEnvironmentsRoute } from './ui_filters'; +import { errorOccurrenceAlertRoute } from './error_occurrence_alert'; import { createApi } from './create_api'; import { serviceMapRoute } from './services'; @@ -110,7 +111,10 @@ const createApmApi = () => { .add(transactionsLocalFiltersRoute) .add(serviceNodesLocalFiltersRoute) .add(uiFiltersEnvironmentsRoute) - .add(serviceMapRoute); + .add(serviceMapRoute) + + // alerts + .add(errorOccurrenceAlertRoute); return api; }; diff --git a/x-pack/legacy/plugins/apm/server/routes/error_occurrence_alert.ts b/x-pack/legacy/plugins/apm/server/routes/error_occurrence_alert.ts new file mode 100644 index 0000000000000..1962254ba8057 --- /dev/null +++ b/x-pack/legacy/plugins/apm/server/routes/error_occurrence_alert.ts @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import * as t from 'io-ts'; +import { createRoute } from './create_route'; +import { createAlert } from '../lib/alerting/error_occurrence/create_alert'; + +export const errorOccurrenceAlertRoute = createRoute(core => ({ + path: '/api/apm/alerts/error_occurrence', + method: 'POST', + params: { + body: t.type({ + serviceName: t.string, + threshold: t.number, + actions: t.type({ + email: t.string, + slack: t.string + }), + interval: t.string + }) + }, + handler: async (req, { body }, h) => { + const alertsClient = + typeof req.getAlertsClient === 'function' ? req.getAlertsClient() : null; + + const actionsClient = + typeof req.getActionsClient === 'function' + ? req.getActionsClient() + : null; + + if (!alertsClient || !actionsClient) { + return h.response().code(404); + } + + return createAlert({ alertsClient, actionsClient }, body); + } +}));