diff --git a/x-pack/plugins/actions/server/usage/actions_telemetry.test.ts b/x-pack/plugins/actions/server/usage/actions_telemetry.test.ts index b5606543b04e5..7216f627ea783 100644 --- a/x-pack/plugins/actions/server/usage/actions_telemetry.test.ts +++ b/x-pack/plugins/actions/server/usage/actions_telemetry.test.ts @@ -721,6 +721,36 @@ Object { }, }, }, + count_connector_types_by_action_run_outcome_per_day: { + actionSavedObjects: { + connector_types: { + buckets: [ + { + key: '.slack', + outcome: { + count: { + buckets: [ + { key: 'success', doc_count: 12 }, + { key: 'failure', doc_count: 1 }, + ], + }, + }, + }, + { + key: '.email', + outcome: { + count: { + buckets: [ + { key: 'success', doc_count: 13 }, + { key: 'failure', doc_count: 2 }, + ], + }, + }, + }, + ], + }, + }, + }, }, } ); @@ -754,6 +784,16 @@ Object { __slack: 7, }, countTotal: 120, + countRunOutcomeByConnectorType: { + __email: { + failure: 2, + success: 13, + }, + __slack: { + failure: 1, + success: 12, + }, + }, hasErrors: false, }); }); @@ -775,6 +815,7 @@ Object { "countByType": Object {}, "countFailed": 0, "countFailedByType": Object {}, + "countRunOutcomeByConnectorType": Object {}, "countTotal": 0, "errorMessage": "oh no", "hasErrors": true, diff --git a/x-pack/plugins/actions/server/usage/actions_telemetry.ts b/x-pack/plugins/actions/server/usage/actions_telemetry.ts index 1e2320e2b1a5d..15d2d6f4d12e5 100644 --- a/x-pack/plugins/actions/server/usage/actions_telemetry.ts +++ b/x-pack/plugins/actions/server/usage/actions_telemetry.ts @@ -7,6 +7,11 @@ import { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types'; import { ElasticsearchClient, Logger } from '@kbn/core/server'; +import { AggregationsTermsAggregateBase } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import { + AvgActionRunOutcomeByConnectorTypeBucket, + parseActionRunOutcomeByConnectorTypesBucket, +} from './lib/parse_connector_type_bucket'; import { AlertHistoryEsIndexConnectorId } from '../../common'; import { ActionResult, PreConfiguredAction } from '../types'; @@ -395,7 +400,7 @@ export async function getInUseTotalCount( } } -function replaceFirstAndLastDotSymbols(strToReplace: string) { +export function replaceFirstAndLastDotSymbols(strToReplace: string) { const hasFirstSymbolDot = strToReplace.startsWith('.'); const appliedString = hasFirstSymbolDot ? strToReplace.replace('.', '__') : strToReplace; const hasLastSymbolDot = strToReplace.endsWith('.'); @@ -415,6 +420,7 @@ export async function getExecutionsPerDayCount( countFailedByType: Record; avgExecutionTime: number; avgExecutionTimeByType: Record; + countRunOutcomeByConnectorType: Record; }> { const scriptedMetric = { scripted_metric: { @@ -536,6 +542,35 @@ export async function getExecutionsPerDayCount( }, }, }, + count_connector_types_by_action_run_outcome_per_day: { + nested: { + path: 'kibana.saved_objects', + }, + aggs: { + actionSavedObjects: { + filter: { term: { 'kibana.saved_objects.type': 'action' } }, + aggs: { + connector_types: { + terms: { + field: 'kibana.saved_objects.type_id', + }, + aggs: { + outcome: { + reverse_nested: {}, + aggs: { + count: { + terms: { + field: 'event.outcome', + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, }, }, }); @@ -564,6 +599,14 @@ export async function getExecutionsPerDayCount( {} ); + const aggsCountConnectorTypeByActionRun = actionResults.aggregations as { + count_connector_types_by_action_run_outcome_per_day: { + actionSavedObjects: { + connector_types: AggregationsTermsAggregateBase; + }; + }; + }; + return { hasErrors: false, countTotal: aggsExecutions.total, @@ -586,6 +629,10 @@ export async function getExecutionsPerDayCount( ), avgExecutionTime: aggsAvgExecutionTime, avgExecutionTimeByType, + countRunOutcomeByConnectorType: parseActionRunOutcomeByConnectorTypesBucket( + aggsCountConnectorTypeByActionRun.count_connector_types_by_action_run_outcome_per_day + .actionSavedObjects.connector_types.buckets + ), }; } catch (err) { const errorMessage = err && err.message ? err.message : err.toString(); @@ -601,6 +648,7 @@ export async function getExecutionsPerDayCount( countFailedByType: {}, avgExecutionTime: 0, avgExecutionTimeByType: {}, + countRunOutcomeByConnectorType: {}, }; } } diff --git a/x-pack/plugins/actions/server/usage/actions_usage_collector.ts b/x-pack/plugins/actions/server/usage/actions_usage_collector.ts index f6e02b2e0fec9..f49525a4eec05 100644 --- a/x-pack/plugins/actions/server/usage/actions_usage_collector.ts +++ b/x-pack/plugins/actions/server/usage/actions_usage_collector.ts @@ -47,6 +47,13 @@ export function createActionsUsageCollector( count_actions_executions_failed_by_type_per_day: byTypeSchema, avg_execution_time_per_day: { type: 'long' }, avg_execution_time_by_type_per_day: byTypeSchema, + count_connector_types_by_action_run_outcome_per_day: { + DYNAMIC_KEY: { + success: { type: 'long' }, + failure: { type: 'long' }, + unknown: { type: 'long' }, + }, + }, }, fetch: async () => { try { @@ -77,6 +84,7 @@ export function createActionsUsageCollector( count_actions_executions_failed_by_type_per_day: {}, avg_execution_time_per_day: 0, avg_execution_time_by_type_per_day: {}, + count_connector_types_by_action_run_outcome_per_day: {}, }; } }, diff --git a/x-pack/plugins/actions/server/usage/lib/parse_connector_type_bucket.test.ts b/x-pack/plugins/actions/server/usage/lib/parse_connector_type_bucket.test.ts new file mode 100644 index 0000000000000..1241cddfeb55e --- /dev/null +++ b/x-pack/plugins/actions/server/usage/lib/parse_connector_type_bucket.test.ts @@ -0,0 +1,88 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { parseActionRunOutcomeByConnectorTypesBucket } from './parse_connector_type_bucket'; + +describe('parseActionRunOutcomeByConnectorTypesBucket', () => { + test('should correctly parse connector type bucket results', () => { + expect( + parseActionRunOutcomeByConnectorTypesBucket([ + { + key: '.server-log', + doc_count: 78, + outcome: { + count: { + buckets: [ + { key: 'success', doc_count: 2 }, + { key: 'failure', doc_count: 1 }, + ], + }, + }, + }, + { + key: '.index', + doc_count: 42, + outcome: { + count: { + buckets: [ + { key: 'success', doc_count: 3 }, + { key: 'failure', doc_count: 4 }, + ], + }, + }, + }, + ]) + ).toEqual({ + __index: { + failure: 4, + success: 3, + }, + '__server-log': { + failure: 1, + success: 2, + }, + }); + }); + + test('should handle missing values', () => { + expect( + parseActionRunOutcomeByConnectorTypesBucket([ + { + key: '.server-log', + doc_count: 78, + outcome: { + count: { + // @ts-expect-error + buckets: [{ key: 'success', doc_count: 2 }, { key: 'failure' }], + }, + }, + }, + { + key: '.index', + outcome: { + // @ts-expect-error + count: {}, + }, + }, + ]) + ).toEqual({ + '__server-log': { + failure: 0, + success: 2, + }, + __index: {}, + }); + }); + + test('should handle empty input', () => { + expect(parseActionRunOutcomeByConnectorTypesBucket([])).toEqual({}); + }); + // + test('should handle undefined input', () => { + expect(parseActionRunOutcomeByConnectorTypesBucket(undefined)).toEqual({}); + }); +}); diff --git a/x-pack/plugins/actions/server/usage/lib/parse_connector_type_bucket.ts b/x-pack/plugins/actions/server/usage/lib/parse_connector_type_bucket.ts new file mode 100644 index 0000000000000..96e1610c635d8 --- /dev/null +++ b/x-pack/plugins/actions/server/usage/lib/parse_connector_type_bucket.ts @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { AggregationsBuckets } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import { replaceFirstAndLastDotSymbols } from '../actions_telemetry'; + +export interface AvgActionRunOutcomeByConnectorTypeBucket { + key: string; + doc_count: number; // Not used for duration telemetry but can be helpful later. + outcome: { count: { buckets: Array<{ key: string; doc_count: number }> } }; +} + +export function parseActionRunOutcomeByConnectorTypesBucket( + connectorTypeBuckets: AggregationsBuckets = [] +) { + const connectorTypes = connectorTypeBuckets as AvgActionRunOutcomeByConnectorTypeBucket[]; + return connectorTypes.reduce((acc, connectorType) => { + const outcomes = connectorType.outcome?.count?.buckets ?? []; + return { + ...acc, + [replaceFirstAndLastDotSymbols(connectorType.key)]: outcomes.reduce((accBucket, bucket) => { + return { ...accBucket, [replaceFirstAndLastDotSymbols(bucket.key)]: bucket.doc_count || 0 }; + }, {}), + }; + }, {}); +} diff --git a/x-pack/plugins/actions/server/usage/task.ts b/x-pack/plugins/actions/server/usage/task.ts index 15f70529d3852..e8861b30ae8b4 100644 --- a/x-pack/plugins/actions/server/usage/task.ts +++ b/x-pack/plugins/actions/server/usage/task.ts @@ -130,6 +130,8 @@ export function telemetryTaskRunner( totalExecutionsPerDay.countFailedByType, avg_execution_time_per_day: totalExecutionsPerDay.avgExecutionTime, avg_execution_time_by_type_per_day: totalExecutionsPerDay.avgExecutionTimeByType, + count_connector_types_by_action_run_outcome_per_day: + totalExecutionsPerDay.countRunOutcomeByConnectorType, }, runAt: getNextMidnight(), }; diff --git a/x-pack/plugins/actions/server/usage/types.ts b/x-pack/plugins/actions/server/usage/types.ts index b58ac9c096f63..62ee11af72f8f 100644 --- a/x-pack/plugins/actions/server/usage/types.ts +++ b/x-pack/plugins/actions/server/usage/types.ts @@ -22,6 +22,7 @@ export interface ActionsUsage { count_actions_executions_by_type_per_day: Record; count_actions_executions_failed_per_day: number; count_actions_executions_failed_by_type_per_day: Record; + count_connector_types_by_action_run_outcome_per_day: Record>; avg_execution_time_per_day: number; avg_execution_time_by_type_per_day: Record; } diff --git a/x-pack/plugins/alerting/server/usage/alerting_usage_collector.ts b/x-pack/plugins/alerting/server/usage/alerting_usage_collector.ts index 6e6eb3cb43f0d..5efd2d7a49152 100644 --- a/x-pack/plugins/alerting/server/usage/alerting_usage_collector.ts +++ b/x-pack/plugins/alerting/server/usage/alerting_usage_collector.ts @@ -116,6 +116,13 @@ const byStatusSchema: MakeSchemaFrom['count_rules_by_execution_st warning: { type: 'long' }, }; +const byStatusPerDaySchema: MakeSchemaFrom['count_rules_by_execution_status_per_day'] = + { + success: { type: 'long' }, + failure: { type: 'long' }, + unknown: { type: 'long' }, + }; + const byNotifyWhenSchema: MakeSchemaFrom['count_rules_by_notify_when'] = { on_action_group_change: { type: 'long' }, on_active_alert: { type: 'long' }, @@ -200,6 +207,7 @@ export function createAlertingUsageCollector( count_rules_muted: 0, count_rules_with_muted_alerts: 0, count_connector_types_by_consumers: {}, + count_rules_by_execution_status_per_day: {}, avg_execution_time_per_day: 0, avg_execution_time_by_type_per_day: {}, avg_es_search_duration_per_day: 0, @@ -283,6 +291,7 @@ export function createAlertingUsageCollector( count_rules_muted: { type: 'long' }, count_rules_with_muted_alerts: { type: 'long' }, count_connector_types_by_consumers: { DYNAMIC_KEY: { DYNAMIC_KEY: { type: 'long' } } }, + count_rules_by_execution_status_per_day: byStatusPerDaySchema, avg_execution_time_per_day: { type: 'long' }, avg_execution_time_by_type_per_day: byTypeSchema, avg_es_search_duration_per_day: { type: 'long' }, diff --git a/x-pack/plugins/alerting/server/usage/lib/get_telemetry_from_event_log.test.ts b/x-pack/plugins/alerting/server/usage/lib/get_telemetry_from_event_log.test.ts index f3e21f6a161fa..64bc0ae8be0fb 100644 --- a/x-pack/plugins/alerting/server/usage/lib/get_telemetry_from_event_log.test.ts +++ b/x-pack/plugins/alerting/server/usage/lib/get_telemetry_from_event_log.test.ts @@ -1291,6 +1291,14 @@ describe('event log telemetry', () => { avg_total_search_duration: { value: 28.630434782608695, }, + by_execution_status: { + doc_count_error_upper_bound: 0, + sum_other_doc_count: 0, + buckets: [ + { key: 'success', doc_count: 21 }, + { key: 'failure', doc_count: 22 }, + ], + }, }, }); @@ -1377,7 +1385,6 @@ describe('event log telemetry', () => { logs__alert__document__count: 0, }, }, - alertsPercentilesByType: { p50: { '__index-threshold': 1, @@ -1398,6 +1405,10 @@ describe('event log telemetry', () => { logs__alert__document__count: 0, }, }, + countRulesByExecutionStatus: { + failure: 22, + success: 21, + }, hasErrors: false, }); }); @@ -1437,6 +1448,7 @@ describe('event log telemetry', () => { generatedActionsPercentilesByType: {}, alertsPercentiles: {}, alertsPercentilesByType: {}, + countRulesByExecutionStatus: {}, }); }); }); diff --git a/x-pack/plugins/alerting/server/usage/lib/get_telemetry_from_event_log.ts b/x-pack/plugins/alerting/server/usage/lib/get_telemetry_from_event_log.ts index 703d579e66c25..e21df2376f4f8 100644 --- a/x-pack/plugins/alerting/server/usage/lib/get_telemetry_from_event_log.ts +++ b/x-pack/plugins/alerting/server/usage/lib/get_telemetry_from_event_log.ts @@ -54,15 +54,14 @@ interface GetExecutionsPerDayCountResults { generatedActionsPercentilesByType: Record>; alertsPercentiles: Record; alertsPercentilesByType: Record>; + countRulesByExecutionStatus: Record; } - interface GetExecutionTimeoutsPerDayCountResults { hasErrors: boolean; errorMessage?: string; countExecutionTimeouts: number; countExecutionTimeoutsByType: Record; } - interface GetExecutionCountsExecutionFailures extends AggregationsSingleBucketAggregateBase { by_reason: AggregationsTermsAggregateBase; } @@ -145,6 +144,11 @@ export async function getExecutionsPerDayCount({ }, aggs: eventLogAggs, }, + by_execution_status: { + terms: { + field: 'event.outcome', + }, + }, }, }, }; @@ -165,6 +169,7 @@ export async function getExecutionsPerDayCount({ avg_execution_time: AggregationsSingleMetricAggregateBase; avg_es_search_duration: AggregationsSingleMetricAggregateBase; avg_total_search_duration: AggregationsSingleMetricAggregateBase; + by_execution_status: AggregationsTermsAggregateBase; }; const aggregationsByRuleTypeId: AggregationsBuckets = @@ -176,6 +181,9 @@ export async function getExecutionsPerDayCount({ ...parseExecutionFailureByRuleType(aggregationsByRuleTypeId), ...parseExecutionCountAggregationResults(aggregations), countTotalRuleExecutions: totalRuleExecutions ?? 0, + countRulesByExecutionStatus: parseSimpleRuleTypeBucket( + aggregations.by_execution_status.buckets + ), }; } catch (err) { const errorMessage = err && err.message ? err.message : err.toString(); @@ -204,6 +212,7 @@ export async function getExecutionsPerDayCount({ generatedActionsPercentilesByType: {}, alertsPercentiles: {}, alertsPercentilesByType: {}, + countRulesByExecutionStatus: {}, }; } } @@ -313,6 +322,14 @@ export async function getExecutionTimeoutsPerDayCount({ * avg_total_search_duration: { // average total search duration across executions * value: 43.74647887323944, * }, + * by_execution_status: { + * "doc_count_error_upper_bound":0, + * "sum_other_doc_count":0, + * "buckets":[ + * {"key":"success","doc_count":48}, + * {"key":"failure","doc_count":1} + * ] + * } * } */ diff --git a/x-pack/plugins/alerting/server/usage/task.ts b/x-pack/plugins/alerting/server/usage/task.ts index 4f835f3c4a64f..ab20844948011 100644 --- a/x-pack/plugins/alerting/server/usage/task.ts +++ b/x-pack/plugins/alerting/server/usage/task.ts @@ -161,6 +161,8 @@ export function telemetryTaskRunner( dailyExecutionCounts.countFailedExecutionsByReason, count_rules_executions_failured_by_reason_by_type_per_day: dailyExecutionCounts.countFailedExecutionsByReasonByType, + count_rules_by_execution_status_per_day: + dailyExecutionCounts.countRulesByExecutionStatus, count_rules_executions_timeouts_per_day: dailyExecutionTimeoutCounts.countExecutionTimeouts, count_rules_executions_timeouts_by_type_per_day: diff --git a/x-pack/plugins/alerting/server/usage/types.ts b/x-pack/plugins/alerting/server/usage/types.ts index 29a32de29bc1a..15c0f0a962710 100644 --- a/x-pack/plugins/alerting/server/usage/types.ts +++ b/x-pack/plugins/alerting/server/usage/types.ts @@ -42,6 +42,7 @@ export interface AlertingUsage { count_rules_snoozed: number; count_rules_muted: number; count_rules_with_muted_alerts: number; + count_rules_by_execution_status_per_day: Record; percentile_num_generated_actions_per_day: { p50: number; p90: number; diff --git a/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json b/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json index 266d8e3d20fe4..90b0efe585a8f 100644 --- a/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json +++ b/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json @@ -265,6 +265,23 @@ "type": "long" } } + }, + "count_connector_types_by_action_run_outcome_per_day": { + "properties": { + "DYNAMIC_KEY": { + "properties": { + "success": { + "type": "long" + }, + "failure": { + "type": "long" + }, + "unknown": { + "type": "long" + } + } + } + } } } }, @@ -1693,6 +1710,19 @@ } } }, + "count_rules_by_execution_status_per_day": { + "properties": { + "success": { + "type": "long" + }, + "failure": { + "type": "long" + }, + "unknown": { + "type": "long" + } + } + }, "avg_execution_time_per_day": { "type": "long" }, diff --git a/x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/telemetry/alerting_and_actions_telemetry.ts b/x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/telemetry/alerting_and_actions_telemetry.ts index 7e6aec1347753..c89e5b48b236b 100644 --- a/x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/telemetry/alerting_and_actions_telemetry.ts +++ b/x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/telemetry/alerting_and_actions_telemetry.ts @@ -246,6 +246,10 @@ export default function createAlertingAndActionsTelemetryTests({ getService }: F expect(telemetry.count_actions_executions_failed_by_type_per_day['test.throw'] > 0).to.be( true ); + + expect( + telemetry.count_connector_types_by_action_run_outcome_per_day['test.throw'].failure + ).to.greaterThan(0); } function verifyAlertingTelemetry(telemetry: any) { @@ -528,6 +532,9 @@ export default function createAlertingAndActionsTelemetryTests({ getService }: F // eslint-disable-next-line @typescript-eslint/naming-convention alertsFixture: { test__noop: 9, test__throw: 9, __slack: 3 }, }); + + expect(telemetry.count_rules_by_execution_status_per_day.failure).to.greaterThan(0); + expect(telemetry.count_rules_by_execution_status_per_day.success).to.greaterThan(0); } it('should retrieve telemetry data in the expected format', async () => {