[Alerting] More telemetry for 8.0 based on Event Log data

x-pack/plugins/actions/server/plugin.ts

@@ -261,7 +261,8 @@ export class ActionsPlugin implements Plugin<PluginSetupContract, PluginStartCon
         plugins.taskManager,
         core,
         this.kibanaIndexConfig.kibana.index,
-        this.preconfiguredActions
+        this.preconfiguredActions,
+        plugins.eventLog
       );
     }
 

x-pack/plugins/actions/server/usage/actions_telemetry.ts

@@ -314,4 +314,191 @@ function replaceFirstAndLastDotSymbols(strToReplace: string) {
   return hasLastSymbolDot ? `${appliedString.slice(0, -1)}__` : appliedString;
 }
 
-// TODO: Implement executions count telemetry with eventLog, when it will write to index
+export async function getExecutionsTotalCount(
+  esClient: ElasticsearchClient,
+  eventLogIndex: string
+): Promise<{
+  countTotal: number;
+  countByType: Record<string, number>;
+  countFailures: number;
+  countFailuresByType: Record<string, number>;
+  avgExecutionTime: number;
+  avgExecutionTimeByType: Record<string, number>;
+}> {
+  const scriptedMetric = {
+    scripted_metric: {
+      init_script: 'state.connectorTypes = [:];  state.total = 0;',
+      map_script: `
+        if (doc['kibana.saved_objects.type'].value == 'action') { 
+          String connectorType = doc['kibana.saved_objects.type_id'].value; 
+          state.connectorTypes.put(connectorType, state.connectorTypes.containsKey(connectorType) ? state.connectorTypes.get(connectorType) + 1 : 1);
+          state.total++;
+        }
+      `,
+      // Combine script is executed per cluster, but we already have a key-value pair per cluster.
+      // Despite docs that say this is optional, this script can't be blank.
+      combine_script: 'return state',
+      // Reduce script is executed across all clusters, so we need to add up all the total from each cluster
+      // This also needs to account for having no data
+      reduce_script: `
+          Map connectorTypes = [:];
+          long total = 0;
+          for (state in states) {
+            if (state !== null) {
+              total += state.total;
+              for (String k : state.connectorTypes.keySet()) {
+                connectorTypes.put(k, connectorTypes.containsKey(k) ? connectorTypes.get(k) + state.connectorTypes.get(k) : state.connectorTypes.get(k));
+              }
+            }
+          }
+          Map result = new HashMap();
+          result.total = total;
+          result.connectorTypes = connectorTypes;
+          return result;
+      `,
+    },
+  };
+
+  const { body: actionResults } = await esClient.search({
+    index: eventLogIndex,
+    body: {
+      query: {
+        bool: {
+          filter: {
+            bool: {
+              must: [
+                {
+                  term: { 'event.action': 'execute' },
+                },
+                {
+                  nested: {
+                    path: 'kibana.saved_objects',
+                    query: {
+                      bool: {
+                        must: [
+                          {
+                            term: {
+                              'kibana.saved_objects.type': {
+                                value: 'action',
+                              },
+                            },
+                          },
+                        ],
+                      },
+                    },
+                  },
+                },
+              ],
+            },
+          },
+        },
+      },
+      aggs: {
+        totalExecutions: {
+          nested: {
+            path: 'kibana.saved_objects',
+          },
+          aggs: {
+            byConnectorTypeId: scriptedMetric,
+          },
+        },
+        failuresExecutions: {
+          filter: {
+            bool: {
+              filter: [
+                {
+                  term: {
+                    'event.outcome': 'failure',
+                  },
+                },
+              ],
+            },
+          },
+          aggs: {
+            refs: {
+              nested: {
+                path: 'kibana.saved_objects',
+              },
+              aggs: {
+                byConnectorTypeId: scriptedMetric,
+              },
+            },
+          },
+        },
+        avgDuration: { avg: { field: 'event.duration' } },
+      },
+    },
+  });
+
+  // @ts-expect-error aggegation type is not specified
+  const aggsExecutions = actionResults.aggregations.totalExecutions?.byConnectorTypeId.value;
+  const aggsAvgExecutionTime = Math.round(
+    // @ts-expect-error aggegation type is not specified
+    actionResults.aggregations.avgDuration.value / (1000 * 1000)
+  ); // nano seconds
+  const aggsFailureExecutions =
+    // @ts-expect-error aggegation type is not specified
+    actionResults.aggregations.failuresExecutions?.refs?.byConnectorTypeId.value;
+
+  const avgExecutionTimeByType: Record<string, number> = {};
+  for (const [key] of Object.entries(aggsExecutions.connectorTypes)) {
+    const { body: connectorTypeResults } = await esClient.search({
+      index: eventLogIndex,
+      body: {
+        query: {
+          bool: {
+            filter: {
+              bool: {
+                must: [
+                  {
+                    term: { 'event.action': 'execute' },
+                  },
+                  {
+                    nested: {
+                      path: 'kibana.saved_objects',
+                      query: {
+                        bool: {
+                          must: [
+                            {
+                              term: {
+                                'kibana.saved_objects.type': {
+                                  value: 'action',
+                                },
+                              },
+                            },
+                            {
+                              term: {
+                                'kibana.saved_objects.type_id': {
+                                  value: key,
+                                },
+                              },
+                            },
+                          ],
+                        },
+                      },
+                    },
+                  },
+                ],
+              },
+            },
+          },
+        },
+        aggs: {
+          avgDuration: { avg: { field: 'event.duration' } },
+        },
+      },
+    });
+    avgExecutionTimeByType[key] =
+      // @ts-expect-error aggegation type is not specified
+      Math.round(connectorTypeResults.aggregations?.avgDuration.value / (1000 * 1000));
+  }
+
+  return {
+    countTotal: aggsExecutions.total,
+    countByType: aggsExecutions.connectorTypes,
+    countFailures: aggsFailureExecutions.total,
+    countFailuresByType: aggsFailureExecutions.connectorTypes,
+    avgExecutionTime: aggsAvgExecutionTime,
+    avgExecutionTimeByType,
+  };
+}

x-pack/plugins/actions/server/usage/actions_usage_collector.ts

@@ -54,6 +54,12 @@ export function createActionsUsageCollector(
       },
       count_by_type: byTypeSchema,
       count_active_by_type: byTypeSchema,
+      count_actions_executions: { type: 'long' },
+      count_actions_executions_by_type: byTypeSchema,
+      count_actions_executions_failured: { type: 'long' },
+      count_actions_executions_failured_by_type: byTypeSchema,
+      avg_execution_time: { type: 'long' },
+      avg_execution_time_by_type: byTypeSchema,
     },
     fetch: async () => {
       try {
@@ -73,6 +79,12 @@ export function createActionsUsageCollector(
           count_active_alert_history_connectors: 0,
           count_active_by_type: {},
           count_by_type: {},
+          count_actions_executions: 0,
+          count_actions_executions_by_type: {},
+          count_actions_executions_failured: 0,
+          count_actions_executions_failured_by_type: {},
+          avg_execution_time: 0,
+          avg_execution_time_by_type: {},
         };
       }
     },

x-pack/plugins/actions/server/usage/task.ts

@@ -7,13 +7,14 @@
 
 import { Logger, CoreSetup } from 'kibana/server';
 import moment from 'moment';
+import { IEventLogService } from '../../../event_log/server';
 import {
   RunContext,
   TaskManagerSetupContract,
   TaskManagerStartContract,
 } from '../../../task_manager/server';
 import { PreConfiguredAction } from '../types';
-import { getTotalCount, getInUseTotalCount } from './actions_telemetry';
+import { getTotalCount, getInUseTotalCount, getExecutionsTotalCount } from './actions_telemetry';
 
 export const TELEMETRY_TASK_TYPE = 'actions_telemetry';
 
@@ -24,9 +25,17 @@ export function initializeActionsTelemetry(
   taskManager: TaskManagerSetupContract,
   core: CoreSetup,
   kibanaIndex: string,
-  preconfiguredActions: PreConfiguredAction[]
+  preconfiguredActions: PreConfiguredAction[],
+  eventLog: IEventLogService
 ) {
-  registerActionsTelemetryTask(logger, taskManager, core, kibanaIndex, preconfiguredActions);
+  registerActionsTelemetryTask(
+    logger,
+    taskManager,
+    core,
+    kibanaIndex,
+    preconfiguredActions,
+    eventLog
+  );
 }
 
 export function scheduleActionsTelemetry(logger: Logger, taskManager: TaskManagerStartContract) {
@@ -38,13 +47,20 @@ function registerActionsTelemetryTask(
   taskManager: TaskManagerSetupContract,
   core: CoreSetup,
   kibanaIndex: string,
-  preconfiguredActions: PreConfiguredAction[]
+  preconfiguredActions: PreConfiguredAction[],
+  eventLog: IEventLogService
 ) {
   taskManager.registerTaskDefinitions({
     [TELEMETRY_TASK_TYPE]: {
       title: 'Actions usage fetch task',
-      timeout: '5m',
-      createTaskRunner: telemetryTaskRunner(logger, core, kibanaIndex, preconfiguredActions),
+      timeout: '5s',
+      createTaskRunner: telemetryTaskRunner(
+        logger,
+        core,
+        kibanaIndex,
+        preconfiguredActions,
+        eventLog
+      ),
     },
   });
 }
@@ -66,10 +82,12 @@ export function telemetryTaskRunner(
   logger: Logger,
   core: CoreSetup,
   kibanaIndex: string,
-  preconfiguredActions: PreConfiguredAction[]
+  preconfiguredActions: PreConfiguredAction[],
+  eventLog: IEventLogService
 ) {
   return ({ taskInstance }: RunContext) => {
     const { state } = taskInstance;
+    const eventLogIndex = eventLog.getIndexPatterns();
     const getEsClient = () =>
       core.getStartServices().then(
         ([
@@ -84,8 +102,9 @@ export function telemetryTaskRunner(
         return Promise.all([
           getTotalCount(esClient, kibanaIndex, preconfiguredActions),
           getInUseTotalCount(esClient, kibanaIndex),
+          getExecutionsTotalCount(esClient, eventLogIndex),
         ])
-          .then(([totalAggegations, totalInUse]) => {
+          .then(([totalAggegations, totalInUse, totalExecutions]) => {
             return {
               state: {
                 runs: (state.runs || 0) + 1,
@@ -94,6 +113,12 @@ export function telemetryTaskRunner(
                 count_active_total: totalInUse.countTotal,
                 count_active_by_type: totalInUse.countByType,
                 count_active_alert_history_connectors: totalInUse.countByAlertHistoryConnectorType,
+                count_actions_executions: totalExecutions.countTotal,
+                count_actions_executions_by_type: totalExecutions.countByType,
+                count_actions_executions_failured: totalExecutions.countFailures,
+                count_actions_executions_failured_by_type: totalExecutions.countFailuresByType,
+                avg_execution_time: totalExecutions.avgExecutionTime,
+                avg_execution_time_by_type: totalExecutions.avgExecutionTimeByType,
               },
               runAt: getNextMidnight(),
             };
@@ -111,5 +136,5 @@ export function telemetryTaskRunner(
 }
 
 function getNextMidnight() {
-  return moment().add(1, 'd').startOf('d').toDate();
+  return moment().add(1, 'm').startOf('m').toDate();
 }

x-pack/plugins/actions/server/usage/types.ts

@@ -12,7 +12,10 @@ export interface ActionsUsage {
   count_active_alert_history_connectors: number;
   count_by_type: Record<string, number>;
   count_active_by_type: Record<string, number>;
-  // TODO: Implement executions count telemetry with eventLog, when it will write to index
-  // executions_by_type: Record<string, number>;
-  // executions_total: number;
+  count_actions_executions: number;
+  count_actions_executions_by_type: Record<string, number>;
+  count_actions_executions_failured: number;
+  count_actions_executions_failured_by_type: Record<string, number>;
+  avg_execution_time: number;
+  avg_execution_time_by_type: Record<string, number>;
 }

x-pack/plugins/alerting/server/plugin.ts

@@ -216,7 +216,8 @@ export class AlertingPlugin {
           this.telemetryLogger,
           core,
           plugins.taskManager,
-          config.kibana.index
+          config.kibana.index,
+          plugins.eventLog
         );
       });
     }