From 9dada56a2046af4a19a42b45c6087ab94b3fc35a Mon Sep 17 00:00:00 2001 From: Yuliia Naumenko Date: Tue, 19 Oct 2021 11:22:37 -0700 Subject: [PATCH 1/6] [Alerting] Telemetry required for 7.16 (#114690) * [Alerting] Telemetry required for 7.16 * rules namespaces * rules fix * failed task fix * fixed due to the discussion about namespaces keys * added missing telemetry props mappings * added missing telemetry props mappings * added missing telemetry props mappings * fixed tests * changed actions active structure under parent key for alerts * fixed tests * fixed tests * fixed test data * added preconfig connectors for email services count * fixed if statement * fixed default namespace * fixed test * fixed tests * fixed task * removed filtering by referenceType Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../server/usage/actions_telemetry.test.ts | 145 +++++++++++++++- .../actions/server/usage/actions_telemetry.ts | 163 ++++++++++++------ .../server/usage/actions_usage_collector.ts | 29 +--- x-pack/plugins/actions/server/usage/task.ts | 4 +- x-pack/plugins/actions/server/usage/types.ts | 35 +++- .../server/usage/alerts_telemetry.test.ts | 10 +- .../alerting/server/usage/alerts_telemetry.ts | 39 +++-- .../server/usage/alerts_usage_collector.ts | 2 + x-pack/plugins/alerting/server/usage/task.ts | 1 + x-pack/plugins/alerting/server/usage/types.ts | 1 + .../usage/task_manager_usage_collector.ts | 10 ++ .../task_manager/server/usage/types.ts | 1 + .../schema/xpack_plugins.json | 52 +++++- 13 files changed, 395 insertions(+), 97 deletions(-) diff --git a/x-pack/plugins/actions/server/usage/actions_telemetry.test.ts b/x-pack/plugins/actions/server/usage/actions_telemetry.test.ts index 0e6b7fff04451..229f06f2e47fa 100644 --- a/x-pack/plugins/actions/server/usage/actions_telemetry.test.ts +++ b/x-pack/plugins/actions/server/usage/actions_telemetry.test.ts @@ -146,6 +146,7 @@ Object { id: '1', actionTypeId: '.server-log', }, + namespaces: ['default'], }, }, { @@ -154,6 +155,7 @@ Object { id: '2', actionTypeId: '.slack', }, + namespaces: ['default'], }, }, ], @@ -170,6 +172,8 @@ Object { "__server-log": 1, "__slack": 1, }, + "countEmailByService": Object {}, + "countNamespaces": 1, "countTotal": 2, } `); @@ -220,6 +224,7 @@ Object { id: '1', actionTypeId: '.server-log', }, + namespaces: ['default'], }, }, { @@ -228,13 +233,35 @@ Object { id: '2', actionTypeId: '.slack', }, + namespaces: ['default'], }, }, ], }, }) ); - const telemetry = await getInUseTotalCount(mockEsClient, 'test'); + const telemetry = await getInUseTotalCount(mockEsClient, 'test', undefined, [ + { + id: 'test', + actionTypeId: '.email', + name: 'test', + isPreconfigured: true, + config: { + tenantId: 'sdsd', + clientId: 'sdfsdf', + }, + secrets: { + clientSecret: 'sdfsdf', + }, + }, + { + id: 'anotherServerLog', + actionTypeId: '.server-log', + name: 'test', + isPreconfigured: true, + secrets: {}, + }, + ]); expect(mockEsClient.search).toHaveBeenCalledTimes(2); expect(telemetry).toMatchInlineSnapshot(` @@ -245,6 +272,8 @@ Object { "__server-log": 1, "__slack": 1, }, + "countEmailByService": Object {}, + "countNamespaces": 1, "countTotal": 4, } `); @@ -423,6 +452,114 @@ Object { id: '1', actionTypeId: '.server-log', }, + namespaces: ['default'], + }, + }, + { + _source: { + action: { + id: '2', + actionTypeId: '.slack', + }, + namespaces: ['default'], + }, + }, + { + _source: { + action: { + id: '3', + actionTypeId: '.email', + }, + namespaces: ['default'], + }, + }, + ], + }, + }) + ); + const telemetry = await getInUseTotalCount(mockEsClient, 'test', undefined, [ + { + id: 'anotherServerLog', + actionTypeId: '.server-log', + name: 'test', + isPreconfigured: true, + secrets: {}, + }, + ]); + + expect(mockEsClient.search).toHaveBeenCalledTimes(2); + expect(telemetry).toMatchInlineSnapshot(` +Object { + "countByAlertHistoryConnectorType": 1, + "countByType": Object { + "__email": 3, + "__index": 1, + "__server-log": 1, + "__slack": 1, + }, + "countEmailByService": Object { + "other": 3, + }, + "countNamespaces": 1, + "countTotal": 6, +} +`); + }); + + test('getInUseTotalCount() accounts for actions namespaces', async () => { + const mockEsClient = elasticsearchClientMock.createClusterClient().asScoped().asInternalUser; + mockEsClient.search.mockReturnValueOnce( + // @ts-expect-error not full search response + elasticsearchClientMock.createSuccessTransportRequestPromise({ + aggregations: { + refs: { + actionRefIds: { + value: { + connectorIds: { + '1': 'action-0', + '123': 'action-1', + '456': 'action-2', + }, + total: 3, + }, + }, + }, + preconfigured_actions: { + preconfiguredActionRefIds: { + value: { + total: 3, + actionRefs: { + 'preconfigured:preconfigured-alert-history-es-index': { + actionRef: 'preconfigured:preconfigured-alert-history-es-index', + actionTypeId: '.index', + }, + 'preconfigured:cloud_email': { + actionRef: 'preconfigured:cloud_email', + actionTypeId: '.email', + }, + 'preconfigured:cloud_email2': { + actionRef: 'preconfigured:cloud_email2', + actionTypeId: '.email', + }, + }, + }, + }, + }, + }, + }) + ); + mockEsClient.search.mockReturnValueOnce( + // @ts-expect-error not full search response + elasticsearchClientMock.createSuccessTransportRequestPromise({ + hits: { + hits: [ + { + _source: { + action: { + id: '1', + actionTypeId: '.server-log', + }, + namespaces: ['test'], }, }, { @@ -431,6 +568,7 @@ Object { id: '2', actionTypeId: '.slack', }, + namespaces: ['default'], }, }, { @@ -439,6 +577,7 @@ Object { id: '3', actionTypeId: '.email', }, + namespaces: ['test2'], }, }, ], @@ -457,6 +596,10 @@ Object { "__server-log": 1, "__slack": 1, }, + "countEmailByService": Object { + "other": 1, + }, + "countNamespaces": 3, "countTotal": 6, } `); diff --git a/x-pack/plugins/actions/server/usage/actions_telemetry.ts b/x-pack/plugins/actions/server/usage/actions_telemetry.ts index 4a3d0c70e535a..1cb6bf8bfc74c 100644 --- a/x-pack/plugins/actions/server/usage/actions_telemetry.ts +++ b/x-pack/plugins/actions/server/usage/actions_telemetry.ts @@ -5,6 +5,7 @@ * 2.0. */ +import { QueryDslQueryContainer } from '@elastic/elasticsearch/api/types'; import { ElasticsearchClient } from 'kibana/server'; import { AlertHistoryEsIndexConnectorId } from '../../common'; import { ActionResult, PreConfiguredAction } from '../types'; @@ -81,11 +82,15 @@ export async function getTotalCount( export async function getInUseTotalCount( esClient: ElasticsearchClient, - kibanaIndex: string + kibanaIndex: string, + referenceType?: string, + preconfiguredActions?: PreConfiguredAction[] ): Promise<{ countTotal: number; countByType: Record; countByAlertHistoryConnectorType: number; + countEmailByService: Record; + countNamespaces: number; }> { const scriptedMetric = { scripted_metric: { @@ -160,6 +165,63 @@ export async function getInUseTotalCount( }, }; + const mustQuery = [ + { + bool: { + should: [ + { + nested: { + path: 'references', + query: { + bool: { + filter: { + bool: { + must: [ + { + term: { + 'references.type': 'action', + }, + }, + ], + }, + }, + }, + }, + }, + }, + { + nested: { + path: 'alert.actions', + query: { + bool: { + filter: { + bool: { + must: [ + { + prefix: { + 'alert.actions.actionRef': { + value: 'preconfigured:', + }, + }, + }, + ], + }, + }, + }, + }, + }, + }, + ], + }, + }, + ] as QueryDslQueryContainer[]; + + if (!!referenceType) { + mustQuery.push({ + term: { type: referenceType }, + }); + } + const { body: actionResults } = await esClient.search({ index: kibanaIndex, body: { @@ -172,54 +234,7 @@ export async function getInUseTotalCount( type: 'action_task_params', }, }, - must: { - bool: { - should: [ - { - nested: { - path: 'references', - query: { - bool: { - filter: { - bool: { - must: [ - { - term: { - 'references.type': 'action', - }, - }, - ], - }, - }, - }, - }, - }, - }, - { - nested: { - path: 'alert.actions', - query: { - bool: { - filter: { - bool: { - must: [ - { - prefix: { - 'alert.actions.actionRef': { - value: 'preconfigured:', - }, - }, - }, - ], - }, - }, - }, - }, - }, - }, - ], - }, - }, + must: mustQuery, }, }, }, @@ -250,13 +265,15 @@ export async function getInUseTotalCount( const preconfiguredActionsAggs = // @ts-expect-error aggegation type is not specified actionResults.aggregations.preconfigured_actions?.preconfiguredActionRefIds.value; + const { body: { hits: actions }, } = await esClient.search<{ action: ActionResult; + namespaces: string[]; }>({ index: kibanaIndex, - _source_includes: ['action'], + _source_includes: ['action', 'namespaces'], body: { query: { bool: { @@ -274,6 +291,7 @@ export async function getInUseTotalCount( }, }, }); + const countByActionTypeId = actions.hits.reduce( (actionTypeCount: Record, action) => { const actionSource = action._source!; @@ -286,6 +304,26 @@ export async function getInUseTotalCount( {} ); + const namespacesList = actions.hits.reduce((_namespaces: Set, action) => { + const namespaces = action._source?.namespaces ?? ['default']; + namespaces.forEach((namespace) => { + if (!_namespaces.has(namespace)) { + _namespaces.add(namespace); + } + }); + return _namespaces; + }, new Set()); + + const countEmailByService = actions.hits + .filter((action) => action._source!.action.actionTypeId === '.email') + .reduce((emailServiceCount: Record, action) => { + const service = (action._source!.action.config?.service ?? 'other') as string; + const currentCount = + emailServiceCount[service] !== undefined ? emailServiceCount[service] : 0; + emailServiceCount[service] = currentCount + 1; + return emailServiceCount; + }, {}); + let preconfiguredAlertHistoryConnectors = 0; const preconfiguredActionsRefs: Array<{ actionTypeId: string; @@ -298,15 +336,40 @@ export async function getInUseTotalCount( if (actionRef === `preconfigured:${AlertHistoryEsIndexConnectorId}`) { preconfiguredAlertHistoryConnectors++; } + if (preconfiguredActions && actionTypeId === '__email') { + const preconfiguredConnectorId = actionRef.split(':')[1]; + const service = (preconfiguredActions.find( + (preconfConnector) => preconfConnector.id === preconfiguredConnectorId + )?.config?.service ?? 'other') as string; + const currentCount = + countEmailByService[service] !== undefined ? countEmailByService[service] : 0; + countEmailByService[service] = currentCount + 1; + } } return { countTotal: aggs.total + (preconfiguredActionsAggs?.total ?? 0), countByType: countByActionTypeId, countByAlertHistoryConnectorType: preconfiguredAlertHistoryConnectors, + countEmailByService, + countNamespaces: namespacesList.size, }; } +export async function getInUseByAlertingTotalCounts( + esClient: ElasticsearchClient, + kibanaIndex: string, + preconfiguredActions?: PreConfiguredAction[] +): Promise<{ + countTotal: number; + countByType: Record; + countByAlertHistoryConnectorType: number; + countEmailByService: Record; + countNamespaces: number; +}> { + return await getInUseTotalCount(esClient, kibanaIndex, 'alert', preconfiguredActions); +} + function replaceFirstAndLastDotSymbols(strToReplace: string) { const hasFirstSymbolDot = strToReplace.startsWith('.'); const appliedString = hasFirstSymbolDot ? strToReplace.replace('.', '__') : strToReplace; diff --git a/x-pack/plugins/actions/server/usage/actions_usage_collector.ts b/x-pack/plugins/actions/server/usage/actions_usage_collector.ts index 80e0c19092c78..9ba9d7390a7b6 100644 --- a/x-pack/plugins/actions/server/usage/actions_usage_collector.ts +++ b/x-pack/plugins/actions/server/usage/actions_usage_collector.ts @@ -5,29 +5,12 @@ * 2.0. */ -import { MakeSchemaFrom, UsageCollectionSetup } from 'src/plugins/usage_collection/server'; +import { UsageCollectionSetup } from 'src/plugins/usage_collection/server'; import { get } from 'lodash'; import { TaskManagerStartContract } from '../../../task_manager/server'; -import { ActionsUsage } from './types'; +import { ActionsUsage, byServiceProviderTypeSchema, byTypeSchema } from './types'; import { ActionsConfig } from '../config'; -const byTypeSchema: MakeSchemaFrom['count_by_type'] = { - // TODO: Find out an automated way to populate the keys or reformat these into an array (and change the Remote Telemetry indexer accordingly) - DYNAMIC_KEY: { type: 'long' }, - // Known actions: - __email: { type: 'long' }, - __index: { type: 'long' }, - __pagerduty: { type: 'long' }, - __swimlane: { type: 'long' }, - '__server-log': { type: 'long' }, - __slack: { type: 'long' }, - __webhook: { type: 'long' }, - __servicenow: { type: 'long' }, - __jira: { type: 'long' }, - __resilient: { type: 'long' }, - __teams: { type: 'long' }, -}; - export function createActionsUsageCollector( usageCollection: UsageCollectionSetup, config: ActionsConfig, @@ -45,6 +28,7 @@ export function createActionsUsageCollector( _meta: { description: 'Indicates if preconfigured alert history connector is enabled.' }, }, count_total: { type: 'long' }, + count_by_type: byTypeSchema, count_active_total: { type: 'long' }, count_active_alert_history_connectors: { type: 'long', @@ -52,8 +36,9 @@ export function createActionsUsageCollector( description: 'The total number of preconfigured alert history connectors used by rules.', }, }, - count_by_type: byTypeSchema, count_active_by_type: byTypeSchema, + count_active_email_connectors_by_service_type: byServiceProviderTypeSchema, + count_actions_namespaces: { type: 'long' }, }, fetch: async () => { try { @@ -69,10 +54,12 @@ export function createActionsUsageCollector( return { alert_history_connector_enabled: false, count_total: 0, + count_by_type: {}, count_active_total: 0, count_active_alert_history_connectors: 0, count_active_by_type: {}, - count_by_type: {}, + count_active_email_connectors_by_service_type: {}, + count_actions_namespaces: 0, }; } }, diff --git a/x-pack/plugins/actions/server/usage/task.ts b/x-pack/plugins/actions/server/usage/task.ts index 7cbfb87dedda6..bacb9e5f72571 100644 --- a/x-pack/plugins/actions/server/usage/task.ts +++ b/x-pack/plugins/actions/server/usage/task.ts @@ -83,7 +83,7 @@ export function telemetryTaskRunner( const esClient = await getEsClient(); return Promise.all([ getTotalCount(esClient, kibanaIndex, preconfiguredActions), - getInUseTotalCount(esClient, kibanaIndex), + getInUseTotalCount(esClient, kibanaIndex, undefined, preconfiguredActions), ]) .then(([totalAggegations, totalInUse]) => { return { @@ -94,6 +94,8 @@ export function telemetryTaskRunner( count_active_total: totalInUse.countTotal, count_active_by_type: totalInUse.countByType, count_active_alert_history_connectors: totalInUse.countByAlertHistoryConnectorType, + count_active_email_connectors_by_service_type: totalInUse.countEmailByService, + count_actions_namespaces: totalInUse.countNamespaces, }, runAt: getNextMidnight(), }; diff --git a/x-pack/plugins/actions/server/usage/types.ts b/x-pack/plugins/actions/server/usage/types.ts index 9221ba8ea5688..52677b35ac75b 100644 --- a/x-pack/plugins/actions/server/usage/types.ts +++ b/x-pack/plugins/actions/server/usage/types.ts @@ -5,14 +5,47 @@ * 2.0. */ +import { MakeSchemaFrom } from 'src/plugins/usage_collection/server'; + export interface ActionsUsage { alert_history_connector_enabled: boolean; count_total: number; + count_by_type: Record; count_active_total: number; count_active_alert_history_connectors: number; - count_by_type: Record; count_active_by_type: Record; + count_active_email_connectors_by_service_type: Record; + count_actions_namespaces: number; // TODO: Implement executions count telemetry with eventLog, when it will write to index // executions_by_type: Record; // executions_total: number; } + +export const byTypeSchema: MakeSchemaFrom['count_by_type'] = { + // TODO: Find out an automated way to populate the keys or reformat these into an array (and change the Remote Telemetry indexer accordingly) + DYNAMIC_KEY: { type: 'long' }, + // Known actions: + __email: { type: 'long' }, + __index: { type: 'long' }, + __pagerduty: { type: 'long' }, + __swimlane: { type: 'long' }, + '__server-log': { type: 'long' }, + __slack: { type: 'long' }, + __webhook: { type: 'long' }, + __servicenow: { type: 'long' }, + __jira: { type: 'long' }, + __resilient: { type: 'long' }, + __teams: { type: 'long' }, +}; + +export const byServiceProviderTypeSchema: MakeSchemaFrom['count_active_email_connectors_by_service_type'] = + { + DYNAMIC_KEY: { type: 'long' }, + // Known services: + exchange_server: { type: 'long' }, + gmail: { type: 'long' }, + outlook365: { type: 'long' }, + elastic_cloud: { type: 'long' }, + other: { type: 'long' }, + ses: { type: 'long' }, + }; diff --git a/x-pack/plugins/alerting/server/usage/alerts_telemetry.test.ts b/x-pack/plugins/alerting/server/usage/alerts_telemetry.test.ts index cce394d70ed6f..15fa6e63ac561 100644 --- a/x-pack/plugins/alerting/server/usage/alerts_telemetry.test.ts +++ b/x-pack/plugins/alerting/server/usage/alerts_telemetry.test.ts @@ -18,7 +18,14 @@ describe('alerts telemetry', () => { aggregations: { byAlertTypeId: { value: { - types: { '.index-threshold': 2, 'logs.alert.document.count': 1, 'document.test.': 1 }, + ruleTypes: { + '.index-threshold': 2, + 'logs.alert.document.count': 1, + 'document.test.': 1, + }, + namespaces: { + default: 1, + }, }, }, }, @@ -39,6 +46,7 @@ Object { "document.test__": 1, "logs.alert.document.count": 1, }, + "countNamespaces": 1, "countTotal": 4, } `); diff --git a/x-pack/plugins/alerting/server/usage/alerts_telemetry.ts b/x-pack/plugins/alerting/server/usage/alerts_telemetry.ts index 7d8c1593f533d..18fa9b590b4e1 100644 --- a/x-pack/plugins/alerting/server/usage/alerts_telemetry.ts +++ b/x-pack/plugins/alerting/server/usage/alerts_telemetry.ts @@ -10,10 +10,14 @@ import { AlertsUsage } from './types'; const alertTypeMetric = { scripted_metric: { - init_script: 'state.types = [:]', + init_script: 'state.ruleTypes = [:]; state.namespaces = [:]', map_script: ` String alertType = doc['alert.alertTypeId'].value; - state.types.put(alertType, state.types.containsKey(alertType) ? state.types.get(alertType) + 1 : 1); + String namespace = doc['namespaces'] !== null ? doc['namespaces'].value : 'default'; + state.ruleTypes.put(alertType, state.ruleTypes.containsKey(alertType) ? state.ruleTypes.get(alertType) + 1 : 1); + if (state.namespaces.containsKey(namespace) === false) { + state.namespaces.put(namespace, 1); + } `, // Combine script is executed per cluster, but we already have a key-value pair per cluster. // Despite docs that say this is optional, this script can't be blank. @@ -40,7 +44,12 @@ export async function getTotalCountAggregations( ): Promise< Pick< AlertsUsage, - 'count_total' | 'count_by_type' | 'throttle_time' | 'schedule_time' | 'connectors_per_alert' + | 'count_total' + | 'count_by_type' + | 'throttle_time' + | 'schedule_time' + | 'connectors_per_alert' + | 'count_rules_namespaces' > > { const throttleTimeMetric = { @@ -247,7 +256,7 @@ export async function getTotalCountAggregations( }); const aggregations = results.aggregations as { - byAlertTypeId: { value: { types: Record } }; + byAlertTypeId: { value: { ruleTypes: Record } }; throttleTime: { value: { min: number; max: number; totalCount: number; totalSum: number } }; intervalTime: { value: { min: number; max: number; totalCount: number; totalSum: number } }; connectorsAgg: { @@ -257,20 +266,20 @@ export async function getTotalCountAggregations( }; }; - const totalAlertsCount = Object.keys(aggregations.byAlertTypeId.value.types).reduce( + const totalAlertsCount = Object.keys(aggregations.byAlertTypeId.value.ruleTypes).reduce( (total: number, key: string) => - parseInt(aggregations.byAlertTypeId.value.types[key], 10) + total, + parseInt(aggregations.byAlertTypeId.value.ruleTypes[key], 10) + total, 0 ); return { count_total: totalAlertsCount, - count_by_type: Object.keys(aggregations.byAlertTypeId.value.types).reduce( + count_by_type: Object.keys(aggregations.byAlertTypeId.value.ruleTypes).reduce( // ES DSL aggregations are returned as `any` by esClient.search // eslint-disable-next-line @typescript-eslint/no-explicit-any (obj: any, key: string) => ({ ...obj, - [replaceFirstAndLastDotSymbols(key)]: aggregations.byAlertTypeId.value.types[key], + [replaceFirstAndLastDotSymbols(key)]: aggregations.byAlertTypeId.value.ruleTypes[key], }), {} ), @@ -300,6 +309,7 @@ export async function getTotalCountAggregations( : 0, max: aggregations.connectorsAgg.connectors.value.max, }, + count_rules_namespaces: 0, }; } @@ -319,24 +329,27 @@ export async function getTotalCountInUse(esClient: ElasticsearchClient, kibanaIn }); const aggregations = searchResult.aggregations as { - byAlertTypeId: { value: { types: Record } }; + byAlertTypeId: { + value: { ruleTypes: Record; namespaces: Record }; + }; }; return { - countTotal: Object.keys(aggregations.byAlertTypeId.value.types).reduce( + countTotal: Object.keys(aggregations.byAlertTypeId.value.ruleTypes).reduce( (total: number, key: string) => - parseInt(aggregations.byAlertTypeId.value.types[key], 10) + total, + parseInt(aggregations.byAlertTypeId.value.ruleTypes[key], 10) + total, 0 ), - countByType: Object.keys(aggregations.byAlertTypeId.value.types).reduce( + countByType: Object.keys(aggregations.byAlertTypeId.value.ruleTypes).reduce( // ES DSL aggregations are returned as `any` by esClient.search // eslint-disable-next-line @typescript-eslint/no-explicit-any (obj: any, key: string) => ({ ...obj, - [replaceFirstAndLastDotSymbols(key)]: aggregations.byAlertTypeId.value.types[key], + [replaceFirstAndLastDotSymbols(key)]: aggregations.byAlertTypeId.value.ruleTypes[key], }), {} ), + countNamespaces: Object.keys(aggregations.byAlertTypeId.value.namespaces).length, }; } diff --git a/x-pack/plugins/alerting/server/usage/alerts_usage_collector.ts b/x-pack/plugins/alerting/server/usage/alerts_usage_collector.ts index 453a29b5884e6..ecea721dfad92 100644 --- a/x-pack/plugins/alerting/server/usage/alerts_usage_collector.ts +++ b/x-pack/plugins/alerting/server/usage/alerts_usage_collector.ts @@ -91,6 +91,7 @@ export function createAlertsUsageCollector( }, count_active_by_type: {}, count_by_type: {}, + count_rules_namespaces: 0, }; } }, @@ -115,6 +116,7 @@ export function createAlertsUsageCollector( }, count_active_by_type: byTypeSchema, count_by_type: byTypeSchema, + count_rules_namespaces: { type: 'long' }, }, }); } diff --git a/x-pack/plugins/alerting/server/usage/task.ts b/x-pack/plugins/alerting/server/usage/task.ts index 043d970ddd231..9d39b3765cb5d 100644 --- a/x-pack/plugins/alerting/server/usage/task.ts +++ b/x-pack/plugins/alerting/server/usage/task.ts @@ -89,6 +89,7 @@ export function telemetryTaskRunner(logger: Logger, core: CoreSetup, kibanaIndex count_active_by_type: totalInUse.countByType, count_active_total: totalInUse.countTotal, count_disabled_total: totalCountAggregations.count_total - totalInUse.countTotal, + count_rules_namespaces: totalInUse.countNamespaces, }, runAt: getNextMidnight(), }; diff --git a/x-pack/plugins/alerting/server/usage/types.ts b/x-pack/plugins/alerting/server/usage/types.ts index c3c750da73a7f..5e420b54e37cb 100644 --- a/x-pack/plugins/alerting/server/usage/types.ts +++ b/x-pack/plugins/alerting/server/usage/types.ts @@ -11,6 +11,7 @@ export interface AlertsUsage { count_disabled_total: number; count_by_type: Record; count_active_by_type: Record; + count_rules_namespaces: number; throttle_time: { min: string; avg: string; diff --git a/x-pack/plugins/task_manager/server/usage/task_manager_usage_collector.ts b/x-pack/plugins/task_manager/server/usage/task_manager_usage_collector.ts index 6e9891ecd6d65..d6eb92b944ce1 100644 --- a/x-pack/plugins/task_manager/server/usage/task_manager_usage_collector.ts +++ b/x-pack/plugins/task_manager/server/usage/task_manager_usage_collector.ts @@ -52,6 +52,15 @@ export function createTaskManagerUsageCollector( }, }, task_type_exclusion: excludeTaskTypes, + failed_tasks: Object.entries(lastMonitoredHealth?.stats.workload?.value.task_types!).reduce( + (numb, [key, val]) => { + if (val.status.failed !== undefined) { + numb += val.status.failed; + } + return numb; + }, + 0 + ), }; }, schema: { @@ -79,6 +88,7 @@ export function createTaskManagerUsageCollector( }, }, task_type_exclusion: { type: 'array', items: { type: 'keyword' } }, + failed_tasks: { type: 'long' }, }, }); } diff --git a/x-pack/plugins/task_manager/server/usage/types.ts b/x-pack/plugins/task_manager/server/usage/types.ts index 0acbfd1d4fab9..f9ac823a58124 100644 --- a/x-pack/plugins/task_manager/server/usage/types.ts +++ b/x-pack/plugins/task_manager/server/usage/types.ts @@ -30,4 +30,5 @@ export interface TaskManagerUsage { p99: number; }; }; + failed_tasks: number; } diff --git a/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json b/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json index 5bb559c137390..12763e4e26e31 100644 --- a/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json +++ b/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json @@ -11,15 +11,6 @@ "count_total": { "type": "long" }, - "count_active_total": { - "type": "long" - }, - "count_active_alert_history_connectors": { - "type": "long", - "_meta": { - "description": "The total number of preconfigured alert history connectors used by rules." - } - }, "count_by_type": { "properties": { "DYNAMIC_KEY": { @@ -60,6 +51,15 @@ } } }, + "count_active_total": { + "type": "long" + }, + "count_active_alert_history_connectors": { + "type": "long", + "_meta": { + "description": "The total number of preconfigured alert history connectors used by rules." + } + }, "count_active_by_type": { "properties": { "DYNAMIC_KEY": { @@ -99,6 +99,34 @@ "type": "long" } } + }, + "count_active_email_connectors_by_service_type": { + "properties": { + "DYNAMIC_KEY": { + "type": "long" + }, + "exchange_server": { + "type": "long" + }, + "gmail": { + "type": "long" + }, + "outlook365": { + "type": "long" + }, + "elastic_cloud": { + "type": "long" + }, + "other": { + "type": "long" + }, + "ses": { + "type": "long" + } + } + }, + "count_actions_namespaces": { + "type": "long" } } }, @@ -321,6 +349,9 @@ "type": "long" } } + }, + "count_rules_namespaces": { + "type": "long" } } }, @@ -7277,6 +7308,9 @@ "items": { "type": "keyword" } + }, + "failed_tasks": { + "type": "long" } } }, From fcb340fdb9220b5894f3da289428e02d9368364f Mon Sep 17 00:00:00 2001 From: Scotty Bollinger Date: Tue, 19 Oct 2021 13:39:47 -0500 Subject: [PATCH 2/6] Handle edge case when estimates are empty (#115476) There is an edge case where a connector does not sync and there is no estimate of next syncs in the UI. In this case, we simply omit the estimates from the UI. --- .../public/applications/workplace_search/types.ts | 2 +- .../synchronization/frequency_item.test.tsx | 6 ++++++ .../components/synchronization/frequency_item.tsx | 14 ++++++++++---- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/types.ts b/x-pack/plugins/enterprise_search/public/applications/workplace_search/types.ts index ab45c54cc5c57..2a982c2be1cfc 100644 --- a/x-pack/plugins/enterprise_search/public/applications/workplace_search/types.ts +++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/types.ts @@ -130,7 +130,7 @@ interface SourceActivity { export interface SyncEstimate { duration?: string; - nextStart: string; + nextStart?: string; lastRun?: string; } diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/synchronization/frequency_item.test.tsx b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/synchronization/frequency_item.test.tsx index 7498a185a80ec..12cd07ac23b7c 100644 --- a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/synchronization/frequency_item.test.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/synchronization/frequency_item.test.tsx @@ -122,4 +122,10 @@ describe('FrequencyItem', () => { expect(setSyncFrequency).toHaveBeenCalledWith('full', '3', 'minutes'); }); }); + + it('handles edge case where estimate is empty', () => { + const wrapper = shallow(); + + expect(wrapper.find('[data-test-subj="SyncEstimates"]')).toHaveLength(0); + }); }); diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/synchronization/frequency_item.tsx b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/synchronization/frequency_item.tsx index a51500e3076a8..f0066f06466a7 100644 --- a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/synchronization/frequency_item.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/synchronization/frequency_item.tsx @@ -53,6 +53,7 @@ export const FrequencyItem: React.FC = ({ const estimateDisplay = durationEstimate && moment.duration(durationEstimate).humanize(); const nextStartIsPast = moment().isAfter(nextStart); const nextStartTime = nextStartIsPast ? NEXT_SYNC_RUNNING_MESSAGE : moment(nextStart).fromNow(); + const showEstimates = lastRun || nextStart || durationEstimate; const frequencyItemLabel = ( = ({ - - - {lastRun && lastRunSummary} {nextStartSummary} {estimateDisplay && estimateSummary} - + {showEstimates && ( + <> + + + {lastRun && lastRunSummary} {nextStart && nextStartSummary}{' '} + {estimateDisplay && estimateSummary} + + + )} ); From 6f56668cc5a0dbddc984e8982d004afec4f1d403 Mon Sep 17 00:00:00 2001 From: Jason Stoltzfus Date: Tue, 19 Oct 2021 14:49:55 -0400 Subject: [PATCH 3/6] Fix style (#115439) --- .../views/curation_suggestion/curation_result_panel.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/views/curation_suggestion/curation_result_panel.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/views/curation_suggestion/curation_result_panel.tsx index bbd61804ad341..b8a659118e736 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/views/curation_suggestion/curation_result_panel.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/views/curation_suggestion/curation_result_panel.tsx @@ -73,7 +73,7 @@ export const CurationResultPanel: React.FC = ({ variant, results }) => { > {results.length > 0 ? ( results.map((result, index) => ( - + Date: Tue, 19 Oct 2021 11:50:23 -0700 Subject: [PATCH 4/6] skip flaky suite (#115614) --- x-pack/test/accessibility/apps/lens.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/test/accessibility/apps/lens.ts b/x-pack/test/accessibility/apps/lens.ts index ff769ddd29bfc..b8ddd774741b6 100644 --- a/x-pack/test/accessibility/apps/lens.ts +++ b/x-pack/test/accessibility/apps/lens.ts @@ -14,7 +14,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { const esArchiver = getService('esArchiver'); const listingTable = getService('listingTable'); - describe('Lens', () => { + // Failing: See https://github.com/elastic/kibana/issues/115614 + describe.skip('Lens', () => { const lensChartName = 'MyLensChart'; before(async () => { await esArchiver.loadIfNeeded('x-pack/test/functional/es_archives/logstash_functional'); From 30ce299589603523d05783a5d309eed23fe8f125 Mon Sep 17 00:00:00 2001 From: Georgii Gorbachev Date: Tue, 19 Oct 2021 20:57:55 +0200 Subject: [PATCH 5/6] [Security Solution][Detections] Enable writing rule execution events to Event Log by default (#115394) * Enable writing rule execution events to Event Log by default * Update event log provider name according to the RFC * Fix SavedObjectClient find method arguments Co-authored-by: Dmitry Shevchenko --- .../detection_alerts/building_block_alerts.spec.ts | 8 ++++++-- x-pack/plugins/security_solution/server/config.ts | 2 +- .../rule_execution_log/event_log_adapter/constants.ts | 2 +- .../rule_status_saved_objects_client.ts | 7 ++++++- .../saved_objects_adapter/saved_objects_adapter.ts | 3 +-- 5 files changed, 15 insertions(+), 7 deletions(-) diff --git a/x-pack/plugins/security_solution/cypress/integration/detection_alerts/building_block_alerts.spec.ts b/x-pack/plugins/security_solution/cypress/integration/detection_alerts/building_block_alerts.spec.ts index 262ffe8163e57..94418e61b4053 100644 --- a/x-pack/plugins/security_solution/cypress/integration/detection_alerts/building_block_alerts.spec.ts +++ b/x-pack/plugins/security_solution/cypress/integration/detection_alerts/building_block_alerts.spec.ts @@ -8,19 +8,23 @@ import { getBuildingBlockRule } from '../../objects/rule'; import { OVERVIEW_ALERTS_HISTOGRAM } from '../../screens/overview'; import { OVERVIEW } from '../../screens/security_header'; +import { waitForAlertsIndexToBeCreated, waitForAlertsPanelToBeLoaded } from '../../tasks/alerts'; import { goToRuleDetails } from '../../tasks/alerts_detection_rules'; import { createCustomRuleActivated } from '../../tasks/api_calls/rules'; import { cleanKibana } from '../../tasks/common'; import { waitForAlertsToPopulate, waitForTheRuleToBeExecuted } from '../../tasks/create_new_rule'; -import { loginAndWaitForPage } from '../../tasks/login'; +import { loginAndWaitForPage, loginAndWaitForPageWithoutDateRange } from '../../tasks/login'; import { navigateFromHeaderTo } from '../../tasks/security_header'; -import { DETECTIONS_RULE_MANAGEMENT_URL } from '../../urls/navigation'; +import { ALERTS_URL, DETECTIONS_RULE_MANAGEMENT_URL } from '../../urls/navigation'; const EXPECTED_NUMBER_OF_ALERTS = 16; describe('Alerts generated by building block rules', () => { beforeEach(() => { cleanKibana(); + loginAndWaitForPageWithoutDateRange(ALERTS_URL); + waitForAlertsPanelToBeLoaded(); + waitForAlertsIndexToBeCreated(); }); it('Alerts should be visible on the Rule Detail page and not visible on the Overview page', () => { diff --git a/x-pack/plugins/security_solution/server/config.ts b/x-pack/plugins/security_solution/server/config.ts index e0b8ad883f4a2..61cbb5641c5f6 100644 --- a/x-pack/plugins/security_solution/server/config.ts +++ b/x-pack/plugins/security_solution/server/config.ts @@ -112,7 +112,7 @@ export const configSchema = schema.object({ schema.literal(UnderlyingLogClient.eventLog), schema.literal(UnderlyingLogClient.savedObjects), ], - { defaultValue: UnderlyingLogClient.savedObjects } + { defaultValue: UnderlyingLogClient.eventLog } ), }), diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/event_log_adapter/constants.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/event_log_adapter/constants.ts index f09eb43bf15f1..55624b56e39a0 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/event_log_adapter/constants.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/event_log_adapter/constants.ts @@ -5,7 +5,7 @@ * 2.0. */ -export const RULE_EXECUTION_LOG_PROVIDER = 'rule-execution.security'; +export const RULE_EXECUTION_LOG_PROVIDER = 'securitySolution.ruleExecution'; export const ALERT_SAVED_OBJECT_TYPE = 'alert'; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/saved_objects_adapter/rule_status_saved_objects_client.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/saved_objects_adapter/rule_status_saved_objects_client.ts index 66b646e96ea53..0026bba24eebe 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/saved_objects_adapter/rule_status_saved_objects_client.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/saved_objects_adapter/rule_status_saved_objects_client.ts @@ -21,7 +21,7 @@ import { IRuleStatusSOAttributes } from '../../rules/types'; export interface RuleStatusSavedObjectsClient { find: ( - options?: Omit + options: Omit & { ruleId: string } ) => Promise>>; findBulk: (ids: string[], statusesPerId: number) => Promise; create: ( @@ -47,9 +47,14 @@ export const ruleStatusSavedObjectsClientFactory = ( savedObjectsClient: SavedObjectsClientContract ): RuleStatusSavedObjectsClient => ({ find: async (options) => { + const references = { + id: options.ruleId, + type: 'alert', + }; const result = await savedObjectsClient.find({ ...options, type: legacyRuleStatusSavedObjectType, + hasReference: references, }); return result.saved_objects; }, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/saved_objects_adapter/saved_objects_adapter.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/saved_objects_adapter/saved_objects_adapter.ts index 9db7afce62ee4..70db3a768fdb1 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/saved_objects_adapter/saved_objects_adapter.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/saved_objects_adapter/saved_objects_adapter.ts @@ -53,8 +53,7 @@ export class SavedObjectsAdapter implements IRuleExecutionLogClient { perPage: logsCount, sortField: 'statusDate', sortOrder: 'desc', - search: ruleId, - searchFields: ['references.id'], + ruleId, }); } From f2ef8b396440275bf996264ee701a60e76effc13 Mon Sep 17 00:00:00 2001 From: Tyler Smalley Date: Tue, 19 Oct 2021 12:19:26 -0700 Subject: [PATCH 6/6] skip flaky suite (#92528) --- .../public/ui/query_string_input/query_bar_top_row.test.tsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/plugins/data/public/ui/query_string_input/query_bar_top_row.test.tsx b/src/plugins/data/public/ui/query_string_input/query_bar_top_row.test.tsx index b7ec5a1f0c286..0541e12cf8172 100644 --- a/src/plugins/data/public/ui/query_string_input/query_bar_top_row.test.tsx +++ b/src/plugins/data/public/ui/query_string_input/query_bar_top_row.test.tsx @@ -103,7 +103,8 @@ function wrapQueryBarTopRowInContext(testProps: any) { ); } -describe('QueryBarTopRowTopRow', () => { +// Failing: See https://github.com/elastic/kibana/issues/92528 +describe.skip('QueryBarTopRowTopRow', () => { const QUERY_INPUT_SELECTOR = 'QueryStringInputUI'; const TIMEPICKER_SELECTOR = 'EuiSuperDatePicker'; const TIMEPICKER_DURATION = '[data-shared-timefilter-duration]';